[lxc-devel] [lxd/master] Storage cleanup
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Thu May 2 15:07:17 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190502/74a7eb4c/attachment-0001.bin>
-------------- next part --------------
From 26d6a52c7a246348da9b69a3ddf3efae187198c4 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 14:46:17 +0200
Subject: [PATCH 01/15] lxd: Add pretty logging function
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/logging.go | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/lxd/logging.go b/lxd/logging.go
index ab244ad5ca..3c16db0a4e 100644
--- a/lxd/logging.go
+++ b/lxd/logging.go
@@ -115,7 +115,7 @@ func expireLogs(ctx context.Context, state *state.State) error {
if logfile.IsDir() {
newest := newestFile(path, logfile)
if time.Since(newest).Hours() >= 48 {
- os.RemoveAll(path)
+ err := os.RemoveAll(path)
if err != nil {
return err
}
@@ -147,3 +147,19 @@ func expireLogs(ctx context.Context, state *state.State) error {
return nil
}
+
+func logAction(infoMsg, successMsg, errorMsg string, ctx *log.Ctx, success *bool, err *error) func() {
+ log.Info(infoMsg, ctx)
+
+ return func() {
+ if *success {
+ log.Info(successMsg, ctx)
+ } else {
+ if (*err) != nil {
+ (*ctx)["error"] = (*err).Error()
+ }
+
+ log.Error(errorMsg, ctx)
+ }
+ }
+}
From 31945bb83af8f2a8596a13c5b14342c0d410b557 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 14:51:19 +0200
Subject: [PATCH 02/15] storage: Remove shared code from backends
This removes common code from the storage backends to the shared code
section.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage_btrfs.go | 29 -----------------------------
lxd/storage_ceph.go | 29 -----------------------------
lxd/storage_dir.go | 29 -----------------------------
lxd/storage_lvm.go | 30 +-----------------------------
lxd/storage_shared.go | 28 ++++++++++++++++++++++++++++
lxd/storage_zfs.go | 29 -----------------------------
6 files changed, 29 insertions(+), 145 deletions(-)
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 3d44e04fff..60b44d12c8 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -18,7 +18,6 @@ import (
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/migration"
- "github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -536,14 +535,6 @@ func (s *storageBtrfs) StoragePoolUpdate(writable *api.StoragePoolPut,
return nil
}
-func (s *storageBtrfs) GetStoragePoolWritable() api.StoragePoolPut {
- return s.pool.Writable()
-}
-
-func (s *storageBtrfs) SetStoragePoolWritable(writable *api.StoragePoolPut) {
- s.pool.StoragePoolPut = *writable
-}
-
func (s *storageBtrfs) GetContainerPoolInfo() (int64, string, string) {
return s.poolID, s.pool.Name, s.pool.Name
}
@@ -805,14 +796,6 @@ func (s *storageBtrfs) StoragePoolVolumeRename(newName string) error {
return nil
}
-func (s *storageBtrfs) GetStoragePoolVolumeWritable() api.StorageVolumePut {
- return s.volume.Writable()
-}
-
-func (s *storageBtrfs) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
- s.volume.StorageVolumePut = *writable
-}
-
// Functions dealing with container storage.
func (s *storageBtrfs) ContainerStorageReady(container container) bool {
containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
@@ -3098,18 +3081,6 @@ func (s *storageBtrfs) StorageMigrationSink(conn *websocket.Conn, op *operation,
return rsyncStorageMigrationSink(conn, op, args)
}
-func (s *storageBtrfs) GetStoragePool() *api.StoragePool {
- return s.pool
-}
-
-func (s *storageBtrfs) GetStoragePoolVolume() *api.StorageVolume {
- return s.volume
-}
-
-func (s *storageBtrfs) GetState() *state.State {
- return s.s
-}
-
func (s *storageBtrfs) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
logger.Infof("Creating BTRFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index 9e0be80504..c5ffe0e71c 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -14,7 +14,6 @@ import (
"github.com/pkg/errors"
"github.com/lxc/lxd/lxd/db"
- "github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/ioprogress"
@@ -314,22 +313,6 @@ func (s *storageCeph) StoragePoolUmount() (bool, error) {
return true, nil
}
-func (s *storageCeph) GetStoragePoolWritable() api.StoragePoolPut {
- return s.pool.StoragePoolPut
-}
-
-func (s *storageCeph) GetStoragePoolVolumeWritable() api.StorageVolumePut {
- return s.volume.Writable()
-}
-
-func (s *storageCeph) SetStoragePoolWritable(writable *api.StoragePoolPut) {
- s.pool.StoragePoolPut = *writable
-}
-
-func (s *storageCeph) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
- s.volume.StorageVolumePut = *writable
-}
-
func (s *storageCeph) GetContainerPoolInfo() (int64, string, string) {
return s.poolID, s.pool.Name, s.OSDPoolName
}
@@ -2729,18 +2712,6 @@ func (s *storageCeph) StorageMigrationSink(conn *websocket.Conn, op *operation,
return rsyncStorageMigrationSink(conn, op, args)
}
-func (s *storageCeph) GetStoragePool() *api.StoragePool {
- return s.pool
-}
-
-func (s *storageCeph) GetStoragePoolVolume() *api.StorageVolume {
- return s.volume
-}
-
-func (s *storageCeph) GetState() *state.State {
- return s.s
-}
-
func (s *storageCeph) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
logger.Debugf("Creating RBD storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
sourcePath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index b7635fd007..af0a22b890 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -13,7 +13,6 @@ import (
"github.com/pkg/errors"
"github.com/lxc/lxd/lxd/migration"
- "github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/storage/quota"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -277,22 +276,6 @@ func (s *storageDir) StoragePoolUmount() (bool, error) {
return true, nil
}
-func (s *storageDir) GetStoragePoolWritable() api.StoragePoolPut {
- return s.pool.Writable()
-}
-
-func (s *storageDir) GetStoragePoolVolumeWritable() api.StorageVolumePut {
- return s.volume.Writable()
-}
-
-func (s *storageDir) SetStoragePoolWritable(writable *api.StoragePoolPut) {
- s.pool.StoragePoolPut = *writable
-}
-
-func (s *storageDir) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
- s.volume.StorageVolumePut = *writable
-}
-
func (s *storageDir) GetContainerPoolInfo() (int64, string, string) {
return s.poolID, s.pool.Name, s.pool.Name
}
@@ -1446,18 +1429,6 @@ func (s *storageDir) StorageMigrationSink(conn *websocket.Conn, op *operation, a
return rsyncStorageMigrationSink(conn, op, args)
}
-func (s *storageDir) GetStoragePool() *api.StoragePool {
- return s.pool
-}
-
-func (s *storageDir) GetStoragePoolVolume() *api.StorageVolume {
- return s.volume
-}
-
-func (s *storageDir) GetState() *state.State {
- return s.s
-}
-
func (s *storageDir) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
logger.Infof("Creating DIR storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index 778c114e52..e4c8e4c444 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -14,7 +14,7 @@ import (
"github.com/pkg/errors"
"github.com/lxc/lxd/lxd/migration"
- "github.com/lxc/lxd/lxd/state"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/ioprogress"
@@ -679,22 +679,6 @@ func (s *storageLvm) StoragePoolVolumeUmount() (bool, error) {
return ourUmount, nil
}
-func (s *storageLvm) GetStoragePoolWritable() api.StoragePoolPut {
- return s.pool.Writable()
-}
-
-func (s *storageLvm) GetStoragePoolVolumeWritable() api.StorageVolumePut {
- return s.volume.Writable()
-}
-
-func (s *storageLvm) SetStoragePoolWritable(writable *api.StoragePoolPut) {
- s.pool.StoragePoolPut = *writable
-}
-
-func (s *storageLvm) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
- s.volume.StorageVolumePut = *writable
-}
-
func (s *storageLvm) GetContainerPoolInfo() (int64, string, string) {
return s.poolID, s.pool.Name, s.getOnDiskPoolName()
}
@@ -2256,18 +2240,6 @@ func (s *storageLvm) StorageMigrationSink(conn *websocket.Conn, op *operation, a
return rsyncStorageMigrationSink(conn, op, args)
}
-func (s *storageLvm) GetStoragePool() *api.StoragePool {
- return s.pool
-}
-
-func (s *storageLvm) GetStoragePoolVolume() *api.StorageVolume {
- return s.volume
-}
-
-func (s *storageLvm) GetState() *state.State {
- return s.s
-}
-
func (s *storageLvm) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
logger.Debugf("Creating LVM storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
diff --git a/lxd/storage_shared.go b/lxd/storage_shared.go
index 8fb89b2da6..11875f06ec 100644
--- a/lxd/storage_shared.go
+++ b/lxd/storage_shared.go
@@ -30,6 +30,34 @@ func (s *storageShared) GetStorageTypeVersion() string {
return s.sTypeVersion
}
+func (s *storageShared) GetStoragePool() *api.StoragePool {
+ return s.pool
+}
+
+func (s *storageShared) GetStoragePoolVolume() *api.StorageVolume {
+ return s.volume
+}
+
+func (s *storageShared) GetState() *state.State {
+ return s.s
+}
+
+func (s *storageShared) GetStoragePoolWritable() api.StoragePoolPut {
+ return s.pool.Writable()
+}
+
+func (s *storageShared) GetStoragePoolVolumeWritable() api.StorageVolumePut {
+ return s.volume.Writable()
+}
+
+func (s *storageShared) SetStoragePoolWritable(writable *api.StoragePoolPut) {
+ s.pool.StoragePoolPut = *writable
+}
+
+func (s *storageShared) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
+ s.volume.StorageVolumePut = *writable
+}
+
func (s *storageShared) createImageDbPoolVolume(fingerprint string) error {
// Fill in any default volume config.
volumeConfig := map[string]string{}
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 5667c557ae..cc35cd298a 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -15,7 +15,6 @@ import (
"github.com/pkg/errors"
"github.com/lxc/lxd/lxd/migration"
- "github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -620,22 +619,6 @@ func (s *storageZfs) StoragePoolVolumeUmount() (bool, error) {
return ourUmount, nil
}
-func (s *storageZfs) GetStoragePoolWritable() api.StoragePoolPut {
- return s.pool.Writable()
-}
-
-func (s *storageZfs) GetStoragePoolVolumeWritable() api.StorageVolumePut {
- return s.volume.Writable()
-}
-
-func (s *storageZfs) SetStoragePoolWritable(writable *api.StoragePoolPut) {
- s.pool.StoragePoolPut = *writable
-}
-
-func (s *storageZfs) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
- s.volume.StorageVolumePut = *writable
-}
-
func (s *storageZfs) GetContainerPoolInfo() (int64, string, string) {
return s.poolID, s.pool.Name, s.getOnDiskPoolName()
}
@@ -3375,18 +3358,6 @@ func (s *storageZfs) StorageMigrationSink(conn *websocket.Conn, op *operation, a
return rsyncStorageMigrationSink(conn, op, args)
}
-func (s *storageZfs) GetStoragePool() *api.StoragePool {
- return s.pool
-}
-
-func (s *storageZfs) GetStoragePoolVolume() *api.StorageVolume {
- return s.volume
-}
-
-func (s *storageZfs) GetState() *state.State {
- return s.s
-}
-
func (s *storageZfs) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
logger.Infof("Creating ZFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
From d3e563dc269ccad9aad7ff6603bbe804937fa230 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 14:55:54 +0200
Subject: [PATCH 03/15] lxd: Remove ContainerCanRestore from storage interface
The function ContainerCanRestore is only used by zfs, and therefore
should be zfs specific.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/container_lxc.go | 6 ------
lxd/storage.go | 1 -
lxd/storage_btrfs.go | 4 ----
lxd/storage_ceph.go | 4 ----
lxd/storage_dir.go | 4 ----
lxd/storage_lvm.go | 4 ----
lxd/storage_mock.go | 4 ----
lxd/storage_zfs.go | 48 ++++++++++++++++++--------------------------
8 files changed, 19 insertions(+), 56 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 68b238e9c6..d44ccfc872 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -3370,12 +3370,6 @@ func (c *containerLXC) Restore(sourceContainer container, stateful bool) error {
defer c.StorageStop()
}
- // Check if we can restore the container
- err = c.storage.ContainerCanRestore(c, sourceContainer)
- if err != nil {
- return err
- }
-
/* let's also check for CRIU if necessary, before doing a bunch of
* filesystem manipulations
*/
diff --git a/lxd/storage.go b/lxd/storage.go
index 2e07d53039..c825b146fb 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -175,7 +175,6 @@ type storage interface {
// ContainerCreateFromImage creates a container from a image.
ContainerCreateFromImage(c container, fingerprint string, tracker *ioprogress.ProgressTracker) error
- ContainerCanRestore(target container, source container) error
ContainerDelete(c container) error
ContainerCopy(target container, source container, containerOnly bool) error
ContainerRefresh(target container, source container, snapshots []container) error
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 60b44d12c8..48553ec478 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -935,10 +935,6 @@ func (s *storageBtrfs) ContainerCreateFromImage(container container, fingerprint
return nil
}
-func (s *storageBtrfs) ContainerCanRestore(container container, sourceContainer container) error {
- return nil
-}
-
func (s *storageBtrfs) ContainerDelete(container container) error {
logger.Debugf("Deleting BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index c5ffe0e71c..02d89e8bd5 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -958,10 +958,6 @@ func (s *storageCeph) ContainerCreateFromImage(container container, fingerprint
return nil
}
-func (s *storageCeph) ContainerCanRestore(container container, sourceContainer container) error {
- return nil
-}
-
func (s *storageCeph) ContainerDelete(container container) error {
containerName := container.Name()
logger.Debugf(`Deleting RBD storage volume for container "%s" on storage pool "%s"`, containerName, s.pool.Name)
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index af0a22b890..788b4a0245 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -565,10 +565,6 @@ func (s *storageDir) ContainerCreateFromImage(container container, imageFingerpr
return nil
}
-func (s *storageDir) ContainerCanRestore(container container, sourceContainer container) error {
- return nil
-}
-
func (s *storageDir) ContainerDelete(container container) error {
logger.Debugf("Deleting DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index e4c8e4c444..fae1d97a37 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -1061,10 +1061,6 @@ func (s *storageLvm) ContainerCreateFromImage(container container, fingerprint s
return nil
}
-func (s *storageLvm) ContainerCanRestore(container container, sourceContainer container) error {
- return nil
-}
-
func lvmContainerDeleteInternal(project, poolName string, ctName string, isSnapshot bool, vgName string, ctPath string) error {
containerMntPoint := ""
containerLvmName := containerNameToLVName(ctName)
diff --git a/lxd/storage_mock.go b/lxd/storage_mock.go
index 58c993f511..950aa4d215 100644
--- a/lxd/storage_mock.go
+++ b/lxd/storage_mock.go
@@ -123,10 +123,6 @@ func (s *storageMock) ContainerCreateFromImage(
return nil
}
-func (s *storageMock) ContainerCanRestore(container container, sourceContainer container) error {
- return nil
-}
-
func (s *storageMock) ContainerDelete(container container) error {
return nil
}
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index cc35cd298a..d22092b8f1 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -913,29 +913,6 @@ func (s *storageZfs) ContainerCreateFromImage(container container, fingerprint s
return nil
}
-func (s *storageZfs) ContainerCanRestore(container container, sourceContainer container) error {
- snaps, err := container.Snapshots()
- if err != nil {
- return err
- }
-
- if snaps[len(snaps)-1].Name() != sourceContainer.Name() {
- if s.pool.Config["volume.zfs.remove_snapshots"] != "" {
- zfsRemoveSnapshots = s.pool.Config["volume.zfs.remove_snapshots"]
- }
- if s.volume.Config["zfs.remove_snapshots"] != "" {
- zfsRemoveSnapshots = s.volume.Config["zfs.remove_snapshots"]
- }
- if !shared.IsTrue(zfsRemoveSnapshots) {
- return fmt.Errorf("ZFS can only restore from the latest snapshot. Delete newer snapshots or copy the snapshot into a new container instead")
- }
-
- return nil
- }
-
- return nil
-}
-
func (s *storageZfs) ContainerDelete(container container) error {
err := s.doContainerDelete(container.Project(), container.Name())
if err != nil {
@@ -1501,6 +1478,25 @@ func (s *storageZfs) ContainerRename(container container, newName string) error
func (s *storageZfs) ContainerRestore(target container, source container) error {
logger.Debugf("Restoring ZFS storage volume for container \"%s\" from %s to %s", s.volume.Name, source.Name(), target.Name())
+ snaps, err := target.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ if snaps[len(snaps)-1].Name() != source.Name() {
+ if s.pool.Config["volume.zfs.remove_snapshots"] != "" {
+ zfsRemoveSnapshots = s.pool.Config["volume.zfs.remove_snapshots"]
+ }
+
+ if s.volume.Config["zfs.remove_snapshots"] != "" {
+ zfsRemoveSnapshots = s.volume.Config["zfs.remove_snapshots"]
+ }
+
+ if !shared.IsTrue(zfsRemoveSnapshots) {
+ return fmt.Errorf("ZFS can only restore from the latest snapshot. Delete newer snapshots or copy the snapshot into a new container instead")
+ }
+ }
+
// Start storage for source container
ourSourceStart, err := source.StorageStart()
if err != nil {
@@ -1519,12 +1515,6 @@ func (s *storageZfs) ContainerRestore(target container, source container) error
defer target.StorageStop()
}
- // Remove any needed snapshot
- snaps, err := target.Snapshots()
- if err != nil {
- return err
- }
-
for i := len(snaps) - 1; i != 0; i-- {
if snaps[i].Name() == source.Name() {
break
From a880d879723b8c5ace62aa6cfcd7f8fce6212d54 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 16:05:56 +0200
Subject: [PATCH 04/15] lxd: Remove Image{Umount,Mount} from storage interface
These functions are not called from anywhere outside of the actual
storage backend code.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage.go | 2 --
1 file changed, 2 deletions(-)
diff --git a/lxd/storage.go b/lxd/storage.go
index c825b146fb..9df1e51632 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -201,8 +201,6 @@ type storage interface {
// Functions dealing with image storage volumes.
ImageCreate(fingerprint string, tracker *ioprogress.ProgressTracker) error
ImageDelete(fingerprint string) error
- ImageMount(fingerprint string) (bool, error)
- ImageUmount(fingerprint string) (bool, error)
// Storage type agnostic functions.
StorageEntitySetQuota(volumeType int, size int64, data interface{}) error
From 1e8855d03192577b3d43334065fb120d3b9e2e10 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 14:58:46 +0200
Subject: [PATCH 05/15] lxd: Add project argument to containerPath function
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/api_internal.go | 4 ++--
lxd/container.go | 6 +++---
lxd/container_lxc.go | 3 +--
lxd/container_test.go | 4 ++--
lxd/storage_dir.go | 7 ++++---
lxd/storage_zfs.go | 2 +-
6 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index 508801f243..fb47c4ef58 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -673,7 +673,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
onDiskPoolName = poolName
}
snapName := fmt.Sprintf("%s/%s", req.Name, od)
- snapPath := containerPath(snapName, true)
+ snapPath := containerPath(project, snapName, true)
err = lvmContainerDeleteInternal(project, poolName, req.Name,
true, onDiskPoolName, snapPath)
case "ceph":
@@ -1015,7 +1015,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
return SmartError(err)
}
- containerPath := containerPath(projectPrefix(project, req.Name), false)
+ containerPath := containerPath(project, req.Name, false)
isPrivileged := false
if backup.Container.Config["security.privileged"] == "" {
isPrivileged = true
diff --git a/lxd/container.go b/lxd/container.go
index 24da0f924d..940a95e617 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -45,12 +45,12 @@ func containerGetParentAndSnapshotName(name string) (string, string, bool) {
return fields[0], fields[1], true
}
-func containerPath(name string, isSnapshot bool) string {
+func containerPath(project string, name string, isSnapshot bool) string {
if isSnapshot {
- return shared.VarPath("snapshots", name)
+ return shared.VarPath("snapshots", projectPrefix(project, name))
}
- return shared.VarPath("containers", name)
+ return shared.VarPath("containers", projectPrefix(project, name))
}
func containerValidName(name string) error {
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index d44ccfc872..373e4ea0ba 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -8900,8 +8900,7 @@ func (c *containerLXC) State() string {
// Various container paths
func (c *containerLXC) Path() string {
- name := projectPrefix(c.Project(), c.Name())
- return containerPath(name, c.IsSnapshot())
+ return containerPath(c.Project(), c.Name(), c.IsSnapshot())
}
func (c *containerLXC) DevicesPath() string {
diff --git a/lxd/container_test.go b/lxd/container_test.go
index caa3adebe0..6bfa73fc36 100644
--- a/lxd/container_test.go
+++ b/lxd/container_test.go
@@ -160,7 +160,7 @@ func (suite *containerTestSuite) TestContainer_Path_Regular() {
suite.Req.False(c.IsSnapshot(), "Shouldn't be a snapshot.")
suite.Req.Equal(shared.VarPath("containers", "testFoo"), c.Path())
- suite.Req.Equal(shared.VarPath("containers", "testFoo2"), containerPath("testFoo2", false))
+ suite.Req.Equal(shared.VarPath("containers", "testFoo2"), containerPath("default", "testFoo2", false))
}
func (suite *containerTestSuite) TestContainer_Path_Snapshot() {
@@ -181,7 +181,7 @@ func (suite *containerTestSuite) TestContainer_Path_Snapshot() {
c.Path())
suite.Req.Equal(
shared.VarPath("snapshots", "test", "snap1"),
- containerPath("test/snap1", true))
+ containerPath("default", "test/snap1", true))
}
func (suite *containerTestSuite) TestContainer_LogPath() {
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index 788b4a0245..ca1b0a6949 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -817,9 +817,10 @@ func (s *storageDir) ContainerRename(container container, newName string) error
}
oldContainerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- oldContainerSymlink := shared.VarPath("containers", projectPrefix(container.Project(), container.Name()))
+ oldContainerSymlink := containerPath(container.Project(), container.Name(), false)
newContainerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, newName)
- newContainerSymlink := shared.VarPath("containers", projectPrefix(container.Project(), newName))
+ newContainerSymlink := containerPath(container.Project(), newName, false)
+
err = renameContainerMountpoint(oldContainerMntPoint, oldContainerSymlink, newContainerMntPoint, newContainerSymlink)
if err != nil {
return err
@@ -1201,7 +1202,7 @@ func (s *storageDir) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, ta
// Create mountpoints
containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, info.Name)
- err = createContainerMountpoint(containerMntPoint, containerPath(projectPrefix(info.Project, info.Name), false), info.Privileged)
+ err = createContainerMountpoint(containerMntPoint, containerPath(info.Project, info.Name, false), info.Privileged)
if err != nil {
return errors.Wrap(err, "Create container mount point")
}
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index d22092b8f1..f049deb595 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -2129,7 +2129,7 @@ func (s *storageZfs) ContainerBackupCreate(backup backup, source container) erro
func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
containerName, _, _ := containerGetParentAndSnapshotName(info.Name)
containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, containerName)
- err := createContainerMountpoint(containerMntPoint, containerPath(info.Name, false), info.Privileged)
+ err := createContainerMountpoint(containerMntPoint, containerPath(info.Project, info.Name, false), info.Privileged)
if err != nil {
return err
}
From b562ab8b1091585104e3d02a9c0307f3fa13cba6 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 15:04:28 +0200
Subject: [PATCH 06/15] lxd: Move storage gco to storage package
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/{ => storage}/storage_cgo.go | 12 +-
lxd/storage/utils.go | 405 +++++++++++++++++++++++++++++++
lxd/storage_btrfs.go | 3 +-
lxd/storage_lvm.go | 6 +-
lxd/storage_utils.go | 3 +-
5 files changed, 418 insertions(+), 11 deletions(-)
rename lxd/{ => storage}/storage_cgo.go (96%)
create mode 100644 lxd/storage/utils.go
diff --git a/lxd/storage_cgo.go b/lxd/storage/storage_cgo.go
similarity index 96%
rename from lxd/storage_cgo.go
rename to lxd/storage/storage_cgo.go
index 1f1c7136f7..b770710cb0 100644
--- a/lxd/storage_cgo.go
+++ b/lxd/storage/storage_cgo.go
@@ -1,7 +1,7 @@
// +build linux
// +build cgo
-package main
+package storage
/*
#define _GNU_SOURCE
@@ -19,8 +19,8 @@ package main
#include <sys/stat.h>
#include <sys/types.h>
-#include "include/macro.h"
-#include "include/memory_utils.h"
+#include "../include/macro.h"
+#include "../include/memory_utils.h"
#ifndef MS_LAZYTIME
#define MS_LAZYTIME (1<<25)
@@ -267,7 +267,7 @@ const MS_LAZYTIME uintptr = C.MS_LAZYTIME
// prepareLoopDev() detects and sets up a loop device for source. It returns an
// open file descriptor to the free loop device and the path of the free loop
// device. It's the callers responsibility to close the open file descriptor.
-func prepareLoopDev(source string, flags int) (*os.File, error) {
+func PrepareLoopDev(source string, flags int) (*os.File, error) {
cLoopDev := C.malloc(C.size_t(C.LO_NAME_SIZE))
if cLoopDev == nil {
return nil, fmt.Errorf("Failed to allocate memory in C")
@@ -293,7 +293,7 @@ func prepareLoopDev(source string, flags int) (*os.File, error) {
return os.NewFile(uintptr(loopFd), C.GoString((*C.char)(cLoopDev))), nil
}
-func setAutoclearOnLoopDev(loopFd int) error {
+func SetAutoclearOnLoopDev(loopFd int) error {
ret, err := C.set_autoclear_loop_device(C.int(loopFd))
if ret < 0 {
if err != nil {
@@ -305,7 +305,7 @@ func setAutoclearOnLoopDev(loopFd int) error {
return nil
}
-func unsetAutoclearOnLoopDev(loopFd int) error {
+func UnsetAutoclearOnLoopDev(loopFd int) error {
ret, err := C.unset_autoclear_loop_device(C.int(loopFd))
if ret < 0 {
if err != nil {
diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go
new file mode 100644
index 0000000000..2fa57b55fc
--- /dev/null
+++ b/lxd/storage/utils.go
@@ -0,0 +1,405 @@
+package storage
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/idmap"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+// Options for filesystem creation
+type mkfsOptions struct {
+ label string
+}
+
+// Export the mount options map since we might find it useful in other parts of
+// LXD.
+type mountOptions struct {
+ capture bool
+ flag uintptr
+}
+
+var MountOptions = map[string]mountOptions{
+ "async": {false, syscall.MS_SYNCHRONOUS},
+ "atime": {false, syscall.MS_NOATIME},
+ "bind": {true, syscall.MS_BIND},
+ "defaults": {true, 0},
+ "dev": {false, syscall.MS_NODEV},
+ "diratime": {false, syscall.MS_NODIRATIME},
+ "dirsync": {true, syscall.MS_DIRSYNC},
+ "exec": {false, syscall.MS_NOEXEC},
+ "lazytime": {true, MS_LAZYTIME},
+ "mand": {true, syscall.MS_MANDLOCK},
+ "noatime": {true, syscall.MS_NOATIME},
+ "nodev": {true, syscall.MS_NODEV},
+ "nodiratime": {true, syscall.MS_NODIRATIME},
+ "noexec": {true, syscall.MS_NOEXEC},
+ "nomand": {false, syscall.MS_MANDLOCK},
+ "norelatime": {false, syscall.MS_RELATIME},
+ "nostrictatime": {false, syscall.MS_STRICTATIME},
+ "nosuid": {true, syscall.MS_NOSUID},
+ "rbind": {true, syscall.MS_BIND | syscall.MS_REC},
+ "relatime": {true, syscall.MS_RELATIME},
+ "remount": {true, syscall.MS_REMOUNT},
+ "ro": {true, syscall.MS_RDONLY},
+ "rw": {false, syscall.MS_RDONLY},
+ "strictatime": {true, syscall.MS_STRICTATIME},
+ "suid": {false, syscall.MS_NOSUID},
+ "sync": {true, syscall.MS_SYNCHRONOUS},
+}
+
+func lxdResolveMountoptions(options string) (uintptr, string) {
+ mountFlags := uintptr(0)
+ tmp := strings.SplitN(options, ",", -1)
+ for i := 0; i < len(tmp); i++ {
+ opt := tmp[i]
+ do, ok := MountOptions[opt]
+ if !ok {
+ continue
+ }
+
+ if do.capture {
+ mountFlags |= do.flag
+ } else {
+ mountFlags &= ^do.flag
+ }
+
+ copy(tmp[i:], tmp[i+1:])
+ tmp[len(tmp)-1] = ""
+ tmp = tmp[:len(tmp)-1]
+ i--
+ }
+
+ return mountFlags, strings.Join(tmp, ",")
+}
+
+// Useful functions for unreliable backends
+func tryMount(src string, dst string, fs string, flags uintptr, options string) error {
+ var err error
+
+ for i := 0; i < 20; i++ {
+ err = syscall.Mount(src, dst, fs, flags, options)
+ if err == nil {
+ break
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func tryUnmount(path string, flags int) error {
+ var err error
+
+ for i := 0; i < 20; i++ {
+ err = syscall.Unmount(path, flags)
+ if err == nil {
+ break
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ if err != nil && err == syscall.EBUSY {
+ return err
+ }
+
+ return nil
+}
+
+func storageValidName(value string) error {
+ if shared.IsSnapshot(value) {
+ return fmt.Errorf("Invalid storage volume name \"%s\". Storage volumes cannot contain \"/\" in their name", value)
+ }
+
+ return nil
+}
+
+func storageConfigDiff(oldConfig map[string]string, newConfig map[string]string) ([]string, bool) {
+ changedConfig := []string{}
+ userOnly := true
+ for key := range oldConfig {
+ if oldConfig[key] != newConfig[key] {
+ if !strings.HasPrefix(key, "user.") {
+ userOnly = false
+ }
+
+ if !shared.StringInSlice(key, changedConfig) {
+ changedConfig = append(changedConfig, key)
+ }
+ }
+ }
+
+ for key := range newConfig {
+ if oldConfig[key] != newConfig[key] {
+ if !strings.HasPrefix(key, "user.") {
+ userOnly = false
+ }
+
+ if !shared.StringInSlice(key, changedConfig) {
+ changedConfig = append(changedConfig, key)
+ }
+ }
+ }
+
+ // Skip on no change
+ if len(changedConfig) == 0 {
+ return nil, false
+ }
+
+ return changedConfig, userOnly
+}
+
+// Default permissions for folders in ${LXD_DIR}
+const storagePoolsDirMode os.FileMode = 0711
+const containersDirMode os.FileMode = 0711
+const customDirMode os.FileMode = 0711
+const imagesDirMode os.FileMode = 0700
+const snapshotsDirMode os.FileMode = 0700
+
+// Detect whether LXD already uses the given storage pool.
+func lxdUsesPool(dbObj *db.Cluster, onDiskPoolName string, driver string, onDiskProperty string) (bool, string, error) {
+ pools, err := dbObj.StoragePools()
+ if err != nil && err != db.ErrNoSuchObject {
+ return false, "", err
+ }
+
+ for _, pool := range pools {
+ _, pl, err := dbObj.StoragePoolGet(pool)
+ if err != nil {
+ continue
+ }
+
+ if pl.Driver != driver {
+ continue
+ }
+
+ if pl.Config[onDiskProperty] == onDiskPoolName {
+ return true, pl.Name, nil
+ }
+ }
+
+ return false, "", nil
+}
+
+func makeFSType(path string, fsType string, options *mkfsOptions) (string, error) {
+ var err error
+ var msg string
+
+ fsOptions := options
+ if fsOptions == nil {
+ fsOptions = &mkfsOptions{}
+ }
+
+ cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
+ if fsOptions.label != "" {
+ cmd = append(cmd, "-L", fsOptions.label)
+ }
+
+ if fsType == "ext4" {
+ cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
+ }
+
+ msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
+ if err != nil {
+ return msg, err
+ }
+
+ return "", nil
+}
+
+func fsGenerateNewUUID(fstype string, lvpath string) (string, error) {
+ switch fstype {
+ case "btrfs":
+ return btrfsGenerateNewUUID(lvpath)
+ case "xfs":
+ return xfsGenerateNewUUID(lvpath)
+ }
+
+ return "", nil
+}
+
+func xfsGenerateNewUUID(lvpath string) (string, error) {
+ msg, err := shared.RunCommand(
+ "xfs_admin",
+ "-U", "generate",
+ lvpath)
+ if err != nil {
+ return msg, err
+ }
+
+ return "", nil
+}
+
+func btrfsGenerateNewUUID(lvpath string) (string, error) {
+ msg, err := shared.RunCommand(
+ "btrfstune",
+ "-f",
+ "-u",
+ lvpath)
+ if err != nil {
+ return msg, err
+ }
+
+ return "", nil
+}
+
+func growFileSystem(fsType string, devPath string, mntpoint string) error {
+ var msg string
+ var err error
+ switch fsType {
+ case "": // if not specified, default to ext4
+ fallthrough
+ case "ext4":
+ msg, err = shared.TryRunCommand("resize2fs", devPath)
+ case "xfs":
+ msg, err = shared.TryRunCommand("xfs_growfs", devPath)
+ case "btrfs":
+ msg, err = shared.TryRunCommand("btrfs", "filesystem", "resize", "max", mntpoint)
+ default:
+ return fmt.Errorf(`Growing not supported for filesystem type "%s"`, fsType)
+ }
+
+ if err != nil {
+ errorMsg := fmt.Sprintf(`Could not extend underlying %s filesystem for "%s": %s`, fsType, devPath, msg)
+ logger.Errorf(errorMsg)
+ return fmt.Errorf(errorMsg)
+ }
+
+ logger.Debugf(`extended underlying %s filesystem for "%s"`, fsType, devPath)
+ return nil
+}
+
+func shrinkFileSystem(fsType string, devPath string, mntpoint string, byteSize int64) error {
+ strSize := fmt.Sprintf("%dK", byteSize/1024)
+
+ switch fsType {
+ case "": // if not specified, default to ext4
+ fallthrough
+ case "ext4":
+ _, err := shared.TryRunCommand("e2fsck", "-f", "-y", devPath)
+ if err != nil {
+ return err
+ }
+
+ _, err = shared.TryRunCommand("resize2fs", devPath, strSize)
+ if err != nil {
+ return err
+ }
+ case "btrfs":
+ _, err := shared.TryRunCommand("btrfs", "filesystem", "resize", strSize, mntpoint)
+ if err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf(`Shrinking not supported for filesystem type "%s"`, fsType)
+ }
+
+ return nil
+}
+
+/*
+func shrinkVolumeFilesystem(s StorageDriver, volumeType int, fsType string, devPath string, mntpoint string, byteSize int64, data interface{}) (func() (bool, error), error) {
+ var cleanupFunc func() (bool, error)
+ switch fsType {
+ case "xfs":
+ logger.Errorf("XFS filesystems cannot be shrunk: dump, mkfs, and restore are required")
+ return nil, fmt.Errorf("xfs filesystems cannot be shrunk: dump, mkfs, and restore are required")
+ case "btrfs":
+ fallthrough
+ case "": // if not specified, default to ext4
+ fallthrough
+ case "ext4":
+ switch volumeType {
+ case storagePoolVolumeTypeContainer:
+ c := data.(Container)
+ ourMount, err := c.StorageStop()
+ if err != nil {
+ return nil, err
+ }
+ if !ourMount {
+ cleanupFunc = c.StorageStart
+ }
+ case storagePoolVolumeTypeCustom:
+ ourMount, err := s.StoragePoolVolumeUmount()
+ if err != nil {
+ return nil, err
+ }
+ if !ourMount {
+ cleanupFunc = s.StoragePoolVolumeMount
+ }
+ default:
+ return nil, fmt.Errorf(`Resizing not implemented for storage volume type %d`, volumeType)
+ }
+
+ default:
+ return nil, fmt.Errorf(`Shrinking not supported for filesystem type "%s"`, fsType)
+ }
+
+ err := shrinkFileSystem(fsType, devPath, mntpoint, byteSize)
+ return cleanupFunc, err
+}
+*/
+
+// Returns the parent container name, snapshot name, and whether it actually was
+// a snapshot name.
+func containerGetParentAndSnapshotName(name string) (string, string, bool) {
+ fields := strings.SplitN(name, shared.SnapshotDelimiter, 2)
+ if len(fields) == 1 {
+ return name, "", false
+ }
+
+ return fields[0], fields[1], true
+}
+
+// /var/lib/lxd/[snapshots|containers]/name
+func containerPath(project string, name string, isSnapshot bool) string {
+ if isSnapshot {
+ return shared.VarPath("snapshots", projectPrefix(project, name))
+ }
+
+ return shared.VarPath("containers", projectPrefix(project, name))
+}
+
+func setUnprivUserACL(idmapset *idmap.IdmapSet, destPath string) error {
+ // Skip for privileged containers
+ if idmapset == nil {
+ return nil
+ }
+
+ // Make sure the map is valid. Skip if container uid 0 == host uid 0
+ uid, _ := idmapset.ShiftIntoNs(0, 0)
+ switch uid {
+ case -1:
+ return fmt.Errorf("Container doesn't have a uid 0 in its map")
+ case 0:
+ return nil
+ }
+
+ // Attempt to set a POSIX ACL first.
+ acl := fmt.Sprintf("%d:rx", uid)
+ _, err := shared.RunCommand("setfacl", "-m", acl, destPath)
+ if err == nil {
+ return nil
+ }
+
+ // Fallback to chmod if the fs doesn't support it.
+ _, err = shared.RunCommand("chmod", "+x", destPath)
+ if err != nil {
+ logger.Debugf("Failed to set executable bit on the container path: %s", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 48553ec478..113cd707d1 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -18,6 +18,7 @@ import (
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/migration"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -420,7 +421,7 @@ func (s *storageBtrfs) StoragePoolMount() (bool, error) {
// Since we mount the loop device LO_FLAGS_AUTOCLEAR is
// fine since the loop device will be kept around for as
// long as the mount exists.
- loopF, loopErr := prepareLoopDev(source, LoFlagsAutoclear)
+ loopF, loopErr := driver.PrepareLoopDev(source, driver.LoFlagsAutoclear)
if loopErr != nil {
return false, loopErr
}
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index fae1d97a37..8be04d6193 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -386,7 +386,7 @@ func (s *storageLvm) StoragePoolDelete() error {
if s.loopInfo != nil {
// Set LO_FLAGS_AUTOCLEAR before we remove the loop file
// otherwise we will get EBADF.
- err = setAutoclearOnLoopDev(int(s.loopInfo.Fd()))
+ err = driver.SetAutoclearOnLoopDev(int(s.loopInfo.Fd()))
if err != nil {
logger.Warnf("Failed to set LO_FLAGS_AUTOCLEAR on loop device: %s, manual cleanup needed", err)
}
@@ -458,12 +458,12 @@ func (s *storageLvm) StoragePoolMount() (bool, error) {
if filepath.IsAbs(source) && !shared.IsBlockdevPath(source) {
// Try to prepare new loop device.
- loopF, loopErr := prepareLoopDev(source, 0)
+ loopF, loopErr := driver.PrepareLoopDev(source, 0)
if loopErr != nil {
return false, loopErr
}
// Make sure that LO_FLAGS_AUTOCLEAR is unset.
- loopErr = unsetAutoclearOnLoopDev(int(loopF.Fd()))
+ loopErr = driver.UnsetAutoclearOnLoopDev(int(loopF.Fd()))
if loopErr != nil {
return false, loopErr
}
diff --git a/lxd/storage_utils.go b/lxd/storage_utils.go
index 23f0450c19..d7d650d414 100644
--- a/lxd/storage_utils.go
+++ b/lxd/storage_utils.go
@@ -8,6 +8,7 @@ import (
"time"
"github.com/lxc/lxd/lxd/db"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/logger"
@@ -34,7 +35,7 @@ var MountOptions = map[string]mountOptions{
"diratime": {false, syscall.MS_NODIRATIME},
"dirsync": {true, syscall.MS_DIRSYNC},
"exec": {false, syscall.MS_NOEXEC},
- "lazytime": {true, MS_LAZYTIME},
+ "lazytime": {true, driver.MS_LAZYTIME},
"mand": {true, syscall.MS_MANDLOCK},
"noatime": {true, syscall.MS_NOATIME},
"nodev": {true, syscall.MS_NODEV},
From e6859ef1d3de5167e54fbc8dc3bbf21f2148b7ae Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 15:13:48 +0200
Subject: [PATCH 07/15] migration: Remove unused Snapshots() function from
interface
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage_btrfs.go | 4 ----
lxd/storage_migration.go | 7 -------
lxd/storage_zfs.go | 4 ----
3 files changed, 15 deletions(-)
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 113cd707d1..7fcab12506 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -2410,10 +2410,6 @@ type btrfsMigrationSourceDriver struct {
stoppedSnapName string
}
-func (s *btrfsMigrationSourceDriver) Snapshots() []container {
- return s.snapshots
-}
-
func (s *btrfsMigrationSourceDriver) send(conn *websocket.Conn, btrfsPath string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
args := []string{"send"}
if btrfsParent != "" {
diff --git a/lxd/storage_migration.go b/lxd/storage_migration.go
index 387f2bef6d..835ae95d24 100644
--- a/lxd/storage_migration.go
+++ b/lxd/storage_migration.go
@@ -17,9 +17,6 @@ import (
// MigrationStorageSourceDriver defines the functions needed to implement a
// migration source driver.
type MigrationStorageSourceDriver interface {
- /* snapshots for this container, if any */
- Snapshots() []container
-
/* send any bits of the container/snapshots that are possible while the
* container is still running.
*/
@@ -46,10 +43,6 @@ type rsyncStorageSourceDriver struct {
rsyncFeatures []string
}
-func (s rsyncStorageSourceDriver) Snapshots() []container {
- return s.snapshots
-}
-
func (s rsyncStorageSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error {
ourMount, err := storage.StoragePoolVolumeMount()
if err != nil {
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index f049deb595..93c60f13d0 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -2513,10 +2513,6 @@ type zfsMigrationSourceDriver struct {
zfsFeatures []string
}
-func (s *zfsMigrationSourceDriver) Snapshots() []container {
- return s.snapshots
-}
-
func (s *zfsMigrationSourceDriver) send(conn *websocket.Conn, zfsName string, zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
sourceParentName, _, _ := containerGetParentAndSnapshotName(s.container.Name())
poolName := s.zfs.getOnDiskPoolName()
From 03926ec8fbacecbe5ee865790deaf05c30576c7d Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 15:17:33 +0200
Subject: [PATCH 08/15] lxd: Add common code to new storage package
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage/lock.go | 109 +++++++++++++++++
lxd/storage/shared.go | 88 ++++++++++++++
lxd/storage/storage.go | 181 ++++++++++++++++++++++++++++
lxd/storage/storage_pools_config.go | 40 ++++++
lxd/storage/storage_pools_utils.go | 26 ++++
lxd/storage/volumes_utils.go | 26 ++++
6 files changed, 470 insertions(+)
create mode 100644 lxd/storage/lock.go
create mode 100644 lxd/storage/shared.go
create mode 100644 lxd/storage/storage.go
create mode 100644 lxd/storage/storage_pools_config.go
create mode 100644 lxd/storage/storage_pools_utils.go
create mode 100644 lxd/storage/volumes_utils.go
diff --git a/lxd/storage/lock.go b/lxd/storage/lock.go
new file mode 100644
index 0000000000..92ffa49aed
--- /dev/null
+++ b/lxd/storage/lock.go
@@ -0,0 +1,109 @@
+package storage
+
+import (
+ "fmt"
+ "sync"
+
+ "github.com/lxc/lxd/shared/logger"
+)
+
+// lxdStorageLockMap is a hashmap that allows functions to check whether the
+// operation they are about to perform is already in progress. If it is the
+// channel can be used to wait for the operation to finish. If it is not, the
+// function that wants to perform the operation should store its code in the
+// hashmap.
+// Note that any access to this map must be done while holding a lock.
+var lxdStorageOngoingOperationMap = map[string]chan bool{}
+
+// lxdStorageMapLock is used to access lxdStorageOngoingOperationMap.
+var lxdStorageMapLock sync.Mutex
+
+// The following functions are used to construct simple operation codes that are
+// unique.
+func getPoolMountLockID(poolName string) string {
+ return fmt.Sprintf("mount/pool/%s", poolName)
+}
+
+func getPoolUmountLockID(poolName string) string {
+ return fmt.Sprintf("umount/pool/%s", poolName)
+}
+
+func getContainerMountLockID(poolName string, containerName string) string {
+ return fmt.Sprintf("mount/container/%s/%s", poolName, containerName)
+}
+
+func getContainerUmountLockID(poolName string, containerName string) string {
+ return fmt.Sprintf("umount/container/%s/%s", poolName, containerName)
+}
+
+func getCustomMountLockID(poolName string, volumeName string) string {
+ return fmt.Sprintf("mount/custom/%s/%s", poolName, volumeName)
+}
+
+func getCustomUmountLockID(poolName string, volumeName string) string {
+ return fmt.Sprintf("umount/custom/%s/%s", poolName, volumeName)
+}
+
+func getImageCreateLockID(poolName string, fingerprint string) string {
+ return fmt.Sprintf("create/image/%s/%s", poolName, fingerprint)
+}
+
+func LockPoolMount(poolName string) func() {
+ return lock(getPoolMountLockID(poolName))
+}
+
+func LockPoolUmount(poolName string) func() {
+ return lock(getPoolUmountLockID(poolName))
+}
+
+func LockContainerMount(poolName string, containerName string) func() {
+ return lock(getContainerMountLockID(poolName, containerName))
+}
+
+func LockContainerUmount(poolName string, containerName string) func() {
+ return lock(getContainerUmountLockID(poolName, containerName))
+}
+
+func LockCustomMount(poolName string, volumeName string) func() {
+ return lock(getCustomMountLockID(poolName, volumeName))
+}
+
+func LockCustomUmount(poolName string, volumeName string) func() {
+ return lock(getCustomUmountLockID(poolName, volumeName))
+}
+
+func LockImageCreate(poolName string, fingerprint string) func() {
+ return lock(getImageCreateLockID(poolName, fingerprint))
+}
+
+func lock(lockID string) func() {
+ lxdStorageMapLock.Lock()
+
+ if waitChannel, ok := lxdStorageOngoingOperationMap[lockID]; ok {
+ lxdStorageMapLock.Unlock()
+
+ _, ok := <-waitChannel
+ if ok {
+ logger.Warnf("Received value over semaphore, this should not have happened")
+ }
+
+ // Give the benefit of the doubt and assume that the other
+ // thread actually succeeded in mounting the storage pool.
+ return nil
+ }
+
+ lxdStorageOngoingOperationMap[lockID] = make(chan bool)
+ lxdStorageMapLock.Unlock()
+
+ return func() {
+ lxdStorageMapLock.Lock()
+
+ waitChannel, ok := lxdStorageOngoingOperationMap[lockID]
+ if ok {
+ close(waitChannel)
+ delete(lxdStorageOngoingOperationMap, lockID)
+ }
+
+ lxdStorageMapLock.Unlock()
+ }
+}
diff --git a/lxd/storage/shared.go b/lxd/storage/shared.go
new file mode 100644
index 0000000000..be5e57244b
--- /dev/null
+++ b/lxd/storage/shared.go
@@ -0,0 +1,88 @@
+package storage
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "syscall"
+
+ "github.com/lxc/lxd/lxd/state"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+)
+
+type driverShared struct {
+ s *state.State
+
+ poolID int64
+ pool *api.StoragePool
+
+ volume *api.StorageVolume
+
+ sTypeVersion string
+}
+
+func (d *driverShared) SharedInit(s *state.State, pool *api.StoragePool, poolID int64, volume *api.StorageVolume) {
+ d.s = s
+ d.pool = pool
+ d.poolID = poolID
+ d.volume = volume
+}
+
+func (d *driverShared) GetVersion() string {
+ return d.sTypeVersion
+}
+
+func (d *driverShared) rsync(source string, dest string) error {
+ var msg string
+ var err error
+ bwlimit := d.pool.Config["rsync.bwlimit"]
+
+ errorMsg := fmt.Errorf("Failed to rsync: %s: %s", string(msg), err)
+
+ err = os.MkdirAll(dest, 0755)
+ if err != nil {
+ return errorMsg
+ }
+
+ rsyncVerbosity := "-q"
+ // Handle debug
+ /*
+ if debug {
+ rsyncVerbosity = "-vi"
+ }
+ */
+
+ if bwlimit == "" {
+ bwlimit = "0"
+ }
+
+ msg, err = shared.RunCommand("rsync",
+ "-a",
+ "-HAX",
+ "--sparse",
+ "--devices",
+ "--delete",
+ "--checksum",
+ "--numeric-ids",
+ "--xattrs",
+ "--bwlimit", bwlimit,
+ rsyncVerbosity,
+ shared.AddSlash(source),
+ dest)
+ if err != nil {
+ runError, ok := err.(shared.RunError)
+ if ok {
+ exitError, ok := runError.Err.(*exec.ExitError)
+ if ok {
+ waitStatus := exitError.Sys().(syscall.WaitStatus)
+ if waitStatus.ExitStatus() == 24 {
+ return nil
+ }
+ }
+ }
+ return errorMsg
+ }
+
+ return nil
+}
diff --git a/lxd/storage/storage.go b/lxd/storage/storage.go
new file mode 100644
index 0000000000..a038fb75c5
--- /dev/null
+++ b/lxd/storage/storage.go
@@ -0,0 +1,181 @@
+package storage
+
+import (
+ "fmt"
+ "os"
+
+ "github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/ioprogress"
+)
+
+type StoragePoolArgs struct {
+ PoolID int64
+ Pool *api.StoragePool
+ BackingFS string
+ Cluster *db.Cluster
+}
+
+type StoragePoolVolumeArgs struct {
+ StoragePoolArgs
+
+ Volume *api.StorageVolume
+}
+
+// volumeType defines the type of a volume
+type VolumeType int
+
+const (
+ VolumeTypeContainer VolumeType = iota
+ VolumeTypeContainerSnapshot
+ VolumeTypeCustom
+ VolumeTypeCustomSnapshot
+ VolumeTypeImage
+ VolumeTypeImageSnapshot
+)
+
+// {LXD_DIR}/storage-pools/<pool>
+func getStoragePoolMountPoint(poolName string) string {
+ return shared.VarPath("storage-pools", poolName)
+}
+
+// ${LXD_DIR}/storage-pools/<pool>/custom-snapshots/<custom volume name>/<snapshot name>
+func getStoragePoolVolumeSnapshotMountPoint(poolName string, snapshotName string) string {
+ return shared.VarPath("storage-pools", poolName, "custom-snapshots", snapshotName)
+}
+
+// ${LXD_DIR}/storage-pools/<pool>/custom/<storage_volume>
+func getStoragePoolVolumeMountPoint(poolName string, volumeName string) string {
+ return shared.VarPath("storage-pools", poolName, "custom", volumeName)
+}
+
+// ${LXD_DIR}/storage-pools/<pool>/containers/[<project_name>_]<container_name>
+func getContainerMountPoint(project string, poolName string, containerName string) string {
+ return shared.VarPath("storage-pools", poolName, "containers", projectPrefix(project, containerName))
+}
+
+// ${LXD_DIR}/storage-pools/<pool>/containers-snapshots/<snapshot_name>
+func getSnapshotMountPoint(project, poolName string, snapshotName string) string {
+ return shared.VarPath("storage-pools", poolName, "containers-snapshots", projectPrefix(project, snapshotName))
+}
+
+func createContainerMountpoint(mountPoint string, mountPointSymlink string, privileged bool) error {
+ var mode os.FileMode
+ if privileged {
+ mode = 0700
+ } else {
+ mode = 0711
+ }
+
+ mntPointSymlinkExist := shared.PathExists(mountPointSymlink)
+ mntPointSymlinkTargetExist := shared.PathExists(mountPoint)
+
+ var err error
+ if !mntPointSymlinkTargetExist {
+ err = os.MkdirAll(mountPoint, 0711)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = os.Chmod(mountPoint, mode)
+ if err != nil {
+ return err
+ }
+
+ if !mntPointSymlinkExist {
+ err := os.Symlink(mountPoint, mountPointSymlink)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func createSnapshotMountpoint(snapshotMountpoint string, snapshotsSymlinkTarget string, snapshotsSymlink string) error {
+ snapshotMntPointExists := shared.PathExists(snapshotMountpoint)
+ mntPointSymlinkExist := shared.PathExists(snapshotsSymlink)
+
+ if !snapshotMntPointExists {
+ err := os.MkdirAll(snapshotMountpoint, 0711)
+ if err != nil {
+ return err
+ }
+ }
+
+ if !mntPointSymlinkExist {
+ err := os.Symlink(snapshotsSymlinkTarget, snapshotsSymlink)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// ${LXD_DIR}/storage-pools/<pool>/images/<fingerprint>
+func getImageMountPoint(poolName string, fingerprint string) string {
+ return shared.VarPath("storage-pools", poolName, "images", fingerprint)
+}
+
+// FIXME: this function doesn't belong here
+// Add the "<project>_" prefix when the given project name is not "default".
+func projectPrefix(project string, s string) string {
+ if project != "default" {
+ s = fmt.Sprintf("%s_%s", project, s)
+ }
+ return s
+}
+
+func renameContainerMountpoint(oldMountPoint string, oldMountPointSymlink string, newMountPoint string, newMountPointSymlink string) error {
+ if shared.PathExists(oldMountPoint) {
+ err := os.Rename(oldMountPoint, newMountPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Rename the symlink target.
+ if shared.PathExists(oldMountPointSymlink) {
+ err := os.Remove(oldMountPointSymlink)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Create the new symlink.
+ err := os.Symlink(newMountPoint, newMountPointSymlink)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func unpackImage(imagefname string, destpath string, blockBackend bool, runningInUserns bool, tracker *ioprogress.ProgressTracker) error {
+ err := shared.Unpack(imagefname, destpath, blockBackend, runningInUserns, tracker)
+ if err != nil {
+ return err
+ }
+
+ rootfsPath := fmt.Sprintf("%s/rootfs", destpath)
+ if shared.PathExists(imagefname + ".rootfs") {
+ err = os.MkdirAll(rootfsPath, 0755)
+ if err != nil {
+ return fmt.Errorf("Error creating rootfs directory")
+ }
+
+ err = shared.Unpack(imagefname+".rootfs", rootfsPath, blockBackend, runningInUserns, tracker)
+ if err != nil {
+ return err
+ }
+ }
+
+ if !shared.PathExists(rootfsPath) {
+ return fmt.Errorf("Image is missing a rootfs: %s", imagefname)
+ }
+
+ return nil
+}
diff --git a/lxd/storage/storage_pools_config.go b/lxd/storage/storage_pools_config.go
new file mode 100644
index 0000000000..4f2c830cff
--- /dev/null
+++ b/lxd/storage/storage_pools_config.go
@@ -0,0 +1,40 @@
+package storage
+
+import "fmt"
+
+func updateStoragePoolError(unchangeable []string, driverName string) error {
+ return fmt.Errorf(`The %v properties cannot be changed for "%s" `+
+ `storage pools`, unchangeable, driverName)
+}
+
+var changeableStoragePoolProperties = map[string][]string{
+ "btrfs": {
+ "rsync.bwlimit",
+ "btrfs.mount_options",
+ },
+
+ "ceph": {
+ "volume.block.filesystem",
+ "volume.block.mount_options",
+ "volume.size",
+ },
+
+ "dir": {
+ "rsync.bwlimit",
+ },
+
+ "lvm": {
+ "lvm.thinpool_name",
+ "lvm.vg_name",
+ "volume.block.filesystem",
+ "volume.block.mount_options",
+ "volume.size",
+ },
+
+ "zfs": {
+ "rsync_bwlimit",
+ "volume.zfs.remove_snapshots",
+ "volume.zfs.use_refquota",
+ "zfs.clone_copy",
+ },
+}
diff --git a/lxd/storage/storage_pools_utils.go b/lxd/storage/storage_pools_utils.go
new file mode 100644
index 0000000000..8a5238cc08
--- /dev/null
+++ b/lxd/storage/storage_pools_utils.go
@@ -0,0 +1,26 @@
+package storage
+
+import (
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+)
+
+func storageResource(path string) (*api.ResourcesStoragePool, error) {
+ st, err := shared.Statvfs(path)
+ if err != nil {
+ return nil, err
+ }
+
+ res := api.ResourcesStoragePool{}
+ res.Space.Total = st.Blocks * uint64(st.Bsize)
+ res.Space.Used = (st.Blocks - st.Bfree) * uint64(st.Bsize)
+
+ // Some filesystems don't report inodes since they allocate them
+ // dynamically e.g. btrfs.
+ if st.Files > 0 {
+ res.Inodes.Total = st.Files
+ res.Inodes.Used = st.Files - st.Ffree
+ }
+
+ return &res, nil
+}
diff --git a/lxd/storage/volumes_utils.go b/lxd/storage/volumes_utils.go
new file mode 100644
index 0000000000..162336e118
--- /dev/null
+++ b/lxd/storage/volumes_utils.go
@@ -0,0 +1,26 @@
+package storage
+
+import (
+ "fmt"
+
+ "github.com/lxc/lxd/lxd/db"
+)
+
+// XXX: backward compatible declarations, introduced when the db code was
+// extracted to its own package. We should eventually clean this up.
+const (
+ storagePoolVolumeTypeContainer = db.StoragePoolVolumeTypeContainer
+ storagePoolVolumeTypeImage = db.StoragePoolVolumeTypeImage
+ storagePoolVolumeTypeCustom = db.StoragePoolVolumeTypeCustom
+)
+
+const (
+ storagePoolVolumeTypeNameContainer = db.StoragePoolVolumeTypeNameContainer
+ storagePoolVolumeTypeNameImage = db.StoragePoolVolumeTypeNameImage
+ storagePoolVolumeTypeNameCustom = db.StoragePoolVolumeTypeNameCustom
+)
+
+func updateStoragePoolVolumeError(unchangeable []string, driverName string) error {
+ return fmt.Errorf(`The %v properties cannot be changed for "%s" `+
+ `storage volumes`, unchangeable, driverName)
+}
From 8fc5f849ca66f7f702448961183adef454cb911e Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 15:30:59 +0200
Subject: [PATCH 09/15] storage: Add btrfs
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage/btrfs.go | 1416 ++++++++++++++++++++++++++++++++
lxd/storage_migration_btrfs.go | 403 +++++++++
2 files changed, 1819 insertions(+)
create mode 100644 lxd/storage/btrfs.go
create mode 100644 lxd/storage_migration_btrfs.go
diff --git a/lxd/storage/btrfs.go b/lxd/storage/btrfs.go
new file mode 100644
index 0000000000..e8288bed63
--- /dev/null
+++ b/lxd/storage/btrfs.go
@@ -0,0 +1,1416 @@
+package storage
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "syscall"
+
+ log "github.com/lxc/lxd/shared/log15"
+
+ "github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/lxd/util"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+type Btrfs struct {
+ driverShared
+
+ remount uintptr
+}
+
+var btrfsVersion = ""
+
+func (s *Btrfs) Init() error {
+ if btrfsVersion != "" {
+ s.sTypeVersion = btrfsVersion
+ return nil
+ }
+
+ out, err := exec.LookPath("btrfs")
+ if err != nil || len(out) == 0 {
+ return fmt.Errorf("The 'btrfs' tool isn't available")
+ }
+
+ output, err := shared.RunCommand("btrfs", "version")
+ if err != nil {
+ return fmt.Errorf("The 'btrfs' tool isn't working properly")
+ }
+
+ count, err := fmt.Sscanf(strings.SplitN(output, " ", 2)[1], "v%s\n", &s.sTypeVersion)
+ if err != nil || count != 1 {
+ return fmt.Errorf("The 'btrfs' tool isn't working properly")
+ }
+
+ btrfsVersion = s.sTypeVersion
+
+ return nil
+}
+
+func (s *Btrfs) StoragePoolCheck() error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Btrfs) StoragePoolCreate() error {
+ isBlockDev := false
+
+ source := s.pool.Config["source"]
+
+ if strings.HasPrefix(source, "/") {
+ source = shared.HostPath(s.pool.Config["source"])
+ }
+
+ defaultSource := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", s.pool.Name))
+
+ if source == "" || source == defaultSource {
+ source = defaultSource
+ s.pool.Config["source"] = source
+
+ f, err := os.Create(source)
+ if err != nil {
+ return fmt.Errorf("Failed to open %s: %s", source, err)
+ }
+ defer f.Close()
+
+ err = f.Chmod(0600)
+ if err != nil {
+ return fmt.Errorf("Failed to chmod %s: %s", source, err)
+ }
+
+ size, err := shared.ParseByteSizeString(s.pool.Config["size"])
+ if err != nil {
+ return err
+ }
+
+ err = f.Truncate(size)
+ if err != nil {
+ return fmt.Errorf("Failed to create sparse file %s: %s", source, err)
+ }
+
+ output, err := makeFSType(source, "btrfs", &mkfsOptions{label: s.pool.Name})
+ if err != nil {
+ return fmt.Errorf("Failed to create the BTRFS pool: %s", output)
+ }
+ } else {
+ // Unset size property since it doesn't make sense.
+ s.pool.Config["size"] = ""
+
+ if filepath.IsAbs(source) {
+ isBlockDev = shared.IsBlockdevPath(source)
+
+ if isBlockDev {
+ output, err := makeFSType(source, "btrfs", &mkfsOptions{label: s.pool.Name})
+ if err != nil {
+ return fmt.Errorf("Failed to create the BTRFS pool: %s", output)
+ }
+ } else {
+ if IsBtrfsSubVolume(source) {
+ subvols, err := btrfsSubVolumesGet(source)
+ if err != nil {
+ return fmt.Errorf("Could not determine if existing BTRFS subvolume ist empty: %s", err)
+ }
+
+ if len(subvols) > 0 {
+ return fmt.Errorf("Requested BTRFS subvolume exists but is not empty")
+ }
+ } else {
+ cleanSource := filepath.Clean(source)
+ lxdDir := shared.VarPath()
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ if shared.PathExists(source) && !isOnBtrfs(source) {
+ return fmt.Errorf("Existing path is neither a BTRFS subvolume nor does it reside on a BTRFS filesystem")
+ } else if strings.HasPrefix(cleanSource, lxdDir) {
+ if cleanSource != poolMntPoint {
+ return fmt.Errorf("BTRFS subvolumes requests in LXD directory \"%s\" are only valid under \"%s\"\n(e.g. source=%s)", shared.VarPath(), shared.VarPath("storage-pools"), poolMntPoint)
+ } else if s.s.OS.BackingFS != "btrfs" {
+ return fmt.Errorf("Creation of BTRFS subvolume requested but \"%s\" does not reside on BTRFS filesystem", source)
+ }
+ }
+
+ err := BtrfsSubVolumeCreate(source)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ } else {
+ return fmt.Errorf("Invalid \"source\" property")
+ }
+ }
+
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ if !shared.PathExists(poolMntPoint) {
+ err := os.MkdirAll(poolMntPoint, storagePoolsDirMode)
+ if err != nil {
+ return err
+ }
+ }
+
+ var err error
+ var devUUID string
+
+ if isBlockDev && filepath.IsAbs(source) {
+ devUUID, _ = shared.LookupUUIDByBlockDevPath(source)
+ // The symlink might not have been created even with the delay
+ // we granted it above. So try to call btrfs filesystem show and
+ // parse it out. (I __hate__ this!)
+ if devUUID == "" {
+ devUUID, err = BtrfsLookupFsUUID(source)
+ if err != nil {
+ return err
+ }
+ }
+ s.pool.Config["source"] = devUUID
+ }
+
+ _, err = s.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ dirs := []string{
+ getContainerMountPoint("default", s.pool.Name, ""),
+ getSnapshotMountPoint("default", s.pool.Name, ""),
+ getImageMountPoint(s.pool.Name, ""),
+ getStoragePoolVolumeMountPoint(s.pool.Name, ""),
+ getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, ""),
+ }
+
+ for _, dir := range dirs {
+ err = BtrfsSubVolumeCreate(dir)
+ if err != nil {
+ return fmt.Errorf("Could not create btrfs subvolume: %s", dir)
+ }
+ }
+
+ return nil
+}
+
+func (s *Btrfs) StoragePoolDelete() error {
+ source := s.pool.Config["source"]
+ if strings.HasPrefix(source, "/") {
+ source = shared.HostPath(s.pool.Config["source"])
+ }
+
+ if source == "" {
+ return fmt.Errorf("no \"source\" property found for the storage pool")
+ }
+
+ dirs := []string{
+ getContainerMountPoint("default", s.pool.Name, ""),
+ getSnapshotMountPoint("default", s.pool.Name, ""),
+ getImageMountPoint(s.pool.Name, ""),
+ getStoragePoolVolumeMountPoint(s.pool.Name, ""),
+ }
+
+ for _, dir := range dirs {
+ BtrfsSubVolumesDelete(dir)
+ }
+
+ _, err := s.StoragePoolUmount()
+ if err != nil {
+ return err
+ }
+
+ // This is a UUID. Check whether we can find the block device.
+ if !filepath.IsAbs(source) {
+ // Try to lookup the disk device by UUID but don't fail. If we
+ // don't find one this might just mean we have been given the
+ // UUID of a subvolume.
+ byUUID := fmt.Sprintf("/dev/disk/by-uuid/%s", source)
+ diskPath, err := os.Readlink(byUUID)
+ msg := ""
+ if err == nil {
+ msg = fmt.Sprintf("Removing disk device %s with UUID: %s.", diskPath, source)
+ } else {
+ msg = fmt.Sprintf("Failed to lookup disk device with UUID: %s: %s.", source, err)
+ }
+ logger.Debugf(msg)
+ } else {
+ var err error
+ cleanSource := filepath.Clean(source)
+ sourcePath := shared.VarPath("disks", s.pool.Name)
+ loopFilePath := sourcePath + ".img"
+ if cleanSource == loopFilePath {
+ // This is a loop file so simply remove it.
+ err = os.Remove(source)
+ } else {
+ if !isBtrfsFilesystem(source) && IsBtrfsSubVolume(source) {
+ err = BtrfsSubVolumesDelete(source)
+ }
+ }
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+
+ // Remove the mountpoint for the storage pool.
+ err = os.RemoveAll(getStoragePoolMountPoint(s.pool.Name))
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Btrfs) StoragePoolMount() (bool, error) {
+ cleanupFunc := LockPoolMount(s.pool.Name)
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ source := s.pool.Config["source"]
+ if strings.HasPrefix(source, "/") {
+ source = shared.HostPath(s.pool.Config["source"])
+ }
+
+ if source == "" {
+ return false, fmt.Errorf("no \"source\" property found for the storage pool")
+ }
+
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ // Check whether the mount poolMntPoint exits.
+ if !shared.PathExists(poolMntPoint) {
+ err := os.MkdirAll(poolMntPoint, storagePoolsDirMode)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ if shared.IsMountPoint(poolMntPoint) && (s.remount&syscall.MS_REMOUNT) == 0 {
+ return false, nil
+ }
+
+ mountFlags, mountOptions := lxdResolveMountoptions(getBtrfsMountOptions(s.pool))
+ mountSource := source
+ isBlockDev := shared.IsBlockdevPath(source)
+ if filepath.IsAbs(source) {
+ cleanSource := filepath.Clean(source)
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+ loopFilePath := shared.VarPath("disks", s.pool.Name+".img")
+ if !isBlockDev && cleanSource == loopFilePath {
+ // If source == "${LXD_DIR}"/disks/{pool_name} it is a
+ // loop file we're dealing with.
+ //
+ // Since we mount the loop device LO_FLAGS_AUTOCLEAR is
+ // fine since the loop device will be kept around for as
+ // long as the mount exists.
+ loopF, loopErr := PrepareLoopDev(source, LoFlagsAutoclear)
+ if loopErr != nil {
+ return false, loopErr
+ }
+ mountSource = loopF.Name()
+ defer loopF.Close()
+ } else if !isBlockDev && cleanSource != poolMntPoint {
+ mountSource = source
+ mountFlags |= syscall.MS_BIND
+ } else if !isBlockDev && cleanSource == poolMntPoint && s.s.OS.BackingFS == "btrfs" {
+ return false, nil
+ }
+ // User is using block device path.
+ } else {
+ // Try to lookup the disk device by UUID but don't fail. If we
+ // don't find one this might just mean we have been given the
+ // UUID of a subvolume.
+ byUUID := fmt.Sprintf("/dev/disk/by-uuid/%s", source)
+ diskPath, err := os.Readlink(byUUID)
+ if err == nil {
+ mountSource = fmt.Sprintf("/dev/%s", strings.Trim(diskPath, "../../"))
+ } else {
+ // We have very likely been given a subvolume UUID. In
+ // this case we should simply assume that the user has
+ // mounted the parent of the subvolume or the subvolume
+ // itself. Otherwise this becomes a really messy
+ // detection task.
+ return false, nil
+ }
+ }
+
+ mountFlags |= s.remount
+ err := syscall.Mount(mountSource, poolMntPoint, "btrfs", mountFlags, mountOptions)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+}
+
+func (s *Btrfs) StoragePoolUmount() (bool, error) {
+ cleanupFunc := LockPoolUmount(s.pool.Name)
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ if shared.IsMountPoint(poolMntPoint) {
+ err := syscall.Unmount(poolMntPoint, 0)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ return true, nil
+}
+
+func (s *Btrfs) StoragePoolResources() (*api.ResourcesStoragePool, error) {
+ ourMount, err := s.StoragePoolMount()
+ if err != nil {
+ return nil, err
+ }
+ if ourMount {
+ defer s.StoragePoolUmount()
+ }
+
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ // Inode allocation is dynamic so no use in reporting them.
+
+ return storageResource(poolMntPoint)
+}
+
+func (s *Btrfs) StoragePoolUpdate(writable *api.StoragePoolPut,
+ changedConfig []string) error {
+ changeable := changeableStoragePoolProperties["btrfs"]
+ unchangeable := []string{}
+ for _, change := range changedConfig {
+ if !shared.StringInSlice(change, changeable) {
+ unchangeable = append(unchangeable, change)
+ }
+ }
+
+ if len(unchangeable) > 0 {
+ return updateStoragePoolError(unchangeable, "btrfs")
+ }
+
+ // "rsync.bwlimit" requires no on-disk modifications.
+
+ if shared.StringInSlice("btrfs.mount_options", changedConfig) {
+ setBtrfsMountOptions(s.pool, writable.Config["btrfs.mount_options"])
+ s.remount |= syscall.MS_REMOUNT
+ _, err := s.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeCreate(project string, volumeName string,
+ volumeType VolumeType) error {
+ logger.Debug("Creating volume", log.Ctx{"project": project, "volume": volumeName})
+ var mountPoint string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ mountPoint = getContainerMountPoint(project, s.pool.Name, volumeName)
+ case VolumeTypeCustom:
+ mountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, volumeName)
+ case VolumeTypeImage:
+ mountPoint = getImageMountPoint(s.pool.Name, volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if IsBtrfsSubVolume(mountPoint) {
+ return nil
+ }
+
+ err := BtrfsSubVolumeCreate(mountPoint)
+ if err != nil {
+ return err
+ }
+
+ switch volumeType {
+ case VolumeTypeCustom:
+ // apply quota
+ if s.volume.Config["size"] != "" {
+ size, err := shared.ParseByteSizeString(s.volume.Config["size"])
+ if err != nil {
+ return err
+ }
+
+ err = s.VolumeSetQuota(project, volumeName, size, false, VolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeCopy(project, source, target string, snapshots []string, volumeType VolumeType) error {
+ var recursive bool
+ var sourcePath string
+ var targetPath string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ recursive = true
+ sourcePath = getContainerMountPoint(project, s.pool.Name, source)
+ targetPath = getContainerMountPoint(project, s.pool.Name, target)
+ case VolumeTypeCustom:
+ recursive = true
+ sourcePath = getStoragePoolVolumeMountPoint(s.pool.Name, source)
+ targetPath = getStoragePoolVolumeMountPoint(s.pool.Name, target)
+ case VolumeTypeImage:
+ recursive = false
+ sourcePath = getImageMountPoint(s.pool.Name, source)
+ targetPath = getContainerMountPoint(project, s.pool.Name, target)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ err := BtrfsPoolVolumesSnapshot(sourcePath, targetPath, false, recursive)
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snapshots {
+ sourceSnapshotName := fmt.Sprintf("%s/%s", source, snap)
+ targetSnapshotName := fmt.Sprintf("%s/%s", target, snap)
+
+ var sourceSnapshotPath string
+ var targetSnapshotPath string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ sourceSnapshotPath = getSnapshotMountPoint(project, s.pool.Name, sourceSnapshotName)
+ targetSnapshotPath = getSnapshotMountPoint(project, s.pool.Name, targetSnapshotName)
+ case VolumeTypeCustom:
+ sourceSnapshotPath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceSnapshotName)
+ targetSnapshotPath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, targetSnapshotName)
+ }
+
+ err := BtrfsPoolVolumesSnapshot(sourceSnapshotPath, targetSnapshotPath, false, true)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeDelete(project, volumeName string, recursive bool, volumeType VolumeType) error {
+ var volumePath string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ volumePath = getContainerMountPoint(project, s.pool.Name, volumeName)
+ case VolumeTypeContainerSnapshot:
+ volumePath = getSnapshotMountPoint(project, s.pool.Name, volumeName)
+ case VolumeTypeCustom:
+ volumePath = getStoragePoolVolumeMountPoint(s.pool.Name, volumeName)
+ case VolumeTypeCustomSnapshot:
+ volumePath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, volumeName)
+ case VolumeTypeImage:
+ volumePath = getImageMountPoint(s.pool.Name, volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type")
+ }
+
+ if recursive {
+ if shared.PathExists(volumePath) && IsBtrfsSubVolume(volumePath) {
+ return BtrfsSubVolumesDelete(volumePath)
+ }
+ } else {
+ return BtrfsSubVolumeDelete(volumePath)
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeMount(project string, name string, volumeType VolumeType) (bool, error) {
+ if volumeType != VolumeTypeContainerSnapshot {
+ // Nothing to do
+ return s.StoragePoolMount()
+ }
+
+ snapshotSubvolumeName := getSnapshotMountPoint(project, s.pool.Name, name)
+ roSnapshotSubvolumeName := fmt.Sprintf("%s.ro", snapshotSubvolumeName)
+
+ if shared.PathExists(roSnapshotSubvolumeName) {
+ logger.Debugf("The BTRFS snapshot is already mounted read-write")
+ return false, nil
+ }
+
+ err := os.Rename(snapshotSubvolumeName, roSnapshotSubvolumeName)
+ if err != nil {
+ return false, err
+ }
+
+ err = BtrfsPoolVolumesSnapshot(roSnapshotSubvolumeName, snapshotSubvolumeName, false, true)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+}
+
+func (s *Btrfs) VolumeUmount(project string, name string, volumeType VolumeType) (bool, error) {
+ if volumeType != VolumeTypeContainerSnapshot {
+ // Nothing to do
+ return true, nil
+ }
+
+ snapshotSubvolumeName := getSnapshotMountPoint(project, s.pool.Name, name)
+ roSnapshotSubvolumeName := fmt.Sprintf("%s.ro", snapshotSubvolumeName)
+
+ if !shared.PathExists(roSnapshotSubvolumeName) {
+ logger.Debugf("The BTRFS snapshot is currently not mounted read-write")
+ return false, nil
+ }
+
+ if shared.PathExists(snapshotSubvolumeName) && IsBtrfsSubVolume(snapshotSubvolumeName) {
+ err := BtrfsSubVolumesDelete(snapshotSubvolumeName)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ err := os.Rename(roSnapshotSubvolumeName, snapshotSubvolumeName)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+}
+
+func (s *Btrfs) VolumeGetUsage(project, name string, path string) (int64, error) {
+ return btrfsPoolVolumeQGroupUsage(path)
+}
+
+func (s *Btrfs) VolumeSetQuota(project, name string, size int64, userns bool, volumeType VolumeType) error {
+ var path string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ path = getContainerMountPoint(project, s.pool.Name, name)
+ case VolumeTypeCustom:
+ path = getStoragePoolVolumeMountPoint(s.pool.Name, name)
+ }
+
+ qgroup, err := btrfsSubVolumeQGroup(path)
+ if err != nil {
+ if err != db.ErrNoSuchObject {
+ return err
+ }
+
+ // Enable quotas
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ output, err := shared.RunCommand("btrfs", "quota", "enable", poolMntPoint)
+ if err != nil && !userns {
+ return fmt.Errorf("Failed to enable quotas on BTRFS pool: %s", output)
+ }
+ }
+
+ // Attempt to make the subvolume writable
+ shared.RunCommand("btrfs", "property", "set", path, "ro", "false")
+ if size > 0 {
+ output, err := shared.RunCommand(
+ "btrfs",
+ "qgroup",
+ "limit",
+ "-e", fmt.Sprintf("%d", size),
+ path)
+
+ if err != nil {
+ return fmt.Errorf("Failed to set btrfs quota: %s", output)
+ }
+ } else if qgroup != "" {
+ output, err := shared.RunCommand(
+ "btrfs",
+ "qgroup",
+ "destroy",
+ qgroup,
+ path)
+
+ if err != nil {
+ return fmt.Errorf("Failed to set btrfs quota: %s", output)
+ }
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeRename(project string, oldName string, newName string, snapshots []string,
+ volumeType VolumeType) error {
+ switch volumeType {
+ case VolumeTypeContainer:
+ case VolumeTypeCustom:
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeRestore(project string, sourceName string, targetName string, volumeType VolumeType) error {
+ return s.VolumeSnapshotRestore(project, sourceName, targetName, volumeType)
+}
+
+func (s *Btrfs) VolumeUpdate(writable *api.StorageVolumePut,
+ changedConfig []string) error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Btrfs) VolumeSnapshotCreate(project, source, target string,
+ volumeType VolumeType) error {
+ var sourcePath string
+ var targetPath string
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ sourcePath = getContainerMountPoint(project, s.pool.Name, source)
+ targetPath = getSnapshotMountPoint(project, s.pool.Name, target)
+ case VolumeTypeCustomSnapshot:
+ sourcePath = getStoragePoolVolumeMountPoint(s.pool.Name, source)
+ targetPath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target)
+ case VolumeTypeImageSnapshot:
+ sourcePath = getImageMountPoint(s.pool.Name, source)
+ targetPath = getImageMountPoint(s.pool.Name, target)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if source == "" {
+ // Create empty snapshot
+ return BtrfsSubVolumeCreate(targetPath)
+ }
+
+ return BtrfsPoolVolumesSnapshot(sourcePath, targetPath, true, true)
+}
+
+func (s *Btrfs) VolumeSnapshotCopy(project, source, target string,
+ volumeType VolumeType) error {
+ var readOnly bool
+ var recursive bool
+ var sourcePath string
+ var targetPath string
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ readOnly = false
+ recursive = true
+ sourcePath = getSnapshotMountPoint(project, s.pool.Name, source)
+
+ if shared.IsSnapshot(target) {
+ targetPath = getSnapshotMountPoint(project, s.pool.Name, target)
+ } else {
+ targetPath = getContainerMountPoint(project, s.pool.Name, target)
+ }
+ case VolumeTypeCustomSnapshot:
+ readOnly = false
+ recursive = true
+ sourcePath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, source)
+
+ if shared.IsSnapshot(target) {
+ targetPath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target)
+ } else {
+ targetPath = getStoragePoolVolumeMountPoint(s.pool.Name, target)
+ }
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return BtrfsPoolVolumesSnapshot(sourcePath, targetPath, readOnly, recursive)
+}
+
+func (s *Btrfs) VolumeSnapshotDelete(project string, volumeName string, recursive bool, volumeType VolumeType) error {
+ return s.VolumeDelete(project, volumeName, recursive, volumeType)
+}
+
+func (s *Btrfs) VolumeSnapshotRestore(project string, sourceName string, targetName string, volumeType VolumeType) error {
+ var sourceMntPoint string
+ var targetMntPoint string
+
+ logger.Debug("Restoring snapshot", log.Ctx{"project": project, "source": sourceName, "target": targetName})
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ sourceMntPoint = getSnapshotMountPoint(project, s.pool.Name, sourceName)
+ targetMntPoint = getContainerMountPoint(project, s.pool.Name, targetName)
+ case VolumeTypeCustomSnapshot:
+ sourceMntPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceName)
+ targetMntPoint = getStoragePoolVolumeMountPoint(s.pool.Name, targetName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ backupTargetMntPoint := fmt.Sprintf("%s.tmp", targetMntPoint)
+
+ err := os.Rename(targetMntPoint, backupTargetMntPoint)
+ if err != nil {
+ return err
+ }
+
+ undo := true
+
+ defer func() {
+ if undo {
+ os.Rename(backupTargetMntPoint, targetMntPoint)
+ }
+ }()
+
+ err = BtrfsPoolVolumesSnapshot(sourceMntPoint, targetMntPoint, false, true)
+ if err != nil {
+ return err
+ }
+
+ undo = false
+
+ return BtrfsSubVolumesDelete(backupTargetMntPoint)
+}
+
+func (s *Btrfs) VolumeSnapshotRename(project string, oldName string, newName string, volumeType VolumeType) error {
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ // Nothing to do
+ case VolumeTypeCustomSnapshot:
+ // Nothing to do
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeReady(project string, name string) bool {
+ containerMntPoint := getContainerMountPoint(project, s.pool.Name, name)
+ return IsBtrfsSubVolume(containerMntPoint)
+}
+
+func (s *Btrfs) doVolumeBackupCreateOptimized(path string, project string, source string,
+ snapshots []string) error {
+ // Handle snapshots
+ finalParent := ""
+
+ if len(snapshots) > 0 {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", path)
+
+ // Create the snapshot path
+ err := os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return err
+ }
+
+ for i, snap := range snapshots {
+ // Figure out previous and current subvolumes
+ prev := ""
+ fullSnapshotName := fmt.Sprintf("%s/%s", source, snap)
+
+ if i > 0 {
+ fullPrevSnapshotName := fmt.Sprintf("%s/%s", source, snapshots[i-1])
+ // /var/lib/lxd/storage-pools/<pool>/containers-snapshots/<container>/<snapshot>
+ prev = getSnapshotMountPoint(project, s.pool.Name, fullPrevSnapshotName)
+ }
+
+ cur := getSnapshotMountPoint(project, s.pool.Name, fullSnapshotName)
+ // Make a binary btrfs backup
+ target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snap)
+
+ err := btrfsBackup(cur, prev, target)
+ if err != nil {
+ return err
+ }
+
+ finalParent = cur
+ }
+ }
+
+ // Make a temporary copy of the container
+ sourceVolume := getContainerMountPoint(project, s.pool.Name, source)
+ containersPath := getContainerMountPoint("default", s.pool.Name, "")
+
+ tmpContainerMntPoint, err := ioutil.TempDir(containersPath, source)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+
+ targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
+ err = BtrfsPoolVolumesSnapshot(sourceVolume, targetVolume, true, true)
+ if err != nil {
+ return err
+ }
+ defer BtrfsSubVolumesDelete(targetVolume)
+
+ // Dump the container to a file
+ fsDump := fmt.Sprintf("%s/container.bin", path)
+
+ err = btrfsBackup(targetVolume, finalParent, fsDump)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Btrfs) doVolumeBackupCreate(path string, project string, source string,
+ snapshots []string) error {
+ // Handle snapshots
+ if len(snapshots) > 0 {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", path)
+
+ // Create the snapshot path
+ err := os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snapshots {
+ fullSnapshotName := fmt.Sprintf("%s/%s", source, snap)
+
+ // Mount the snapshot to a usable path
+ _, err := s.VolumeMount(project, fullSnapshotName, VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, fullSnapshotName)
+ target := fmt.Sprintf("%s/%s", snapshotsPath, snap)
+
+ // Copy the snapshot
+ err = s.rsync(snapshotMntPoint, target)
+ s.VolumeUmount(project, fullSnapshotName, VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Make a temporary copy of the container
+ sourceVolume := getContainerMountPoint(project, s.pool.Name, source)
+ containersPath := getContainerMountPoint("default", s.pool.Name, "")
+
+ tmpContainerMntPoint, err := ioutil.TempDir(containersPath, source)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+
+ targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
+
+ err = BtrfsPoolVolumesSnapshot(sourceVolume, targetVolume, true, true)
+ if err != nil {
+ return err
+ }
+ defer BtrfsSubVolumesDelete(targetVolume)
+
+ // Copy the container
+ containerPath := fmt.Sprintf("%s/container", path)
+
+ err = s.rsync(targetVolume, containerPath)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Btrfs) VolumeBackupCreate(path string, project string, source string,
+ snapshots []string, optimized bool) error {
+ if optimized {
+ return s.doVolumeBackupCreateOptimized(path, project, source, snapshots)
+ }
+
+ return s.doVolumeBackupCreate(path, project, source, snapshots)
+}
+
+func (s *Btrfs) doVolumeBackupLoadOptimized(backupDir string, project string,
+ containerName string, snapshots []string, privileged bool) error {
+ unpackDir := backupDir
+ unpackPath := filepath.Join(unpackDir, ".backup_unpack")
+
+ for _, snapshotOnlyName := range snapshots {
+ snapshotBackup := fmt.Sprintf("%s/snapshots/%s.bin", unpackPath, snapshotOnlyName)
+
+ feeder, err := os.Open(snapshotBackup)
+ if err != nil {
+ return err
+ }
+
+ // create mountpoint
+ snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, containerName)
+ snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(project, containerName))
+ snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(project, containerName))
+
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+ if err != nil {
+ feeder.Close()
+ return err
+ }
+
+ // /var/lib/lxd/storage-pools/<pool>/snapshots/<container>/
+ btrfsRecvCmd := exec.Command("btrfs", "receive", "-e", snapshotMntPoint)
+ btrfsRecvCmd.Stdin = feeder
+
+ msg, err := btrfsRecvCmd.CombinedOutput()
+ feeder.Close()
+ if err != nil {
+ logger.Errorf("Failed to receive contents of btrfs backup \"%s\": %s", snapshotBackup, string(msg))
+ return err
+ }
+ }
+ containerBackupFile := fmt.Sprintf("%s/container.bin", unpackPath)
+
+ feeder, err := os.Open(containerBackupFile)
+ if err != nil {
+ return err
+ }
+ defer feeder.Close()
+
+ btrfsRecvCmd := exec.Command("btrfs", "receive", "-vv", "-e", unpackDir)
+ btrfsRecvCmd.Stdin = feeder
+
+ msg, err := btrfsRecvCmd.CombinedOutput()
+ if err != nil {
+ logger.Errorf("Failed to receive contents of btrfs backup \"%s\": %s", containerBackupFile, string(msg))
+ return err
+ }
+
+ tmpContainerMntPoint := fmt.Sprintf("%s/.backup", unpackDir)
+ defer BtrfsSubVolumesDelete(tmpContainerMntPoint)
+
+ containerMntPoint := getContainerMountPoint(project, s.pool.Name, containerName)
+ err = BtrfsPoolVolumesSnapshot(tmpContainerMntPoint, containerMntPoint, false, true)
+ if err != nil {
+ logger.Errorf("Failed to create btrfs snapshot \"%s\" of \"%s\": %s", tmpContainerMntPoint, containerMntPoint, err)
+ return err
+ }
+
+ // Create mountpoints
+ err = createContainerMountpoint(containerMntPoint, shared.VarPath("containers", projectPrefix(project, containerName)), privileged)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Btrfs) doVolumeBackupLoad(backupDir string, project string,
+ containerName string, snapshots []string, privileged bool) error {
+ return nil
+}
+
+func (s *Btrfs) VolumeBackupLoad(backupDir string, project string,
+ containerName string, snapshots []string, privileged bool, optimized bool) error {
+ logger.Debug("Loading volume backup", log.Ctx{"project": project, "name": containerName, "snapshots": len(snapshots)})
+
+ if optimized {
+ return s.doVolumeBackupLoadOptimized(backupDir, project, containerName, snapshots, privileged)
+ }
+
+ return s.doVolumeBackupLoad(backupDir, project, containerName, snapshots, privileged)
+}
+
+func (s *Btrfs) VolumePrepareRestore(sourceName string, targetName string, targetSnapshots []string, f func() error) error {
+ // Nothing to do
+ return nil
+}
+
+func btrfsBackup(cur string, prev string, target string) error {
+ args := []string{"send"}
+ if prev != "" {
+ args = append(args, "-p", prev)
+ }
+ args = append(args, cur)
+
+ eater, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return err
+ }
+ defer eater.Close()
+
+ btrfsSendCmd := exec.Command("btrfs", args...)
+ btrfsSendCmd.Stdout = eater
+
+ err = btrfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ return err
+}
+
+func getBtrfsMountOptions(pool *api.StoragePool) string {
+ if pool.Config["btrfs.mount_options"] != "" {
+ return pool.Config["btrfs.mount_options"]
+ }
+
+ return "user_subvol_rm_allowed"
+}
+
+func setBtrfsMountOptions(pool *api.StoragePool, mountOptions string) {
+ pool.Config["btrfs.mount_options"] = mountOptions
+}
+
+// btrfsPoolVolumesDelete is the recursive variant on btrfsPoolVolumeDelete,
+// it first deletes subvolumes of the subvolume and then the
+// subvolume itself.
+func BtrfsSubVolumesDelete(subvol string) error {
+ // Delete subsubvols.
+ subsubvols, err := btrfsSubVolumesGet(subvol)
+ if err != nil {
+ return err
+ }
+ sort.Sort(sort.Reverse(sort.StringSlice(subsubvols)))
+
+ for _, subsubvol := range subsubvols {
+ err := BtrfsSubVolumeDelete(path.Join(subvol, subsubvol))
+ if err != nil {
+ return err
+ }
+ }
+
+ // Delete the subvol itself
+ err = BtrfsSubVolumeDelete(subvol)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func BtrfsSubVolumeCreate(subvol string) error {
+ parentDestPath := filepath.Dir(subvol)
+
+ // TODO: remove this, and create parent directory in the Storage functions.
+ // Then, we will also be able to use *DirMode for the permissions.
+ if !shared.PathExists(parentDestPath) {
+ err := os.MkdirAll(parentDestPath, 0711)
+ if err != nil {
+ return err
+ }
+ }
+
+ output, err := shared.RunCommand("btrfs", "subvolume", "create", subvol)
+ if err != nil {
+ logger.Errorf("Failed to create BTRFS subvolume \"%s\": %s", subvol, output)
+ return err
+ }
+
+ return nil
+}
+
+func BtrfsSubVolumeDelete(subvol string) error {
+ // Attempt (but don't fail on) to delete any qgroup on the subvolume
+ qgroup, err := btrfsSubVolumeQGroup(subvol)
+ if err == nil {
+ shared.RunCommand(
+ "btrfs",
+ "qgroup",
+ "destroy",
+ qgroup,
+ subvol)
+ }
+
+ // Attempt to make the subvolume writable
+ shared.RunCommand("btrfs", "property", "set", subvol, "ro", "false")
+
+ // Delete the subvolume itself
+ _, err = shared.RunCommand(
+ "btrfs",
+ "subvolume",
+ "delete",
+ subvol)
+
+ return err
+}
+
+func BtrfsPoolVolumesSnapshot(source string, dest string, readonly bool, recursive bool) error {
+ // Now snapshot all subvolumes of the root.
+ if recursive {
+ // Get a list of subvolumes of the root
+ subsubvols, err := btrfsSubVolumesGet(source)
+ if err != nil {
+ return err
+ }
+ sort.Strings(subsubvols)
+
+ if len(subsubvols) > 0 && readonly {
+ // A root with subvolumes can never be readonly,
+ // also don't make subvolumes readonly.
+ readonly = false
+
+ logger.Warnf("Subvolumes detected, ignoring ro flag")
+ }
+
+ // First snapshot the root
+ err = BtrfsSnapshot(source, dest, readonly)
+ if err != nil {
+ return err
+ }
+
+ for _, subsubvol := range subsubvols {
+ // Clear the target for the subvol to use
+ os.Remove(path.Join(dest, subsubvol))
+
+ err := BtrfsSnapshot(path.Join(source, subsubvol), path.Join(dest, subsubvol), readonly)
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ err := BtrfsSnapshot(source, dest, readonly)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func btrfsSubVolumesGet(path string) ([]string, error) {
+ result := []string{}
+
+ if !strings.HasSuffix(path, "/") {
+ path = path + "/"
+ }
+
+ // Unprivileged users can't get to fs internals
+ filepath.Walk(path, func(fpath string, fi os.FileInfo, err error) error {
+ // Skip walk errors
+ if err != nil {
+ return nil
+ }
+
+ // Ignore the base path
+ if strings.TrimRight(fpath, "/") == strings.TrimRight(path, "/") {
+ return nil
+ }
+
+ // Subvolumes can only be directories
+ if !fi.IsDir() {
+ return nil
+ }
+
+ // Check if a btrfs subvolume
+ if IsBtrfsSubVolume(fpath) {
+ result = append(result, strings.TrimPrefix(fpath, path))
+ }
+
+ return nil
+ })
+
+ return result, nil
+}
+
+/*
+ * BtrfsSnapshot creates a snapshot of "source" to "dest"
+ * the result will be readonly if "readonly" is True.
+ */
+func BtrfsSnapshot(source string, dest string, readonly bool) error {
+ logger.Debug("Creating btrfs snapshot", log.Ctx{"source": source, "destination": dest, "readonly": readonly})
+ var output string
+ var err error
+ if readonly {
+ output, err = shared.RunCommand(
+ "btrfs",
+ "subvolume",
+ "snapshot",
+ "-r",
+ source,
+ dest)
+ } else {
+ output, err = shared.RunCommand(
+ "btrfs",
+ "subvolume",
+ "snapshot",
+ source,
+ dest)
+ }
+ if err != nil {
+ return fmt.Errorf(
+ "subvolume snapshot failed, source=%s, dest=%s, output=%s",
+ source,
+ dest,
+ output,
+ )
+ }
+
+ return err
+}
+
+// IsBtrfsSubVolume returns true if the given Path is a btrfs subvolume else
+// false.
+func IsBtrfsSubVolume(subvolPath string) bool {
+ fs := syscall.Stat_t{}
+ err := syscall.Lstat(subvolPath, &fs)
+ if err != nil {
+ return false
+ }
+
+ // Check if BTRFS_FIRST_FREE_OBJECTID
+ if fs.Ino != 256 {
+ return false
+ }
+
+ return true
+}
+
+func btrfsSubVolumeQGroup(subvol string) (string, error) {
+ output, err := shared.RunCommand(
+ "btrfs",
+ "qgroup",
+ "show",
+ subvol,
+ "-e",
+ "-f")
+
+ if err != nil {
+ return "", db.ErrNoSuchObject
+ }
+
+ var qgroup string
+ for _, line := range strings.Split(output, "\n") {
+ if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
+ continue
+ }
+
+ fields := strings.Fields(line)
+ if len(fields) != 4 {
+ continue
+ }
+
+ qgroup = fields[0]
+ }
+
+ if qgroup == "" {
+ return "", fmt.Errorf("Unable to find quota group")
+ }
+
+ return qgroup, nil
+}
+
+func btrfsPoolVolumeQGroupUsage(subvol string) (int64, error) {
+ output, err := shared.RunCommand(
+ "btrfs",
+ "qgroup",
+ "show",
+ subvol,
+ "-e",
+ "-f")
+
+ if err != nil {
+ return -1, fmt.Errorf("BTRFS quotas not supported. Try enabling them with \"btrfs quota enable\"")
+ }
+
+ for _, line := range strings.Split(output, "\n") {
+ if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
+ continue
+ }
+
+ fields := strings.Fields(line)
+ if len(fields) != 4 {
+ continue
+ }
+
+ usage, err := strconv.ParseInt(fields[2], 10, 64)
+ if err != nil {
+ continue
+ }
+
+ return usage, nil
+ }
+
+ return -1, fmt.Errorf("Unable to find current qgroup usage")
+}
+
+func isOnBtrfs(path string) bool {
+ fs := syscall.Statfs_t{}
+
+ err := syscall.Statfs(path, &fs)
+ if err != nil {
+ return false
+ }
+
+ if fs.Type != util.FilesystemSuperMagicBtrfs {
+ return false
+ }
+
+ return true
+}
+
+func BtrfsLookupFsUUID(fs string) (string, error) {
+ output, err := shared.RunCommand(
+ "btrfs",
+ "filesystem",
+ "show",
+ "--raw",
+ fs)
+ if err != nil {
+ return "", fmt.Errorf("failed to detect UUID")
+ }
+
+ outputString := output
+ idx := strings.Index(outputString, "uuid: ")
+ outputString = outputString[idx+6:]
+ outputString = strings.TrimSpace(outputString)
+ idx = strings.Index(outputString, "\t")
+ outputString = outputString[:idx]
+ outputString = strings.Trim(outputString, "\n")
+
+ return outputString, nil
+}
+
+func isBtrfsFilesystem(path string) bool {
+ _, err := shared.RunCommand("btrfs", "filesystem", "show", path)
+ if err != nil {
+ return false
+ }
+
+ return true
+}
+
+func BtrfsSnapshotDeleteInternal(project, poolName string, snapshotName string) error {
+ snapshotSubvolumeName := getSnapshotMountPoint(project, poolName, snapshotName)
+ if shared.PathExists(snapshotSubvolumeName) && IsBtrfsSubVolume(snapshotSubvolumeName) {
+ err := BtrfsSubVolumesDelete(snapshotSubvolumeName)
+ if err != nil {
+ return err
+ }
+ }
+
+ sourceSnapshotMntPoint := shared.VarPath("snapshots", projectPrefix(project, snapshotName))
+ os.Remove(sourceSnapshotMntPoint)
+ os.Remove(snapshotSubvolumeName)
+
+ sourceName, _, _ := containerGetParentAndSnapshotName(snapshotName)
+ snapshotSubvolumePath := getSnapshotMountPoint(project, poolName, sourceName)
+ os.Remove(snapshotSubvolumePath)
+ if !shared.PathExists(snapshotSubvolumePath) {
+ snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceName))
+ os.Remove(snapshotMntPointSymlink)
+ }
+
+ return nil
+}
diff --git a/lxd/storage_migration_btrfs.go b/lxd/storage_migration_btrfs.go
new file mode 100644
index 0000000000..9b8a9064ca
--- /dev/null
+++ b/lxd/storage_migration_btrfs.go
@@ -0,0 +1,403 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+
+ "github.com/gorilla/websocket"
+ driver "github.com/lxc/lxd/lxd/storage"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+type btrfsMigrationSourceDriver2 struct {
+ container container
+ snapshots []container
+ btrfsSnapshotNames []string
+ pool *api.StoragePool
+ runningSnapName string
+ stoppedSnapName string
+}
+
+func btrfsMigrationSource(args MigrationSourceArgs, pool *api.StoragePool) (MigrationStorageSourceDriver, error) {
+ /* List all the snapshots in order of reverse creation. The idea here
+ * is that we send the oldest to newest snapshot, hopefully saving on
+ * xfer costs. Then, after all that, we send the container itself.
+ */
+ var err error
+ var snapshots = []container{}
+ if !args.ContainerOnly {
+ snapshots, err = args.Container.Snapshots()
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ driver := &btrfsMigrationSourceDriver2{
+ container: args.Container,
+ snapshots: snapshots,
+ btrfsSnapshotNames: []string{},
+ pool: pool,
+ }
+
+ if !args.ContainerOnly {
+ for _, snap := range snapshots {
+ btrfsPath := getSnapshotMountPoint(snap.Project(), pool.Name, snap.Name())
+ driver.btrfsSnapshotNames = append(driver.btrfsSnapshotNames, btrfsPath)
+ }
+ }
+
+ return driver, nil
+}
+
+func (s *btrfsMigrationSourceDriver2) send(conn *websocket.Conn, btrfsPath string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
+ args := []string{"send"}
+ if btrfsParent != "" {
+ args = append(args, "-p", btrfsParent)
+ }
+ args = append(args, btrfsPath)
+
+ cmd := exec.Command("btrfs", args...)
+
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return err
+ }
+
+ readPipe := io.ReadCloser(stdout)
+ if readWrapper != nil {
+ readPipe = readWrapper(stdout)
+ }
+
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+
+ err = cmd.Start()
+ if err != nil {
+ return err
+ }
+
+ <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
+
+ output, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ logger.Errorf("Problem reading btrfs send stderr: %s", err)
+ }
+
+ err = cmd.Wait()
+ if err != nil {
+ logger.Errorf("Problem with btrfs send: %s", string(output))
+ }
+
+ return err
+}
+
+func (s *btrfsMigrationSourceDriver2) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error {
+ _, containerPool, _ := s.container.Storage().GetContainerPoolInfo()
+ containerName := s.container.Name()
+ containersPath := getContainerMountPoint("default", containerPool, "")
+ sourceName := containerName
+
+ // Deal with sending a snapshot to create a container on another LXD
+ // instance.
+ if s.container.IsSnapshot() {
+ sourceName, _, _ := containerGetParentAndSnapshotName(containerName)
+ snapshotsPath := getSnapshotMountPoint(s.container.Project(), containerPool, sourceName)
+ tmpContainerMntPoint, err := ioutil.TempDir(snapshotsPath, sourceName)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+
+ migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint)
+ snapshotMntPoint := getSnapshotMountPoint(s.container.Project(), containerPool, containerName)
+ err = driver.BtrfsPoolVolumesSnapshot(snapshotMntPoint, migrationSendSnapshot, true, true)
+ if err != nil {
+ return err
+ }
+ defer driver.BtrfsSubVolumesDelete(migrationSendSnapshot)
+
+ wrapper := StorageProgressReader(op, "fs_progress", containerName)
+ return s.send(conn, migrationSendSnapshot, "", wrapper)
+ }
+
+ if !containerOnly {
+ for i, snap := range s.snapshots {
+ prev := ""
+ if i > 0 {
+ prev = getSnapshotMountPoint(snap.Project(), containerPool, s.snapshots[i-1].Name())
+ }
+
+ snapMntPoint := getSnapshotMountPoint(snap.Project(), containerPool, snap.Name())
+ wrapper := StorageProgressReader(op, "fs_progress", snap.Name())
+ if err := s.send(conn, snapMntPoint, prev, wrapper); err != nil {
+ return err
+ }
+ }
+ }
+
+ tmpContainerMntPoint, err := ioutil.TempDir(containersPath, containerName)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+
+ migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint)
+ containerMntPoint := getContainerMountPoint(s.container.Project(), containerPool, sourceName)
+ err = driver.BtrfsPoolVolumesSnapshot(containerMntPoint, migrationSendSnapshot, true, true)
+ if err != nil {
+ return err
+ }
+ defer driver.BtrfsSubVolumesDelete(migrationSendSnapshot)
+
+ btrfsParent := ""
+ if len(s.btrfsSnapshotNames) > 0 {
+ btrfsParent = s.btrfsSnapshotNames[len(s.btrfsSnapshotNames)-1]
+ }
+
+ wrapper := StorageProgressReader(op, "fs_progress", containerName)
+ return s.send(conn, migrationSendSnapshot, btrfsParent, wrapper)
+}
+
+func (s *btrfsMigrationSourceDriver2) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error {
+ tmpPath := getSnapshotMountPoint(s.container.Project(), s.pool.Name,
+ fmt.Sprintf("%s/.migration-send", s.container.Name()))
+ err := os.MkdirAll(tmpPath, 0711)
+ if err != nil {
+ return err
+ }
+
+ err = os.Chmod(tmpPath, 0700)
+ if err != nil {
+ return err
+ }
+
+ s.stoppedSnapName = fmt.Sprintf("%s/.root", tmpPath)
+ parentName, _, _ := containerGetParentAndSnapshotName(s.container.Name())
+ containerMntPt := getContainerMountPoint(s.container.Project(), s.pool.Name, parentName)
+ err = driver.BtrfsPoolVolumesSnapshot(containerMntPt, s.stoppedSnapName, true, true)
+ if err != nil {
+ return err
+ }
+
+ return s.send(conn, s.stoppedSnapName, s.runningSnapName, nil)
+}
+
+func (s *btrfsMigrationSourceDriver2) Cleanup() {
+ if s.stoppedSnapName != "" {
+ driver.BtrfsSubVolumesDelete(s.stoppedSnapName)
+ }
+
+ if s.runningSnapName != "" {
+ driver.BtrfsSubVolumesDelete(s.runningSnapName)
+ }
+}
+
+func (s *btrfsMigrationSourceDriver2) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error {
+ msg := fmt.Sprintf("Function not implemented")
+ logger.Errorf(msg)
+ return fmt.Errorf(msg)
+}
+
+func btrfsMigrationSink(pool *api.StoragePool, conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
+ btrfsRecv := func(snapName string, btrfsPath string, targetPath string, isSnapshot bool, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
+ args := []string{"receive", "-e", btrfsPath}
+ cmd := exec.Command("btrfs", args...)
+
+ // Remove the existing pre-created subvolume
+ err := driver.BtrfsSubVolumesDelete(targetPath)
+ if err != nil {
+ logger.Errorf("Failed to delete pre-created BTRFS subvolume: %s: %v", btrfsPath, err)
+ return err
+ }
+
+ stdin, err := cmd.StdinPipe()
+ if err != nil {
+ return err
+ }
+
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+
+ err = cmd.Start()
+ if err != nil {
+ return err
+ }
+
+ writePipe := io.WriteCloser(stdin)
+ if writeWrapper != nil {
+ writePipe = writeWrapper(stdin)
+ }
+
+ <-shared.WebsocketRecvStream(writePipe, conn)
+
+ output, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ logger.Debugf("Problem reading btrfs receive stderr %s", err)
+ }
+
+ err = cmd.Wait()
+ if err != nil {
+ logger.Errorf("Problem with btrfs receive: %s", string(output))
+ return err
+ }
+
+ receivedSnapshot := fmt.Sprintf("%s/.migration-send", btrfsPath)
+ // handle older lxd versions
+ if !shared.PathExists(receivedSnapshot) {
+ receivedSnapshot = fmt.Sprintf("%s/.root", btrfsPath)
+ }
+ if isSnapshot {
+ receivedSnapshot = fmt.Sprintf("%s/%s", btrfsPath, snapName)
+ err = driver.BtrfsPoolVolumesSnapshot(receivedSnapshot, targetPath, true, true)
+ } else {
+ err = driver.BtrfsPoolVolumesSnapshot(receivedSnapshot, targetPath, false, true)
+ }
+ if err != nil {
+ logger.Errorf("Problem with btrfs snapshot: %s", err)
+ return err
+ }
+
+ err = driver.BtrfsSubVolumesDelete(receivedSnapshot)
+ if err != nil {
+ logger.Errorf("Failed to delete BTRFS subvolume \"%s\": %s", btrfsPath, err)
+ return err
+ }
+
+ return nil
+ }
+
+ containerName := args.Container.Name()
+ _, containerPool, _ := args.Container.Storage().GetContainerPoolInfo()
+ containersPath := getSnapshotMountPoint(args.Container.Project(), containerPool, containerName)
+ if !args.ContainerOnly && len(args.Snapshots) > 0 {
+ err := os.MkdirAll(containersPath, containersDirMode)
+ if err != nil {
+ return err
+ }
+
+ snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", containerPool, "containers-snapshots", projectPrefix(args.Container.Project(), containerName))
+ snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(args.Container.Project(), containerName))
+ if !shared.PathExists(snapshotMntPointSymlink) {
+ err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // At this point we have already figured out the parent
+ // container's root disk device so we can simply
+ // retrieve it from the expanded devices.
+ parentStoragePool := ""
+ parentExpandedDevices := args.Container.ExpandedDevices()
+ parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := shared.GetRootDiskDevice(parentExpandedDevices)
+ if parentLocalRootDiskDeviceKey != "" {
+ parentStoragePool = parentLocalRootDiskDevice["pool"]
+ }
+
+ // A little neuroticism.
+ if parentStoragePool == "" {
+ return fmt.Errorf("Detected that the container's root device is missing the pool property during BTRFS migration")
+ }
+
+ if !args.ContainerOnly {
+ for _, snap := range args.Snapshots {
+ ctArgs := snapshotProtobufToContainerArgs(args.Container.Project(), containerName, snap)
+
+ // Ensure that snapshot and parent container have the
+ // same storage pool in their local root disk device.
+ // If the root disk device for the snapshot comes from a
+ // profile on the new instance as well we don't need to
+ // do anything.
+ if ctArgs.Devices != nil {
+ snapLocalRootDiskDeviceKey, _, _ := shared.GetRootDiskDevice(ctArgs.Devices)
+ if snapLocalRootDiskDeviceKey != "" {
+ ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
+ }
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(args.Container.Project(), containerPool, ctArgs.Name)
+ _, err := containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs)
+ if err != nil {
+ return err
+ }
+
+ snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", pool.Name, "containers-snapshots", projectPrefix(args.Container.Project(), containerName))
+ snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(args.Container.Project(), containerName))
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+
+ tmpSnapshotMntPoint, err := ioutil.TempDir(containersPath, projectPrefix(args.Container.Project(), containerName))
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpSnapshotMntPoint)
+
+ err = os.Chmod(tmpSnapshotMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+
+ wrapper := StorageProgressWriter(op, "fs_progress", *snap.Name)
+ err = btrfsRecv(*(snap.Name), tmpSnapshotMntPoint, snapshotMntPoint, true, wrapper)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ containersMntPoint := getContainerMountPoint(args.Container.Project(), pool.Name, "")
+ err := createContainerMountpoint(containersMntPoint, args.Container.Path(), args.Container.IsPrivileged())
+ if err != nil {
+ return err
+ }
+
+ /* finally, do the real container */
+ wrapper := StorageProgressWriter(op, "fs_progress", containerName)
+ tmpContainerMntPoint, err := ioutil.TempDir(containersMntPoint, projectPrefix(args.Container.Project(), containerName))
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+
+ containerMntPoint := getContainerMountPoint(args.Container.Project(), pool.Name, containerName)
+ err = btrfsRecv("", tmpContainerMntPoint, containerMntPoint, false, wrapper)
+ if err != nil {
+ return err
+ }
+
+ if args.Live {
+ err = btrfsRecv("", tmpContainerMntPoint, containerMntPoint, false, wrapper)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
From 4bb7790ff2626d691e36be3f66fb4746c6491fec Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 15:31:12 +0200
Subject: [PATCH 10/15] storage: Add dir
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage/dir.go | 546 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 546 insertions(+)
create mode 100644 lxd/storage/dir.go
diff --git a/lxd/storage/dir.go b/lxd/storage/dir.go
new file mode 100644
index 0000000000..1235f6e094
--- /dev/null
+++ b/lxd/storage/dir.go
@@ -0,0 +1,546 @@
+package storage
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "syscall"
+
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+type Dir struct {
+ driverShared
+}
+
+func (s *Dir) Init() error {
+ s.sTypeVersion = "1"
+
+ return nil
+}
+
+func (s *Dir) StoragePoolCheck() error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Dir) StoragePoolCreate() error {
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ source := shared.HostPath(s.pool.Config["source"])
+ if source == "" {
+ source = filepath.Join(shared.VarPath("storage-pools"), s.pool.Name)
+ s.pool.Config["source"] = source
+ } else {
+ cleanSource := filepath.Clean(source)
+ lxdDir := shared.VarPath()
+
+ if strings.HasPrefix(cleanSource, lxdDir) &&
+ cleanSource != poolMntPoint {
+ return fmt.Errorf(`DIR storage pool requests in LXD `+
+ `directory "%s" are only valid under `+
+ `"%s"\n(e.g. source=%s)`, shared.VarPath(),
+ shared.VarPath("storage-pools"), poolMntPoint)
+ }
+
+ source = filepath.Clean(source)
+ }
+
+ revert := true
+
+ if !shared.PathExists(source) {
+ err := os.MkdirAll(source, 0711)
+ if err != nil {
+ return err
+ }
+
+ defer func() {
+ if !revert {
+ return
+ }
+ os.Remove(source)
+ }()
+ } else {
+ empty, err := shared.PathIsEmpty(source)
+ if err != nil {
+ return err
+ }
+
+ if !empty {
+ return fmt.Errorf("The provided directory is not empty")
+ }
+ }
+
+ if !shared.PathExists(poolMntPoint) {
+ err := os.MkdirAll(poolMntPoint, 0711)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if !revert {
+ return
+ }
+ os.Remove(poolMntPoint)
+ }()
+ }
+
+ revert = false
+
+ return nil
+}
+
+func (s *Dir) StoragePoolDelete() error {
+ source := shared.HostPath(s.pool.Config["source"])
+ if source == "" {
+ return fmt.Errorf("no \"source\" property found for the storage pool")
+ }
+
+ _, err := s.StoragePoolUmount()
+ if err != nil {
+ return err
+ }
+
+ if shared.PathExists(source) {
+ err := os.RemoveAll(source)
+ if err != nil {
+ return err
+ }
+ }
+
+ prefix := shared.VarPath("storage-pools")
+ if !strings.HasPrefix(source, prefix) {
+ storagePoolSymlink := getStoragePoolMountPoint(s.pool.Name)
+ if !shared.PathExists(storagePoolSymlink) {
+ return nil
+ }
+
+ err := os.Remove(storagePoolSymlink)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *Dir) StoragePoolMount() (bool, error) {
+ cleanupFunc := LockPoolMount(s.pool.Name)
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ source := shared.HostPath(s.pool.Config["source"])
+ if source == "" {
+ return false, fmt.Errorf("no \"source\" property found for the storage pool")
+ }
+
+ cleanSource := filepath.Clean(source)
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ if cleanSource == poolMntPoint {
+ return true, nil
+ }
+
+ mountSource := cleanSource
+ mountFlags := syscall.MS_BIND
+
+ if shared.IsMountPoint(poolMntPoint) {
+ return false, nil
+ }
+
+ err := syscall.Mount(mountSource, poolMntPoint, "", uintptr(mountFlags), "")
+ if err != nil {
+ logger.Errorf(`Failed to mount DIR storage pool "%s" onto "%s": %s`, mountSource, poolMntPoint, err)
+ return false, err
+ }
+
+ return true, nil
+}
+
+func (s *Dir) StoragePoolUmount() (bool, error) {
+ cleanupFunc := LockPoolUmount(s.pool.Name)
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ source := s.pool.Config["source"]
+ if source == "" {
+ return false, fmt.Errorf("no \"source\" property found for the storage pool")
+ }
+
+ cleanSource := filepath.Clean(source)
+ poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+
+ if cleanSource == poolMntPoint {
+ return true, nil
+ }
+
+ if !shared.IsMountPoint(poolMntPoint) {
+ return false, nil
+ }
+
+ err := syscall.Unmount(poolMntPoint, 0)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+}
+
+func (s *Dir) StoragePoolResources() (*api.ResourcesStoragePool, error) {
+ ourMount, err := s.StoragePoolMount()
+ if err != nil {
+ return nil, err
+ }
+
+ if ourMount {
+ defer s.StoragePoolUmount()
+ }
+
+ return storageResource(getStoragePoolMountPoint(s.pool.Name))
+}
+
+func (s *Dir) StoragePoolUpdate(writable *api.StoragePoolPut,
+ changedConfig []string) error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Dir) VolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Dir) VolumeRestore(project string, sourceName string, targetName string, volumeType VolumeType) error {
+ return s.VolumeSnapshotRestore(project, sourceName, targetName, volumeType)
+}
+
+func (s *Dir) VolumeCreate(project string, volumeName string, volumeType VolumeType) error {
+ var volumePath string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ volumePath = getContainerMountPoint(project, s.pool.Name, volumeName)
+ case VolumeTypeImage:
+ volumePath = getImageMountPoint(s.pool.Name, volumeName)
+ case VolumeTypeCustom:
+ volumePath = getStoragePoolVolumeMountPoint(s.pool.Name, volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ err := os.MkdirAll(volumePath, 0711)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Dir) VolumeCopy(project string, source string, target string, snapshots []string, volumeType VolumeType) error {
+ var sourceMountPoint string
+ var targetMountPoint string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ for _, snap := range snapshots {
+ sourceMountPoint = getSnapshotMountPoint(project, s.pool.Name, fmt.Sprintf("%s/%s", source, snap))
+ targetMountPoint = getSnapshotMountPoint(project, s.pool.Name, fmt.Sprintf("%s/%s", target, snap))
+
+ err := s.rsync(sourceMountPoint, targetMountPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ sourceMountPoint = getContainerMountPoint(project, s.pool.Name, source)
+ targetMountPoint = getContainerMountPoint(project, s.pool.Name, target)
+ case VolumeTypeCustom:
+ for _, snap := range snapshots {
+ sourceMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, fmt.Sprintf("%s/%s", source, snap))
+ targetMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, fmt.Sprintf("%s/%s", target, snap))
+
+ err := s.rsync(sourceMountPoint, targetMountPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ sourceMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, source)
+ targetMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, target)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return s.rsync(sourceMountPoint, targetMountPoint)
+}
+
+func (s *Dir) doCopy(source string, target string) error {
+ err := s.rsync(source, target)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Dir) VolumeDelete(project string, volumeName string, recursive bool, volumeType VolumeType) error {
+ var path string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ path = getContainerMountPoint(project, s.pool.Name, volumeName)
+ case VolumeTypeCustom:
+ path = getStoragePoolVolumeMountPoint(s.pool.Name, volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ err := os.RemoveAll(path)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Dir) VolumeRename(project string, oldName string, newName string, snapshots []string,
+ volumeType VolumeType) error {
+ var oldPath string
+ var newPath string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ oldPath = getContainerMountPoint(project, s.pool.Name, oldName)
+ newPath = getContainerMountPoint(project, s.pool.Name, newName)
+ case VolumeTypeCustom:
+ oldPath = getStoragePoolVolumeMountPoint(s.pool.Name, oldName)
+ newPath = getStoragePoolVolumeMountPoint(s.pool.Name, newName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if shared.PathExists(newPath) {
+ // Nothing to do
+ return nil
+ }
+
+ return os.Rename(oldPath, newPath)
+}
+
+func (s *Dir) VolumeMount(project string, name string, volumeType VolumeType) (bool, error) {
+ // Nothing to do
+ return true, nil
+}
+
+func (s *Dir) VolumeUmount(project string, name string, volumeType VolumeType) (bool, error) {
+ // Nothing to do
+ return true, nil
+}
+
+func (s *Dir) VolumeGetUsage(project, name string,
+ path string) (int64, error) {
+ return -1, fmt.Errorf("The directory container backend doesn't support quotas")
+}
+
+func (s *Dir) VolumeSetQuota(project, name string,
+ size int64, userns bool, volumeType VolumeType) error {
+ return nil
+}
+
+func (s *Dir) VolumeSnapshotCreate(project string, source string, target string, volumeType VolumeType) error {
+ var sourceMountPoint string
+ var targetMountPoint string
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ sourceMountPoint = getContainerMountPoint(project, s.pool.Name, source)
+ targetMountPoint = getSnapshotMountPoint(project, s.pool.Name, target)
+ case VolumeTypeCustomSnapshot:
+ sourceMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, source)
+ targetMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target)
+
+ err := os.MkdirAll(targetMountPoint, 0711)
+ if err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if source == "" {
+ // Nothing to do
+ return nil
+ }
+
+ return s.doCopy(sourceMountPoint, targetMountPoint)
+}
+
+func (s *Dir) VolumeSnapshotCopy(project string, source string, target string, volumeType VolumeType) error {
+ var sourceMountPoint string
+ var targetMountPoint string
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ sourceMountPoint = getSnapshotMountPoint(project, s.pool.Name, source)
+
+ if shared.IsSnapshot(target) {
+ targetMountPoint = getSnapshotMountPoint(project, s.pool.Name, target)
+ } else {
+ targetMountPoint = getContainerMountPoint(project, s.pool.Name, target)
+ }
+ case VolumeTypeCustomSnapshot:
+ sourceMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, source)
+
+ if shared.IsSnapshot(target) {
+ targetMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target)
+ } else {
+ targetMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, target)
+ }
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return s.doCopy(sourceMountPoint, targetMountPoint)
+}
+
+func (s *Dir) VolumeSnapshotDelete(project string, volumeName string, recursive bool, volumeType VolumeType) error {
+ var path string
+
+ switch volumeType {
+ case VolumeTypeCustomSnapshot:
+ path = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, volumeName)
+ case VolumeTypeContainerSnapshot:
+ path = getSnapshotMountPoint(project, s.pool.Name, volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ err := os.RemoveAll(path)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Dir) VolumeSnapshotRestore(project string, sourceName string, targetName string, volumeType VolumeType) error {
+ var sourceMountPoint string
+ var targetMountPoint string
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ sourceMountPoint = getSnapshotMountPoint(project, s.pool.Name, sourceName)
+ targetMountPoint = getContainerMountPoint(project, s.pool.Name, targetName)
+ case VolumeTypeCustomSnapshot:
+ sourceMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceName)
+ targetMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, targetName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return s.doCopy(sourceMountPoint, targetMountPoint)
+}
+
+func (s *Dir) VolumeSnapshotRename(project string, oldName string, newName string, volumeType VolumeType) error {
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ // Nothing to do as this is handled by the storage itself
+ case VolumeTypeCustomSnapshot:
+ // Nothing to do as this is handled by the storage itself
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+
+ }
+
+ return nil
+}
+
+func (s *Dir) VolumeBackupCreate(path string, project string, source string,
+ snapshots []string, optimized bool) error {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", path)
+
+ if len(snapshots) > 0 {
+ err := os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return err
+ }
+ }
+ for _, snap := range snapshots {
+ fullSnapshotName := fmt.Sprintf("%s/%s", s.volume.Name, snap)
+ snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, fullSnapshotName)
+ target := fmt.Sprintf("%s/%s", snapshotsPath, snap)
+
+ // Copy the snapshot
+ err := s.rsync(snapshotMntPoint, target)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Copy the container
+ sourcePath := containerPath(project, source, false)
+ targetPath := fmt.Sprintf("%s/container", path)
+ err := s.rsync(sourcePath, targetPath)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Dir) VolumeBackupLoad(backupDir string, project string,
+ containerName string, snapshots []string, privileged bool, optimized bool) error {
+ if optimized {
+ return fmt.Errorf("Dir storage doesn't support binary backups")
+ }
+
+ // Nothing to do
+ return nil
+}
+
+func (s *Dir) VolumePrepareRestore(sourceName string, targetName string, targetSnapshots []string, f func() error) error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Dir) VolumeReady(project string, name string) bool {
+ containerMntPoint := getContainerMountPoint(project, s.pool.Name, name)
+ ok, _ := shared.PathIsEmpty(containerMntPoint)
+ return !ok
+}
+
+func DirSnapshotDeleteInternal(project, poolName string, snapshotName string) error {
+ snapshotContainerMntPoint := getSnapshotMountPoint(project, poolName, snapshotName)
+ if shared.PathExists(snapshotContainerMntPoint) {
+ err := os.RemoveAll(snapshotContainerMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ sourceContainerName, _, _ := containerGetParentAndSnapshotName(snapshotName)
+ snapshotContainerPath := getSnapshotMountPoint(project, poolName, sourceContainerName)
+ empty, _ := shared.PathIsEmpty(snapshotContainerPath)
+ if empty == true {
+ err := os.Remove(snapshotContainerPath)
+ if err != nil {
+ return err
+ }
+
+ snapshotSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceContainerName))
+ if shared.PathExists(snapshotSymlink) {
+ err := os.Remove(snapshotSymlink)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
From 42555cfec2b3687de241a5ac55160a07032c81ee Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 15:31:25 +0200
Subject: [PATCH 11/15] storage: Add zfs
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage/zfs.go | 2759 ++++++++++++++++++++++++++++++++++
lxd/storage_migration_zfs.go | 372 +++++
2 files changed, 3131 insertions(+)
create mode 100644 lxd/storage/zfs.go
create mode 100644 lxd/storage_migration_zfs.go
diff --git a/lxd/storage/zfs.go b/lxd/storage/zfs.go
new file mode 100644
index 0000000000..27a5718945
--- /dev/null
+++ b/lxd/storage/zfs.go
@@ -0,0 +1,2759 @@
+package storage
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/lxc/lxd/lxd/util"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/logger"
+ "github.com/pborman/uuid"
+ "github.com/pkg/errors"
+)
+
+type Zfs struct {
+ driverShared
+
+ dataset string
+}
+
+// Global defaults
+var zfsUseRefquota = "false"
+
+// Cache
+var ZfsVersion = ""
+
+func (s *Zfs) Init() error {
+ if s.pool != nil && s.pool.Config["zfs.pool_name"] != "" {
+ s.dataset = s.pool.Config["zfs.pool_name"]
+ }
+
+ if ZfsVersion != "" {
+ s.sTypeVersion = ZfsVersion
+ return nil
+ }
+
+ util.LoadModule("zfs")
+
+ if !zfsIsEnabled() {
+ return fmt.Errorf("The \"zfs\" tool is not enabled")
+ }
+
+ var err error
+
+ s.sTypeVersion, err = zfsToolVersionGet()
+ if err != nil {
+ s.sTypeVersion, err = zfsModuleVersionGet()
+ if err != nil {
+ return err
+ }
+ }
+
+ ZfsVersion = s.sTypeVersion
+
+ return nil
+}
+
+func (s *Zfs) StoragePoolCheck() error {
+ source := s.pool.Config["source"]
+ if source == "" {
+ return fmt.Errorf("no \"source\" property found for the storage pool")
+ }
+
+ poolName := s.getOnDiskPoolName()
+ purePoolName := strings.Split(poolName, "/")[0]
+
+ exists := ZfsFilesystemEntityExists(purePoolName, "")
+ if exists {
+ return nil
+ }
+
+ var err error
+ var msg string
+
+ if filepath.IsAbs(source) {
+ disksPath := shared.VarPath("disks")
+ msg, err = shared.RunCommand("zpool", "import", "-d", disksPath, poolName)
+ } else {
+ msg, err = shared.RunCommand("zpool", "import", purePoolName)
+ }
+
+ if err != nil {
+ return fmt.Errorf("ZFS storage pool \"%s\" could not be imported: %s", poolName, msg)
+ }
+
+ return nil
+}
+
+func (s *Zfs) StoragePoolCreate() error {
+ err := s.zfsPoolCreate()
+ if err != nil {
+ return err
+ }
+ revert := true
+ defer func() {
+ if !revert {
+ return
+ }
+ s.StoragePoolDelete()
+ }()
+
+ storagePoolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+ err = os.MkdirAll(storagePoolMntPoint, 0711)
+ if err != nil {
+ return err
+ }
+
+ err = s.StoragePoolCheck()
+ if err != nil {
+ return err
+ }
+
+ revert = false
+
+ return nil
+}
+
+func (s *Zfs) StoragePoolDelete() error {
+ poolName := s.getOnDiskPoolName()
+
+ if ZfsFilesystemEntityExists(poolName, "") {
+ err := zfsFilesystemEntityDelete(s.pool.Config["source"], poolName)
+ if err != nil {
+ return err
+ }
+ }
+
+ storagePoolMntPoint := getStoragePoolMountPoint(s.pool.Name)
+ if shared.PathExists(storagePoolMntPoint) {
+ err := os.RemoveAll(storagePoolMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *Zfs) StoragePoolMount() (bool, error) {
+ return true, nil
+}
+
+func (s *Zfs) StoragePoolUmount() (bool, error) {
+ return true, nil
+}
+
+func (s *Zfs) StoragePoolResources() (*api.ResourcesStoragePool, error) {
+ poolName := s.getOnDiskPoolName()
+
+ totalBuf, err := zfsFilesystemEntityPropertyGet(poolName, "", "available")
+ if err != nil {
+ return nil, err
+ }
+
+ totalStr := string(totalBuf)
+ totalStr = strings.TrimSpace(totalStr)
+
+ total, err := strconv.ParseUint(totalStr, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+
+ usedBuf, err := zfsFilesystemEntityPropertyGet(poolName, "", "used")
+ if err != nil {
+ return nil, err
+ }
+
+ usedStr := string(usedBuf)
+ usedStr = strings.TrimSpace(usedStr)
+
+ used, err := strconv.ParseUint(usedStr, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+
+ res := api.ResourcesStoragePool{}
+ res.Space.Total = total
+ res.Space.Used = used
+
+ // Inode allocation is dynamic so no use in reporting them.
+ return &res, nil
+}
+
+func (s *Zfs) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error {
+ // Nothing to do
+ return nil
+}
+
+func (s *Zfs) doImageCreate(fingerprint string) error {
+ poolName := s.getOnDiskPoolName()
+ imageMntPoint := getImageMountPoint(s.pool.Name, fingerprint)
+ fs := fmt.Sprintf("images/%s", fingerprint)
+ revert := true
+ subrevert := true
+
+ if ZfsFilesystemEntityExists(poolName, fmt.Sprintf("deleted/%s", fs)) {
+ err := zfsPoolVolumeRename(poolName, fmt.Sprintf("deleted/%s", fs), fs, true)
+ if err != nil {
+ return err
+ }
+
+ defer func() {
+ if !revert {
+ return
+ }
+ s.VolumeDelete("default", fingerprint, true, VolumeTypeImage)
+ }()
+
+ // In case this is an image from an older lxd instance, wipe the
+ // mountpoint.
+ err = zfsPoolVolumeSet(poolName, fs, "mountpoint", "none")
+ if err != nil {
+ return err
+ }
+
+ revert = false
+ subrevert = false
+
+ return nil
+ }
+
+ err := os.MkdirAll(imageMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if !subrevert {
+ return
+ }
+ os.RemoveAll(imageMntPoint)
+ }()
+
+ // Create temporary mountpoint directory.
+ tmp := getImageMountPoint(s.pool.Name, "")
+
+ tmpImageDir, err := ioutil.TempDir(tmp, "")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpImageDir)
+
+ imagePath := shared.VarPath("images", fingerprint)
+
+ // Create a new storage volume on the storage pool for the image.
+ dataset := fmt.Sprintf("%s/%s", poolName, fs)
+
+ _, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ return err
+ }
+
+ subrevert = false
+ defer func() {
+ if !revert {
+ return
+ }
+ s.VolumeDelete("default", fingerprint, true, VolumeTypeImage)
+ }()
+
+ // Set a temporary mountpoint for the image.
+ err = zfsPoolVolumeSet(poolName, fs, "mountpoint", tmpImageDir)
+ if err != nil {
+ return err
+ }
+
+ // Make sure that the image actually got mounted.
+ if !shared.IsMountPoint(tmpImageDir) {
+ ZfsMount(poolName, fs)
+ }
+
+ // Unpack the image into the temporary mountpoint.
+ err = unpackImage(imagePath, tmpImageDir, false, s.s.OS.RunningInUserNS, nil)
+ if err != nil {
+ return err
+ }
+
+ // Mark the new storage volume for the image as readonly.
+ err = zfsPoolVolumeSet(poolName, fs, "readonly", "on")
+ if err != nil {
+ return err
+ }
+
+ // Remove the temporary mountpoint from the image storage volume.
+ err = zfsPoolVolumeSet(poolName, fs, "mountpoint", "none")
+ if err != nil {
+ return err
+ }
+
+ // Make sure that the image actually got unmounted.
+ if shared.IsMountPoint(tmpImageDir) {
+ ZfsUmount(poolName, fs, tmpImageDir)
+ }
+
+ // Create a snapshot of that image on the storage pool which we clone for
+ // container creation.
+ err = ZfsPoolVolumeSnapshotCreate(poolName, fs, "readonly")
+ if err != nil {
+ return err
+ }
+
+ revert = false
+
+ return nil
+}
+
+func (s *Zfs) VolumeCreate(project string, volumeName string, volumeType VolumeType) error {
+ var volumeMntPoint string
+ var dataset string
+ var fs string
+
+ if shared.IsSnapshot(volumeName) {
+ return fmt.Errorf("Volume is a snapshot")
+ }
+
+ poolName := s.getOnDiskPoolName()
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ dataset = fmt.Sprintf("%s/containers/%s", poolName, projectPrefix(project, volumeName))
+ volumeMntPoint = getContainerMountPoint(project, s.pool.Name, volumeName)
+ fs = s.getDataset(project, volumeName, "containers")
+ case VolumeTypeCustom:
+ dataset = fmt.Sprintf("%s/custom/%s", poolName, volumeName)
+ volumeMntPoint = getStoragePoolVolumeMountPoint(s.pool.Name, volumeName)
+ fs = s.getDataset("default", volumeName, "custom")
+ case VolumeTypeImage:
+ return s.doImageCreate(volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ _, err := zfsPoolVolumeCreate(dataset, fmt.Sprintf("mountpoint=%s", volumeMntPoint),
+ "canmount=noauto")
+ if err != nil {
+ return err
+ }
+
+ if !shared.IsMountPoint(volumeMntPoint) {
+ err := ZfsMount(poolName, fs)
+ if err != nil {
+ return err
+ }
+ defer ZfsUmount(poolName, fs, volumeMntPoint)
+ }
+
+ /*
+ // apply quota
+ if s.volume.Config["size"] != "" {
+ size, err := shared.ParseByteSizeString(s.volume.Config["size"])
+ if err != nil {
+ return err
+ }
+
+ err = s.StorageEntitySetQuota(storagePoolVolumeTypeCustom, size, nil)
+ if err != nil {
+ return err
+ }
+ }
+
+ revert = false
+ */
+
+ return nil
+}
+
+func (s *Zfs) VolumeCopy(project, source string, target string, snapshots []string, volumeType VolumeType) error {
+ switch volumeType {
+ case VolumeTypeContainer:
+ if len(snapshots) == 0 {
+ return s.doContainerCopy(project, source, target)
+ }
+
+ return s.doContainerCopyWithSnapshots(project, source, target, snapshots)
+ case VolumeTypeCustom:
+ if len(snapshots) == 0 {
+ return s.doVolumeCopy(source, target)
+ }
+
+ return s.doVolumeCopyWithSnapshots(source, target, snapshots)
+ case VolumeTypeImage:
+ poolName := s.getOnDiskPoolName()
+ sourceFs := fmt.Sprintf("images/%s", source)
+ targetFs := fmt.Sprintf("containers/%s", projectPrefix(project, target))
+ volumeMountPoint := getContainerMountPoint(project, s.pool.Name, target)
+
+ return zfsPoolVolumeClone(project, poolName, sourceFs, "readonly", targetFs, volumeMountPoint)
+ }
+
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+}
+
+func (s *Zfs) doContainerCopy(project string, source string, target string) error {
+ if s.pool.Config["zfs.clone_copy"] != "" && !shared.IsTrue(s.pool.Config["zfs.clone_copy"]) {
+ return s.doCopyWithoutSnapshotFull(project, source, target)
+ }
+
+ return s.doCopyWithoutSnapshotsSparse(project, source, target)
+}
+
+func (s *Zfs) doCopyWithoutSnapshotsSparse(project string, source string, target string) error {
+ poolName := s.getOnDiskPoolName()
+
+ sourceContainerName := source
+ sourceContainerPath := containerPath(project, source, false)
+
+ targetContainerName := target
+ targetContainerPath := containerPath(project, target, false)
+ targetContainerMountPoint := getContainerMountPoint(project, s.pool.Name, targetContainerName)
+
+ sourceZfsDataset := ""
+ sourceZfsDatasetSnapshot := ""
+ sourceName, sourceSnapOnlyName, isSnapshotName := containerGetParentAndSnapshotName(sourceContainerName)
+
+ targetZfsDataset := s.getDataset(project, target, "containers")
+
+ if isSnapshotName {
+ sourceZfsDatasetSnapshot = sourceSnapOnlyName
+ }
+
+ revert := true
+ if sourceZfsDatasetSnapshot == "" {
+ if ZfsFilesystemEntityExists(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, sourceName))) {
+ sourceZfsDatasetSnapshot = fmt.Sprintf("copy-%s", uuid.NewRandom().String())
+ sourceZfsDataset = fmt.Sprintf("containers/%s", projectPrefix(project, sourceName))
+
+ err := ZfsPoolVolumeSnapshotCreate(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if !revert {
+ return
+ }
+ ZfsPoolVolumeSnapshotDestroy(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
+ }()
+ }
+ } else {
+ if ZfsFilesystemEntityExists(poolName, fmt.Sprintf("containers/%s at snapshot-%s", projectPrefix(project, sourceName), sourceZfsDatasetSnapshot)) {
+ sourceZfsDataset = fmt.Sprintf("containers/%s", projectPrefix(project, sourceName))
+ sourceZfsDatasetSnapshot = fmt.Sprintf("snapshot-%s", sourceZfsDatasetSnapshot)
+ }
+ }
+
+ if sourceZfsDataset != "" {
+ err := zfsPoolVolumeClone(project, poolName, sourceZfsDataset, sourceZfsDatasetSnapshot, targetZfsDataset, targetContainerMountPoint)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if !revert {
+ return
+ }
+ zfsPoolVolumeDestroy(poolName, targetZfsDataset)
+ }()
+
+ // TODO: This should be called in the storage not the driver
+ /*
+ ourMount, err := s.ContainerMount(target)
+ if err != nil {
+ return err
+ }
+ if ourMount {
+ defer s.ContainerUmount(target, targetContainerPath)
+ }
+
+ err = createContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged())
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if !revert {
+ return
+ }
+ deleteContainerMountpoint(targetContainerMountPoint, targetContainerPath, s.GetStorageTypeName())
+ }()
+ */
+ } else {
+ err := s.VolumeCreate(project, target, VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if !revert {
+ return
+ }
+ s.VolumeDelete(project, target, false, VolumeTypeContainer)
+ }()
+
+ err = s.rsync(sourceContainerPath, targetContainerPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ revert = false
+
+ return nil
+}
+
+func (s *Zfs) doCopyWithoutSnapshotFull(project string, source string, target string) error {
+ sourceIsSnapshot := shared.IsSnapshot(source)
+ poolName := s.getOnDiskPoolName()
+
+ sourceDataset := ""
+ snapshotSuffix := ""
+
+ targetName := target
+ targetDataset := fmt.Sprintf("%s/containers/%s", poolName, projectPrefix(project, targetName))
+ targetSnapshotDataset := ""
+
+ if sourceIsSnapshot {
+ sourceParentName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(source)
+ snapshotSuffix = fmt.Sprintf("snapshot-%s", sourceSnapOnlyName)
+ sourceDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, sourceParentName), snapshotSuffix)
+ targetSnapshotDataset = fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(project, targetName), sourceSnapOnlyName)
+ } else {
+ snapshotSuffix = uuid.NewRandom().String()
+ sourceDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, source), snapshotSuffix)
+ targetSnapshotDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, targetName), snapshotSuffix)
+
+ fs := fmt.Sprintf("containers/%s", projectPrefix(project, source))
+ err := ZfsPoolVolumeSnapshotCreate(poolName, fs, snapshotSuffix)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ err := ZfsPoolVolumeSnapshotDestroy(poolName, fs, snapshotSuffix)
+ if err != nil {
+ logger.Warnf("Failed to delete temporary ZFS snapshot \"%s\", manual cleanup needed", sourceDataset)
+ }
+ }()
+ }
+
+ zfsSendCmd := exec.Command("zfs", "send", sourceDataset)
+
+ zfsRecvCmd := exec.Command("zfs", "receive", targetDataset)
+
+ zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
+ zfsRecvCmd.Stdout = os.Stdout
+ zfsRecvCmd.Stderr = os.Stderr
+
+ err := zfsRecvCmd.Start()
+ if err != nil {
+ return err
+ }
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ err = zfsRecvCmd.Wait()
+ if err != nil {
+ return err
+ }
+
+ msg, err := shared.RunCommand("zfs", "rollback", "-r", "-R", targetSnapshotDataset)
+ if err != nil {
+ logger.Errorf("Failed to rollback ZFS dataset: %s", msg)
+ return err
+ }
+
+ targetContainerMountPoint := getContainerMountPoint(project, s.pool.Name, targetName)
+ targetfs := fmt.Sprintf("containers/%s", projectPrefix(project, targetName))
+
+ err = zfsPoolVolumeSet(poolName, targetfs, "canmount", "noauto")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(poolName, targetfs, "mountpoint", targetContainerMountPoint)
+ if err != nil {
+ return err
+ }
+
+ err = ZfsPoolVolumeSnapshotDestroy(poolName, targetfs, snapshotSuffix)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) getFullDataset(project string, name string, datasetType string) string {
+ return fmt.Sprintf("%s/%s", s.getOnDiskPoolName(), s.getDataset(project, name, datasetType))
+}
+
+func (s *Zfs) getDataset(project string, name string, datasetType string) string {
+ var out string
+
+ if shared.IsSnapshot(name) {
+ parentName, snapshotName, _ := containerGetParentAndSnapshotName(name)
+
+ if project != "" {
+ parentName = projectPrefix(project, parentName)
+ }
+
+ out = fmt.Sprintf("%s/%s at snapshot-%s", datasetType, parentName, snapshotName)
+ } else {
+ if project != "" {
+ name = projectPrefix(project, name)
+ }
+
+ out = fmt.Sprintf("%s/%s", datasetType, name)
+ }
+
+ return out
+}
+
+func (s *Zfs) doCopyWithSnapshots(project string, source string, target string, parentSnapshot string) error {
+ currentSnapshotDataset := s.getFullDataset(project, source, "containers")
+ targetSnapshotDataset := s.getFullDataset(project, target, "containers")
+
+ args := []string{"send", currentSnapshotDataset}
+
+ if parentSnapshot != "" {
+ parentSnapshotDataset := s.getFullDataset(project, parentSnapshot, "containers")
+ args = append(args, "-i", parentSnapshotDataset)
+ }
+
+ logger.Debugf("zfs args: %+v", args)
+ logger.Debugf("zfs targetSnapshotDataset: %v", targetSnapshotDataset)
+
+ zfsSendCmd := exec.Command("zfs", args...)
+
+ zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetSnapshotDataset)
+ zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
+ zfsRecvCmd.Stdout = os.Stdout
+ zfsRecvCmd.Stderr = os.Stderr
+
+ err := zfsRecvCmd.Start()
+ if err != nil {
+ return err
+ }
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ err = zfsRecvCmd.Wait()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) doContainerCopyWithSnapshots(project string, source string, target string, snapshots []string) error {
+ targetContainerName := target
+ //targetContainerPath := containerPath(project, target, false)
+
+ targetContainerMountPoint := getContainerMountPoint(project, s.pool.Name, targetContainerName)
+
+ prev := ""
+ prevSnapOnlyName := ""
+
+ for i, snap := range snapshots {
+ if i > 0 {
+ prev = fmt.Sprintf("%s/%s", source, snapshots[i-1])
+ }
+
+ oldSnapName := fmt.Sprintf("%s/%s", source, snap)
+ newSnapName := fmt.Sprintf("%s/%s", target, snap)
+ prevSnapOnlyName = snap
+
+ err := s.doCopyWithSnapshots(project, oldSnapName, newSnapName, prev)
+ if err != nil {
+ logger.Error("Failed to copy snapshots")
+ return err
+ }
+ }
+
+ poolName := s.getOnDiskPoolName()
+
+ // send actual container
+ tmpSnapshotName := fmt.Sprintf("copy-send-%s", uuid.NewRandom().String())
+
+ err := ZfsPoolVolumeSnapshotCreate(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, source)), tmpSnapshotName)
+ if err != nil {
+ return err
+ }
+
+ currentSnapshotDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, source), tmpSnapshotName)
+ args := []string{"send", currentSnapshotDataset}
+ if prevSnapOnlyName != "" {
+ parentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(project, source), prevSnapOnlyName)
+ args = append(args, "-i", parentSnapshotDataset)
+ }
+
+ zfsSendCmd := exec.Command("zfs", args...)
+ targetSnapshotDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, target), tmpSnapshotName)
+ zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetSnapshotDataset)
+
+ zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
+ zfsRecvCmd.Stdout = os.Stdout
+ zfsRecvCmd.Stderr = os.Stderr
+
+ err = zfsRecvCmd.Start()
+ if err != nil {
+ return err
+ }
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ err = zfsRecvCmd.Wait()
+ if err != nil {
+ return err
+ }
+
+ ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, source)), tmpSnapshotName)
+ ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, target)), tmpSnapshotName)
+
+ fs := fmt.Sprintf("containers/%s", projectPrefix(project, target))
+ err = zfsPoolVolumeSet(poolName, fs, "canmount", "noauto")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(poolName, fs, "mountpoint", targetContainerMountPoint)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) doVolumeCopy(source string, target string) error {
+ var err error
+
+ //fs := fmt.Sprintf("custom/%s", target)
+
+ if s.pool.Config["zfs.clone_copy"] != "" && !shared.IsTrue(s.pool.Config["zfs.clone_copy"]) {
+ err = s.doCopyVolumeWithoutSnapshotsFull(source)
+ } else {
+ err = s.doCopyVolumeWithoutSnapshotsSparse(source)
+ }
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) doCopyVolumeWithSnapshots(source string, target string, parentSnapshot string) error {
+ poolName := s.getOnDiskPoolName()
+
+ currentSnapshotDataset := s.getFullDataset("default", source, "custom")
+ targetSnapshotDataset := s.getFullDataset("default", target, "custom")
+
+ args := []string{"send", currentSnapshotDataset}
+
+ if parentSnapshot != "" {
+ parentName, parentSnaponlyName, _ := containerGetParentAndSnapshotName(parentSnapshot)
+ parentSnapshotDataset := fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, parentName, parentSnaponlyName)
+ args = append(args, "-i", parentSnapshotDataset)
+ }
+
+ zfsSendCmd := exec.Command("zfs", args...)
+
+ zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetSnapshotDataset)
+ zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
+ zfsRecvCmd.Stdout = os.Stdout
+ zfsRecvCmd.Stderr = os.Stderr
+
+ err := zfsRecvCmd.Start()
+ if err != nil {
+ return err
+ }
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ err = zfsRecvCmd.Wait()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) doVolumeCopyWithSnapshots(source string, target string, snapshots []string) error {
+ targetVolumeMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, target)
+
+ snapshots, err := ZfsPoolListSnapshots(s.getOnDiskPoolName(), fmt.Sprintf("custom/%s", source))
+ if err != nil {
+ return err
+ }
+
+ prev := ""
+ prevSnapOnlyName := ""
+
+ for i, snap := range snapshots {
+ if i > 0 {
+ prev = snapshots[i-1]
+ }
+
+ oldSnapName := fmt.Sprintf("%s/%s", source, snap)
+ newSnapName := fmt.Sprintf("%s/%s", target, snap)
+
+ err = s.doCopyVolumeWithSnapshots(oldSnapName, newSnapName, prev)
+ if err != nil {
+ return err
+ }
+ }
+
+ poolName := s.getOnDiskPoolName()
+
+ // send actual container
+ tmpSnapshotName := fmt.Sprintf("copy-send-%s", uuid.NewRandom().String())
+ err = ZfsPoolVolumeSnapshotCreate(poolName, fmt.Sprintf("custom/%s", source), tmpSnapshotName)
+ if err != nil {
+ return err
+ }
+
+ currentSnapshotDataset := fmt.Sprintf("%s/custom/%s@%s", poolName, source, tmpSnapshotName)
+ args := []string{"send", currentSnapshotDataset}
+ if prevSnapOnlyName != "" {
+ parentSnapshotDataset := fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, source, prevSnapOnlyName)
+ args = append(args, "-i", parentSnapshotDataset)
+ }
+
+ zfsSendCmd := exec.Command("zfs", args...)
+ targetSnapshotDataset := fmt.Sprintf("%s/custom/%s@%s", poolName, target, tmpSnapshotName)
+ zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetSnapshotDataset)
+
+ zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
+ zfsRecvCmd.Stdout = os.Stdout
+ zfsRecvCmd.Stderr = os.Stderr
+
+ err = zfsRecvCmd.Start()
+ if err != nil {
+ return err
+ }
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ err = zfsRecvCmd.Wait()
+ if err != nil {
+ return err
+ }
+
+ ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("custom/%s", source), tmpSnapshotName)
+ ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("custom/%s", target), tmpSnapshotName)
+
+ fs := fmt.Sprintf("custom/%s", target)
+ err = zfsPoolVolumeSet(poolName, fs, "canmount", "noauto")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(poolName, fs, "mountpoint", targetVolumeMountPoint)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumeDelete(project, volumeName string, recursive bool, volumeType VolumeType) error {
+ var fs string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ fs = fmt.Sprintf("containers/%s", projectPrefix(project, volumeName))
+ case VolumeTypeCustom:
+ fs = fmt.Sprintf("custom/%s", volumeName)
+ case VolumeTypeImage:
+ fs = fmt.Sprintf("images/%s", volumeName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ poolName := s.getOnDiskPoolName()
+
+ if ZfsFilesystemEntityExists(poolName, fs) {
+ removable := true
+ snaps, err := ZfsPoolListSnapshots(poolName, fs)
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snaps {
+ var err error
+ removable, err = zfsPoolVolumeSnapshotRemovable(poolName, fs, snap)
+ if err != nil {
+ return err
+ }
+
+ if !removable {
+ break
+ }
+ }
+
+ if removable {
+ origin, err := zfsFilesystemEntityPropertyGet(poolName, fs, "origin")
+ if err != nil {
+ return err
+ }
+
+ origin = strings.TrimPrefix(origin, fmt.Sprintf("%s/", poolName))
+
+ err = zfsPoolVolumeDestroy(poolName, fs)
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeCleanup(poolName, origin)
+ if err != nil {
+ return err
+ }
+ } else {
+ err := zfsPoolVolumeSet(poolName, fs, "mountpoint", "none")
+ if err != nil {
+ return err
+ }
+
+ var target string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ target = fmt.Sprintf("deleted/containers/%s", uuid.NewRandom().String())
+ case VolumeTypeCustom:
+ target = fmt.Sprintf("deleted/custom/%s", uuid.NewRandom().String())
+ case VolumeTypeImage:
+ target = fmt.Sprintf("deleted/images/%s", uuid.NewRandom().String())
+ }
+
+ err = zfsPoolVolumeRename(poolName, fs, target, true)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumeRename(project, oldName string, newName string, snapshots []string, volumeType VolumeType) error {
+ var oldDataset string
+ var newDataset string
+ var newMountPoint string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ oldDataset = fmt.Sprintf("containers/%s", projectPrefix(project, oldName))
+ newDataset = fmt.Sprintf("containers/%s", projectPrefix(project, newName))
+ newMountPoint = getContainerMountPoint(project, s.pool.Name, newName)
+ case VolumeTypeCustom:
+ oldDataset = fmt.Sprintf("custom/%s", oldName)
+ newDataset = fmt.Sprintf("custom/%s", newName)
+ newMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, newName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ poolName := s.getOnDiskPoolName()
+
+ err := zfsPoolVolumeRename(poolName, oldDataset, newDataset, false)
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(poolName, newDataset, "mountpoint", newMountPoint)
+ if err != nil {
+ return err
+ }
+
+ _, err = s.VolumeUmount(project, newName, VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumeMount(project string, name string, volumeType VolumeType) (bool, error) {
+ var cleanupFunc func()
+ var fs string
+ var mountPoint string
+ poolName := s.getOnDiskPoolName()
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ cleanupFunc = LockContainerMount(poolName, name)
+ fs = fmt.Sprintf("containers/%s", projectPrefix(project, name))
+ mountPoint = getContainerMountPoint(project, s.pool.Name, name)
+ case VolumeTypeCustom:
+ cleanupFunc = LockCustomMount(poolName, name)
+ fs = fmt.Sprintf("custom/%s", name)
+ mountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, name)
+ case VolumeTypeContainerSnapshot:
+ cName, sName, _ := containerGetParentAndSnapshotName(name)
+ sourceSnap := fmt.Sprintf("snapshot-%s", sName)
+ sourceFs := s.getDataset(project, cName, "containers")
+ destFs := fmt.Sprintf("snapshots/%s/%s", projectPrefix(project, cName), sName)
+ poolName := s.getOnDiskPoolName()
+ snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, name)
+
+ err := zfsPoolVolumeClone(project, poolName, sourceFs, sourceSnap, destFs, snapshotMntPoint)
+ if err != nil {
+ return false, err
+ }
+
+ err = ZfsMount(poolName, destFs)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+ default:
+ return false, fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ ourMount := false
+
+ if !shared.IsMountPoint(mountPoint) {
+ ourMount = true
+
+ err := ZfsMount(s.getOnDiskPoolName(), fs)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ return ourMount, nil
+}
+
+func (s *Zfs) VolumeUmount(project, name string, volumeType VolumeType) (bool, error) {
+ var cleanupFunc func()
+ var mountPoint string
+ var fs string
+
+ poolName := s.getOnDiskPoolName()
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ cleanupFunc = LockContainerUmount(poolName, name)
+ mountPoint = getContainerMountPoint(project, s.pool.Name, name)
+ fs = fmt.Sprintf("containers/%s", projectPrefix(project, name))
+ case VolumeTypeCustom:
+ cleanupFunc = LockCustomUmount(poolName, name)
+ mountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, name)
+ fs = fmt.Sprintf("custom/%s", name)
+ case VolumeTypeContainerSnapshot:
+ cleanupFunc = LockContainerUmount(poolName, name)
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ cName, sName, _ := containerGetParentAndSnapshotName(name)
+ destFs := fmt.Sprintf("snapshots/%s/%s", projectPrefix(project, cName), sName)
+
+ err := zfsPoolVolumeDestroy(poolName, destFs)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+ default:
+ return false, fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if cleanupFunc == nil {
+ return false, nil
+ }
+ defer cleanupFunc()
+
+ ourUmount := false
+
+ if shared.IsMountPoint(mountPoint) {
+ ourUmount = true
+
+ err := ZfsUmount(poolName, fs, mountPoint)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ return ourUmount, nil
+}
+
+func (s *Zfs) VolumeGetUsage(project, name, path string) (int64, error) {
+ fs := fmt.Sprintf("containers/%s", projectPrefix(project, name))
+ property := "used"
+
+ if s.pool.Config["volume.zfs.use_refquota"] != "" {
+ zfsUseRefquota = s.pool.Config["volume.zfs.use_refquota"]
+ }
+
+ if s.volume.Config["zfs.use_refquota"] != "" {
+ zfsUseRefquota = s.volume.Config["zfs.use_refquota"]
+ }
+
+ if shared.IsTrue(zfsUseRefquota) {
+ property = "referenced"
+ }
+
+ // Shortcut for refquota
+ if property == "referenced" && shared.IsMountPoint(path) {
+ var stat syscall.Statfs_t
+
+ err := syscall.Statfs(path, &stat)
+ if err != nil {
+ return -1, err
+ }
+
+ return int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize), nil
+ }
+
+ value, err := zfsFilesystemEntityPropertyGet(s.getOnDiskPoolName(), fs, property)
+ if err != nil {
+ return -1, err
+ }
+
+ valueInt, err := strconv.ParseInt(value, 10, 64)
+ if err != nil {
+ return -1, err
+ }
+
+ return valueInt, nil
+}
+
+func (s *Zfs) VolumeSetQuota(project, name string, size int64, userns bool, volumeType VolumeType) error {
+ var fs string
+
+ switch volumeType {
+ case VolumeTypeContainer:
+ fs = fmt.Sprintf("containers/%s", projectPrefix(project, name))
+ case VolumeTypeCustom:
+ fs = fmt.Sprintf("custom/%s", name)
+ }
+
+ property := "quota"
+
+ if s.pool.Config["volume.zfs.use_refquota"] != "" {
+ zfsUseRefquota = s.pool.Config["volume.zfs.use_refquota"]
+ }
+ if s.volume.Config["zfs.use_refquota"] != "" {
+ zfsUseRefquota = s.volume.Config["zfs.use_refquota"]
+ }
+
+ if shared.IsTrue(zfsUseRefquota) {
+ property = "refquota"
+ }
+
+ poolName := s.getOnDiskPoolName()
+ var err error
+ if size > 0 {
+ err = zfsPoolVolumeSet(poolName, fs, property, fmt.Sprintf("%d", size))
+ } else {
+ err = zfsPoolVolumeSet(poolName, fs, property, "none")
+ }
+
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error {
+ if !shared.StringInSlice("size", changedConfig) {
+ return nil
+ }
+
+ if s.volume.Type != storagePoolVolumeTypeNameCustom {
+ return updateStoragePoolVolumeError([]string{"size"}, "zfs")
+ }
+
+ if s.volume.Config["size"] != writable.Config["size"] {
+ size, err := shared.ParseByteSizeString(writable.Config["size"])
+ if err != nil {
+ return err
+ }
+
+ err = s.VolumeSetQuota("default", fmt.Sprintf("custom/%s", s.volume.Name), size, false, VolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumePrepareRestore(sourceName string, targetName string, targetSnapshots []string, f func() error) error {
+ zfsRemoveSnapshots := "false"
+
+ if targetSnapshots[len(targetSnapshots)-1] != sourceName {
+ if s.pool.Config["volume.zfs.remove_snapshots"] != "" {
+ zfsRemoveSnapshots = s.pool.Config["volume.zfs.remove_snapshots"]
+ }
+
+ if s.volume.Config["zfs.remove_snapshots"] != "" {
+ zfsRemoveSnapshots = s.volume.Config["zfs.remove_snapshots"]
+ }
+
+ if !shared.IsTrue(zfsRemoveSnapshots) {
+ return fmt.Errorf("ZFS can only restore from the latest snapshot. Delete newer snapshots or copy the snapshot into a new container instead")
+ }
+ }
+
+ return f()
+}
+
+// TODO: Get the snapshots using zfs tools.
+// Return list of removed snapshots since drivers cannot actually delete
+// containers. Storage should be able to remove the container even if the
+// zfs volumes/snapshots have been deleted beforehand.
+// TODO: (stgraber) Pass function instead of having separate PrepareVolumeRestore function.
+/// XXX: Deprecated. Use VolumeSnapshotRestore instead.
+func (s *Zfs) VolumeRestore(project string, sourceName string, targetName string, volumeType VolumeType) error {
+ return s.VolumeSnapshotRestore(project, sourceName, targetName, volumeType)
+}
+
+func (s *Zfs) VolumeSnapshotCreate(project string, source string, target string, volumeType VolumeType) error {
+ var dataset string
+ var mountpoint string
+
+ isSnapshot := shared.IsSnapshot(target)
+
+ if !isSnapshot {
+ return fmt.Errorf("Target volume is not a snapshot")
+ }
+
+ volumeName, snapshotOnlyName, _ := containerGetParentAndSnapshotName(target)
+ snapName := fmt.Sprintf("snapshot-%s", snapshotOnlyName)
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ dataset = fmt.Sprintf("containers/%s", projectPrefix(project, volumeName))
+ case VolumeTypeCustomSnapshot:
+ dataset = fmt.Sprintf("custom/%s", volumeName)
+ mountpoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ if source == "" {
+ // Nothing to do
+ return nil
+ }
+
+ err := ZfsPoolVolumeSnapshotCreate(s.getOnDiskPoolName(), dataset, snapName)
+ if err != nil {
+ return err
+ }
+
+ if mountpoint != "" && !shared.PathExists(mountpoint) {
+ err := os.MkdirAll(mountpoint, 0700)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumeSnapshotCopy(project, source string, target string, volumeType VolumeType) error {
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ return s.doContainerCopy(project, source, target)
+ case VolumeTypeCustomSnapshot:
+ return s.doVolumeCopy(source, target)
+ }
+
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+}
+
+func (s *Zfs) VolumeSnapshotDelete(project string, volumeName string, recursive bool, volumeType VolumeType) error {
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ case VolumeTypeCustomSnapshot:
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ return ZfsSnapshotDeleteInternal(project, s.pool.Name, volumeName, s.getOnDiskPoolName())
+}
+
+func (s *Zfs) VolumeSnapshotRestore(project string, sourceName string, targetName string, volumeType VolumeType) error {
+ var path string
+
+ parentName, snapOnlyName, _ := containerGetParentAndSnapshotName(sourceName)
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ path = fmt.Sprintf("containers/%s", projectPrefix(project, parentName))
+ case VolumeTypeCustomSnapshot:
+ path = fmt.Sprintf("custom/%s", parentName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ // Restore the snapshot
+ snapName := fmt.Sprintf("snapshot-%s", snapOnlyName)
+
+ return zfsPoolVolumeSnapshotRestore(s.getOnDiskPoolName(), path, snapName)
+}
+
+func (s *Zfs) VolumeSnapshotRename(project string, oldName string, newName string, volumeType VolumeType) error {
+ var oldSnapshotMntPoint string
+ var newSnapshotMntPoint string
+
+ switch volumeType {
+ case VolumeTypeContainerSnapshot:
+ oldSnapshotMntPoint = getSnapshotMountPoint(project, s.pool.Name, oldName)
+ newSnapshotMntPoint = getSnapshotMountPoint(project, s.pool.Name, newName)
+ case VolumeTypeCustomSnapshot:
+ oldSnapshotMntPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, oldName)
+ newSnapshotMntPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, newName)
+ default:
+ return fmt.Errorf("Unsupported volume type: %v", volumeType)
+ }
+
+ parentName, oldSnapOnlyName, _ := containerGetParentAndSnapshotName(oldName)
+ newSnapOnlyName := shared.ExtractSnapshotName(newName)
+
+ oldZfsDatasetName := fmt.Sprintf("snapshot-%s", oldSnapOnlyName)
+ newZfsDatasetName := fmt.Sprintf("snapshot-%s", newSnapOnlyName)
+
+ if oldZfsDatasetName == newZfsDatasetName {
+ // Nothing to do
+ return nil
+ }
+
+ err := zfsPoolVolumeSnapshotRename(
+ s.getOnDiskPoolName(), fmt.Sprintf("containers/%s", projectPrefix(project, parentName)), oldZfsDatasetName, newZfsDatasetName)
+ if err != nil {
+ return err
+ }
+
+ revert := true
+
+ defer func() {
+ if !revert {
+ return
+ }
+
+ zfsPoolVolumeSnapshotRename(s.getOnDiskPoolName(), fmt.Sprintf("containers/%s", projectPrefix(project, parentName)), newZfsDatasetName, oldZfsDatasetName)
+ }()
+
+ err = os.Rename(oldSnapshotMntPoint, newSnapshotMntPoint)
+ if err != nil {
+ return err
+ }
+
+ if volumeType == VolumeTypeCustomSnapshot {
+ revert = false
+ return nil
+ }
+
+ snapshotMntPointSymlinkTarget := getSnapshotMountPoint(project, s.pool.Name, parentName)
+ snapshotMntPointSymlink := containerPath(project, parentName, true)
+
+ if !shared.PathExists(snapshotMntPointSymlink) {
+ err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+ }
+
+ revert = false
+ return nil
+}
+
+func (s *Zfs) doVolumeBackupCreate(path string, project string, source string, snapshots []string) error {
+ var snapshotsPath string
+
+ // Handle snapshots
+ if len(snapshots) > 0 {
+ snapshotsPath = fmt.Sprintf("%s/snapshots", path)
+
+ // Create the snapshot path
+ err := os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return errors.Wrap(err, "Create snapshot path")
+ }
+ }
+
+ for _, snap := range snapshots {
+ // Mount the snapshot to a usable path
+ fullSnapshotName := fmt.Sprintf("%s/%s", source, snap)
+ _, err := s.VolumeMount(project, fullSnapshotName, VolumeTypeContainerSnapshot)
+ if err != nil {
+ return errors.Wrap(err, "Mount snapshot")
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, fullSnapshotName)
+ target := fmt.Sprintf("%s/%s", snapshotsPath, snap)
+
+ // Copy the snapshot
+ err = s.rsync(snapshotMntPoint, target)
+ s.VolumeUmount(project, fullSnapshotName, VolumeTypeContainerSnapshot)
+ if err != nil {
+ return errors.Wrap(err, "Copy snapshot")
+ }
+ }
+
+ // Make a temporary copy of the container
+ containersPath := getContainerMountPoint("default", s.pool.Name, "")
+
+ tmpContainerMntPoint, err := ioutil.TempDir(containersPath, source)
+ if err != nil {
+ return errors.Wrap(err, "Create temporary copy dir")
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0700)
+ if err != nil {
+ return errors.Wrap(err, "Change temporary mount point permissions")
+ }
+
+ snapshotSuffix := uuid.NewRandom().String()
+ fs := fmt.Sprintf("containers/%s", projectPrefix(project, source))
+ sourceZfsDatasetSnapshot := fmt.Sprintf("snapshot-%s", snapshotSuffix)
+ poolName := s.getOnDiskPoolName()
+
+ err = ZfsPoolVolumeSnapshotCreate(poolName, fs, sourceZfsDatasetSnapshot)
+ if err != nil {
+ return err
+ }
+ defer ZfsPoolVolumeSnapshotDestroy(poolName, fs, sourceZfsDatasetSnapshot)
+
+ targetZfsDataset := fmt.Sprintf("containers/%s", snapshotSuffix)
+
+ err = zfsPoolVolumeClone(project, poolName, fs, sourceZfsDatasetSnapshot, targetZfsDataset, tmpContainerMntPoint)
+ if err != nil {
+ return errors.Wrap(err, "Clone volume")
+ }
+ defer zfsPoolVolumeDestroy(poolName, targetZfsDataset)
+
+ // Mount the temporary copy
+ if !shared.IsMountPoint(tmpContainerMntPoint) {
+ err = ZfsMount(poolName, targetZfsDataset)
+ if err != nil {
+ return errors.Wrap(err, "Mount temporary copy")
+ }
+ defer ZfsUmount(poolName, targetZfsDataset, tmpContainerMntPoint)
+ }
+
+ // Copy the container
+ containerPath := fmt.Sprintf("%s/container", path)
+
+ err = s.rsync(tmpContainerMntPoint, containerPath)
+ if err != nil {
+ return errors.Wrap(err, "Copy container")
+ }
+
+ return nil
+}
+
+func (s *Zfs) doVolumeBackupCreateOptimized(path string, project string, source string) error {
+ poolName := s.getOnDiskPoolName()
+
+ snapshotSuffix := uuid.NewRandom().String()
+ sourceDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, source), snapshotSuffix)
+ fs := fmt.Sprintf("containers/%s", projectPrefix(project, source))
+
+ err := ZfsPoolVolumeSnapshotCreate(poolName, fs, snapshotSuffix)
+ if err != nil {
+ return err
+ }
+ defer ZfsPoolVolumeSnapshotDestroy(poolName, fs, snapshotSuffix)
+
+ // Dump the container to a file
+ backupFile := fmt.Sprintf("%s/%s", path, "container.bin")
+
+ f, err := os.OpenFile(backupFile, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ zfsSendCmd := exec.Command("zfs", "send", sourceDataset)
+ zfsSendCmd.Stdout = f
+
+ return zfsSendCmd.Run()
+}
+
+func (s *Zfs) doSnapshotBackup(path string, project string, source string, parentSnapshot string) error {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", path)
+
+ // Create backup path for snapshots
+ err := os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return err
+ }
+
+ currentSnapshotDataset := s.getFullDataset(project, source, "containers")
+ args := []string{"send", currentSnapshotDataset}
+ if parentSnapshot != "" {
+ parentSnapshotDataset := s.getFullDataset(project, parentSnapshot, "containers")
+ args = append(args, "-i", parentSnapshotDataset)
+ }
+
+ backupFile := fmt.Sprintf("%s/%s.bin", snapshotsPath, shared.ExtractSnapshotName(source))
+ f, err := os.OpenFile(backupFile, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ zfsSendCmd := exec.Command("zfs", args...)
+ zfsSendCmd.Stdout = f
+ return zfsSendCmd.Run()
+}
+
+func (s *Zfs) doVolumeBackupCreateOptimizedWithSnapshots(path string, project string, source string, snapshots []string) error {
+ prev := ""
+ prevSnapOnlyName := ""
+
+ for i, snap := range snapshots {
+ if i > 0 {
+ prev = fmt.Sprintf("%s/%s", source, snapshots[i-1])
+ }
+
+ prevSnapOnlyName = snap
+
+ err := s.doSnapshotBackup(path, project, fmt.Sprintf("%s/%s", source, snap), prev)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Dump the container to a file
+ poolName := s.getOnDiskPoolName()
+ tmpSnapshotName := fmt.Sprintf("backup-%s", uuid.NewRandom().String())
+
+ err := ZfsPoolVolumeSnapshotCreate(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, source)), tmpSnapshotName)
+ if err != nil {
+ return err
+ }
+
+ currentSnapshotDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(project, source), tmpSnapshotName)
+ parentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(project, source), prevSnapOnlyName)
+ args := []string{"send", currentSnapshotDataset, "-i", parentSnapshotDataset}
+
+ backupFile := fmt.Sprintf("%s/container.bin", path)
+
+ f, err := os.OpenFile(backupFile, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ zfsSendCmd := exec.Command("zfs", args...)
+ zfsSendCmd.Stdout = f
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ return ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, source)), tmpSnapshotName)
+}
+
+func (s *Zfs) VolumeBackupCreate(path string, project string, source string, snapshots []string, optimized bool) error {
+ if optimized {
+ if len(snapshots) == 0 {
+ return s.doVolumeBackupCreateOptimized(path, project, source)
+ }
+
+ return s.doVolumeBackupCreateOptimizedWithSnapshots(path, project, source, snapshots)
+ }
+
+ return s.doVolumeBackupCreate(path, project, source, snapshots)
+}
+
+func (s *Zfs) doVolumeBackupLoadOptimized(backupDir string, project string, containerName string, snapshots []string) error {
+ poolName := s.getOnDiskPoolName()
+ unpackPath := filepath.Join(backupDir, ".backup_unpack")
+
+ for _, snapshotOnlyName := range snapshots {
+ snapshotBackup := fmt.Sprintf("%s/snapshots/%s.bin", unpackPath, snapshotOnlyName)
+
+ feeder, err := os.Open(snapshotBackup)
+ if err != nil {
+ // can't use defer because it needs to run before the mount
+ os.RemoveAll(unpackPath)
+ return err
+ }
+
+ fullSnapshotName := fmt.Sprintf("%s/%s", containerName, snapshotOnlyName)
+ snapshotDataset := s.getFullDataset(project, fullSnapshotName, "containers")
+
+ zfsRecvCmd := exec.Command("zfs", "receive", "-F", snapshotDataset)
+ zfsRecvCmd.Stdin = feeder
+
+ err = zfsRecvCmd.Run()
+ feeder.Close()
+ if err != nil {
+ // can't use defer because it needs to run before the mount
+ os.RemoveAll(unpackPath)
+ return err
+ }
+
+ // create mountpoint
+ snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, fullSnapshotName)
+ snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(project, containerName))
+
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPoint, snapshotMntPointSymlink)
+ if err != nil {
+ // can't use defer because it needs to run before the mount
+ os.RemoveAll(unpackPath)
+ return err
+ }
+ }
+
+ containerBackup := fmt.Sprintf("%s/container.bin", unpackPath)
+
+ feeder, err := os.Open(containerBackup)
+ if err != nil {
+ // can't use defer because it needs to run before the mount
+ os.RemoveAll(unpackPath)
+ return err
+ }
+ defer feeder.Close()
+
+ containerSnapshotDataset := fmt.Sprintf("%s/containers/%s at backup", poolName, projectPrefix(project, containerName))
+ zfsRecvCmd := exec.Command("zfs", "receive", "-F", containerSnapshotDataset)
+ zfsRecvCmd.Stdin = feeder
+
+ err = zfsRecvCmd.Run()
+ os.RemoveAll(backupDir)
+ ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(project, containerName)), "backup")
+ if err != nil {
+ return err
+ }
+
+ fs := fmt.Sprintf("containers/%s", projectPrefix(project, containerName))
+ err = zfsPoolVolumeSet(poolName, fs, "canmount", "noauto")
+ if err != nil {
+ return err
+ }
+
+ containerMntPoint := getContainerMountPoint(project, s.pool.Name, containerName)
+
+ err = zfsPoolVolumeSet(poolName, fs, "mountpoint", containerMntPoint)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) VolumeBackupLoad(backupDir string, project string, containerName string, snapshots []string, privileged bool, optimized bool) error {
+ if optimized {
+ return s.doVolumeBackupLoadOptimized(backupDir, project, containerName, snapshots)
+ }
+
+ // Non-optimized container backup is handled in the storage code.
+ return nil
+}
+
+func (s *Zfs) VolumeReady(project string, name string) bool {
+ volumeName := projectPrefix(project, name)
+ fs := fmt.Sprintf("containers/%s", volumeName)
+ return ZfsFilesystemEntityExists(s.getOnDiskPoolName(), fs)
+}
+
+func (s *Zfs) getOnDiskPoolName() string {
+ if s.dataset != "" {
+ return s.dataset
+ }
+
+ return s.pool.Name
+}
+
+// zfsPoolVolumeCreate creates a ZFS dataset with a set of given properties.
+func zfsPoolVolumeCreate(dataset string, properties ...string) (string, error) {
+ cmd := []string{"zfs", "create"}
+
+ for _, prop := range properties {
+ cmd = append(cmd, []string{"-o", prop}...)
+ }
+
+ cmd = append(cmd, []string{"-p", dataset}...)
+
+ return shared.RunCommand(cmd[0], cmd[1:]...)
+}
+
+func zfsPoolVolumeRename(pool string, source string, dest string, ignoreMounts bool) error {
+ var err error
+ var output string
+
+ for i := 0; i < 20; i++ {
+ if ignoreMounts {
+ output, err = shared.RunCommand(
+ "/proc/self/exe",
+ "forkzfs",
+ "--",
+ "rename",
+ "-p",
+ fmt.Sprintf("%s/%s", pool, source),
+ fmt.Sprintf("%s/%s", pool, dest))
+ } else {
+ output, err = shared.RunCommand(
+ "zfs",
+ "rename",
+ "-p",
+ fmt.Sprintf("%s/%s", pool, source),
+ fmt.Sprintf("%s/%s", pool, dest))
+ }
+
+ // Success
+ if err == nil {
+ return nil
+ }
+
+ // zfs rename can fail because of descendants, yet still manage the rename
+ if !ZfsFilesystemEntityExists(pool, source) && ZfsFilesystemEntityExists(pool, dest) {
+ return nil
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ // Timeout
+ logger.Errorf("zfs rename failed: %s", output)
+ return fmt.Errorf("Failed to rename ZFS filesystem: %s", output)
+}
+
+func zfsPoolVolumeSet(pool string, path string, key string, value string) error {
+ vdev := pool
+ if path != "" {
+ vdev = fmt.Sprintf("%s/%s", pool, path)
+ }
+ output, err := shared.RunCommand(
+ "zfs",
+ "set",
+ fmt.Sprintf("%s=%s", key, value),
+ vdev)
+ if err != nil {
+ logger.Errorf("zfs set failed: %s", output)
+ return fmt.Errorf("Failed to set ZFS config: %s", output)
+ }
+
+ return nil
+}
+
+func ZfsFilesystemEntityExists(pool string, path string) bool {
+ vdev := pool
+ if path != "" {
+ vdev = fmt.Sprintf("%s/%s", pool, path)
+ }
+ output, err := shared.RunCommand(
+ "zfs",
+ "get",
+ "-H",
+ "-o",
+ "name",
+ "type",
+ vdev)
+ if err != nil {
+ return false
+ }
+
+ detectedName := strings.TrimSpace(output)
+ return detectedName == vdev
+}
+
+func (s *Zfs) zfsPoolCreate() error {
+ s.pool.Config["volatile.initial_source"] = s.pool.Config["source"]
+
+ zpoolName := s.getOnDiskPoolName()
+ vdev := s.pool.Config["source"]
+ logger.Debugf("vdev=%s", vdev)
+ defaultVdev := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", s.pool.Name))
+
+ if vdev == "" || vdev == defaultVdev {
+ vdev = defaultVdev
+ s.pool.Config["source"] = vdev
+
+ if s.pool.Config["zfs.pool_name"] == "" {
+ s.pool.Config["zfs.pool_name"] = zpoolName
+ }
+
+ f, err := os.Create(vdev)
+ if err != nil {
+ return fmt.Errorf("Failed to open %s: %s", vdev, err)
+ }
+ defer f.Close()
+
+ err = f.Chmod(0600)
+ if err != nil {
+ return fmt.Errorf("Failed to chmod %s: %s", vdev, err)
+ }
+
+ size, err := shared.ParseByteSizeString(s.pool.Config["size"])
+ if err != nil {
+ return err
+ }
+ err = f.Truncate(size)
+ if err != nil {
+ return fmt.Errorf("Failed to create sparse file %s: %s", vdev, err)
+ }
+
+ err = zfsPoolCreate(zpoolName, vdev)
+ if err != nil {
+ return err
+ }
+ } else {
+ // Unset size property since it doesn't make sense.
+ s.pool.Config["size"] = ""
+
+ if filepath.IsAbs(vdev) {
+ if !shared.IsBlockdevPath(vdev) {
+ return fmt.Errorf("Custom loop file locations are not supported")
+ }
+
+ if s.pool.Config["zfs.pool_name"] == "" {
+ s.pool.Config["zfs.pool_name"] = zpoolName
+ }
+
+ // This is a block device. Note, that we do not store the
+ // block device path or UUID or PARTUUID or similar in
+ // the database. All of those might change or might be
+ // used in a special way (For example, zfs uses a single
+ // UUID in a multi-device pool for all devices.). The
+ // safest way is to just store the name of the zfs pool
+ // we create.
+ s.pool.Config["source"] = zpoolName
+ err := zfsPoolCreate(zpoolName, vdev)
+ if err != nil {
+ return err
+ }
+ } else {
+ if s.pool.Config["zfs.pool_name"] != "" && s.pool.Config["zfs.pool_name"] != vdev {
+ return fmt.Errorf("Invalid combination of \"source\" and \"zfs.pool_name\" property")
+ }
+
+ s.pool.Config["zfs.pool_name"] = vdev
+ s.dataset = vdev
+
+ if strings.Contains(vdev, "/") {
+ if !ZfsFilesystemEntityExists(vdev, "") {
+ err := zfsPoolCreate("", vdev)
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ err := zfsPoolCheck(vdev)
+ if err != nil {
+ return err
+ }
+ }
+
+ subvols, err := zfsPoolListSubvolumes(zpoolName, vdev)
+ if err != nil {
+ return err
+ }
+
+ if len(subvols) > 0 {
+ return fmt.Errorf("Provided ZFS pool (or dataset) isn't empty")
+ }
+
+ err = zfsPoolApplyDefaults(vdev)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Create default dummy datasets to avoid zfs races during container
+ // creation.
+ poolName := s.getOnDiskPoolName()
+ dataset := fmt.Sprintf("%s/containers", poolName)
+ msg, err := zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ logger.Errorf("Failed to create containers dataset: %s", msg)
+ return err
+ }
+
+ fixperms := shared.VarPath("storage-pools", s.pool.Name, "containers")
+ err = os.MkdirAll(fixperms, containersDirMode)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+
+ err = os.Chmod(fixperms, containersDirMode)
+ if err != nil {
+ logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(containersDirMode), 8), err)
+ }
+
+ dataset = fmt.Sprintf("%s/images", poolName)
+ msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ logger.Errorf("Failed to create images dataset: %s", msg)
+ return err
+ }
+
+ fixperms = shared.VarPath("storage-pools", s.pool.Name, "images")
+ err = os.MkdirAll(fixperms, imagesDirMode)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ err = os.Chmod(fixperms, imagesDirMode)
+ if err != nil {
+ logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(imagesDirMode), 8), err)
+ }
+
+ dataset = fmt.Sprintf("%s/custom", poolName)
+ msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ logger.Errorf("Failed to create custom dataset: %s", msg)
+ return err
+ }
+
+ fixperms = shared.VarPath("storage-pools", s.pool.Name, "custom")
+ err = os.MkdirAll(fixperms, customDirMode)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ err = os.Chmod(fixperms, customDirMode)
+ if err != nil {
+ logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(customDirMode), 8), err)
+ }
+
+ dataset = fmt.Sprintf("%s/deleted", poolName)
+ msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ logger.Errorf("Failed to create deleted dataset: %s", msg)
+ return err
+ }
+
+ dataset = fmt.Sprintf("%s/snapshots", poolName)
+ msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ logger.Errorf("Failed to create snapshots dataset: %s", msg)
+ return err
+ }
+
+ fixperms = shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots")
+ err = os.MkdirAll(fixperms, snapshotsDirMode)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ err = os.Chmod(fixperms, snapshotsDirMode)
+ if err != nil {
+ logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(snapshotsDirMode), 8), err)
+ }
+
+ dataset = fmt.Sprintf("%s/custom-snapshots", poolName)
+ msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
+ if err != nil {
+ logger.Errorf("Failed to create snapshots dataset: %s", msg)
+ return err
+ }
+
+ fixperms = shared.VarPath("storage-pools", s.pool.Name, "custom-snapshots")
+ err = os.MkdirAll(fixperms, snapshotsDirMode)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ err = os.Chmod(fixperms, snapshotsDirMode)
+ if err != nil {
+ logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(snapshotsDirMode), 8), err)
+ }
+
+ return nil
+}
+
+func zfsPoolCreate(pool string, vdev string) error {
+ var output string
+ var err error
+
+ dataset := ""
+
+ if pool == "" {
+ output, err := shared.RunCommand(
+ "zfs", "create", "-p", "-o", "mountpoint=none", vdev)
+ if err != nil {
+ logger.Errorf("zfs create failed: %s", output)
+ return fmt.Errorf("Failed to create ZFS filesystem: %s", output)
+ }
+ dataset = vdev
+ } else {
+ output, err = shared.RunCommand(
+ "zpool", "create", "-f", "-m", "none", "-O", "compression=on", pool, vdev)
+ if err != nil {
+ logger.Errorf("zfs create failed: %s", output)
+ return fmt.Errorf("Failed to create the ZFS pool: %s", output)
+ }
+
+ dataset = pool
+ }
+
+ err = zfsPoolApplyDefaults(dataset)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Zfs) doCopyVolumeWithoutSnapshotsFull(source string) error {
+ sourceIsSnapshot := strings.Contains(source, "/")
+
+ var snapshotSuffix string
+ var sourceDataset string
+ var targetDataset string
+ var targetSnapshotDataset string
+
+ poolName := s.getOnDiskPoolName()
+
+ if sourceIsSnapshot {
+ sourceVolumeName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(source)
+ snapshotSuffix = fmt.Sprintf("snapshot-%s", sourceSnapOnlyName)
+ sourceDataset = fmt.Sprintf("%s/custom/%s@%s", poolName, sourceVolumeName, snapshotSuffix)
+ targetSnapshotDataset = fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, s.volume.Name, sourceSnapOnlyName)
+ } else {
+ snapshotSuffix = uuid.NewRandom().String()
+ sourceDataset = fmt.Sprintf("%s/custom/%s@%s", poolName, source, snapshotSuffix)
+ targetSnapshotDataset = fmt.Sprintf("%s/custom/%s@%s", poolName, s.volume.Name, snapshotSuffix)
+
+ fs := fmt.Sprintf("custom/%s", source)
+ err := ZfsPoolVolumeSnapshotCreate(poolName, fs, snapshotSuffix)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ err := ZfsPoolVolumeSnapshotDestroy(poolName, fs, snapshotSuffix)
+ if err != nil {
+ logger.Warnf("Failed to delete temporary ZFS snapshot \"%s\", manual cleanup needed", sourceDataset)
+ }
+ }()
+ }
+
+ zfsSendCmd := exec.Command("zfs", "send", sourceDataset)
+
+ zfsRecvCmd := exec.Command("zfs", "receive", targetDataset)
+
+ zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
+ zfsRecvCmd.Stdout = os.Stdout
+ zfsRecvCmd.Stderr = os.Stderr
+
+ err := zfsRecvCmd.Start()
+ if err != nil {
+ return err
+ }
+
+ err = zfsSendCmd.Run()
+ if err != nil {
+ return err
+ }
+
+ err = zfsRecvCmd.Wait()
+ if err != nil {
+ return err
+ }
+
+ msg, err := shared.RunCommand("zfs", "rollback", "-r", "-R", targetSnapshotDataset)
+ if err != nil {
+ logger.Errorf("Failed to rollback ZFS dataset: %s", msg)
+ return err
+ }
+
+ targetContainerMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
+ targetfs := fmt.Sprintf("custom/%s", s.volume.Name)
+
+ err = zfsPoolVolumeSet(poolName, targetfs, "canmount", "noauto")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(poolName, targetfs, "mountpoint", targetContainerMountPoint)
+ if err != nil {
+ return err
+ }
+
+ err = ZfsPoolVolumeSnapshotDestroy(poolName, targetfs, snapshotSuffix)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// TODO:
+func (s *Zfs) doCopyVolumeWithoutSnapshotsSparse(source string) error {
+ poolName := s.getOnDiskPoolName()
+
+ sourceVolumeName := source
+ sourceVolumePath := getStoragePoolVolumeMountPoint(s.pool.Name, source)
+
+ targetVolumeName := s.volume.Name
+ targetVolumePath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
+
+ sourceZfsDataset := ""
+ sourceZfsDatasetSnapshot := ""
+ sourceName, sourceSnapOnlyName, isSnapshotName := containerGetParentAndSnapshotName(sourceVolumeName)
+
+ targetZfsDataset := fmt.Sprintf("custom/%s", targetVolumeName)
+
+ if isSnapshotName {
+ sourceZfsDatasetSnapshot = sourceSnapOnlyName
+ }
+
+ revert := true
+ if sourceZfsDatasetSnapshot == "" {
+ if ZfsFilesystemEntityExists(poolName, fmt.Sprintf("custom/%s", sourceName)) {
+ sourceZfsDatasetSnapshot = fmt.Sprintf("copy-%s", uuid.NewRandom().String())
+ sourceZfsDataset = fmt.Sprintf("custom/%s", sourceName)
+
+ err := ZfsPoolVolumeSnapshotCreate(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
+ if err != nil {
+ return err
+ }
+
+ defer func() {
+ if !revert {
+ return
+ }
+ ZfsPoolVolumeSnapshotDestroy(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
+ }()
+ }
+ } else {
+ if ZfsFilesystemEntityExists(poolName, fmt.Sprintf("custom/%s at snapshot-%s", sourceName, sourceZfsDatasetSnapshot)) {
+ sourceZfsDataset = fmt.Sprintf("custom/%s", sourceName)
+ sourceZfsDatasetSnapshot = fmt.Sprintf("snapshot-%s", sourceZfsDatasetSnapshot)
+ }
+ }
+
+ if sourceZfsDataset != "" {
+ err := zfsPoolVolumeClone("default", poolName, sourceZfsDataset, sourceZfsDatasetSnapshot, targetZfsDataset, targetVolumePath)
+ if err != nil {
+ return err
+ }
+
+ defer func() {
+ if !revert {
+ return
+ }
+ zfsPoolVolumeDestroy(poolName, targetZfsDataset)
+ }()
+ } else {
+ err := s.rsync(sourceVolumePath, targetVolumePath)
+ if err != nil {
+ return err
+ }
+ }
+
+ revert = false
+
+ return nil
+}
+
+func zfsPoolCheck(pool string) error {
+ output, err := shared.RunCommand(
+ "zfs", "get", "-H", "-o", "value", "type", pool)
+ if err != nil {
+ return fmt.Errorf(strings.Split(output, "\n")[0])
+ }
+
+ poolType := strings.Split(output, "\n")[0]
+ if poolType != "filesystem" {
+ return fmt.Errorf("Unsupported pool type: %s", poolType)
+ }
+
+ return nil
+}
+
+func zfsPoolListSubvolumes(pool string, path string) ([]string, error) {
+ output, err := shared.RunCommand(
+ "zfs",
+ "list",
+ "-t", "filesystem",
+ "-o", "name",
+ "-H",
+ "-r", path)
+ if err != nil {
+ logger.Errorf("zfs list failed: %s", output)
+ return []string{}, fmt.Errorf("Failed to list ZFS filesystems: %s", output)
+ }
+
+ children := []string{}
+ for _, entry := range strings.Split(output, "\n") {
+ if entry == "" {
+ continue
+ }
+
+ if entry == path {
+ continue
+ }
+
+ children = append(children, strings.TrimPrefix(entry, fmt.Sprintf("%s/", pool)))
+ }
+
+ return children, nil
+}
+
+func zfsPoolApplyDefaults(dataset string) error {
+ err := zfsPoolVolumeSet(dataset, "", "mountpoint", "none")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(dataset, "", "setuid", "on")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(dataset, "", "exec", "on")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(dataset, "", "devices", "on")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(dataset, "", "acltype", "posixacl")
+ if err != nil {
+ return err
+ }
+
+ err = zfsPoolVolumeSet(dataset, "", "xattr", "sa")
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func zfsFilesystemEntityDelete(vdev string, pool string) error {
+ var output string
+ var err error
+ if strings.Contains(pool, "/") {
+ // Command to destroy a zfs dataset.
+ output, err = shared.RunCommand("zfs", "destroy", "-r", pool)
+ } else {
+ // Command to destroy a zfs pool.
+ output, err = shared.RunCommand("zpool", "destroy", "-f", pool)
+ }
+ if err != nil {
+ return fmt.Errorf("Failed to delete the ZFS pool: %s", output)
+ }
+
+ // Cleanup storage
+ if filepath.IsAbs(vdev) && !shared.IsBlockdevPath(vdev) {
+ os.RemoveAll(vdev)
+ }
+
+ return nil
+}
+
+func ZfsPoolListSnapshots(pool string, path string) ([]string, error) {
+ path = strings.TrimRight(path, "/")
+ fullPath := pool
+ if path != "" {
+ fullPath = fmt.Sprintf("%s/%s", pool, path)
+ }
+
+ output, err := shared.RunCommand(
+ "zfs",
+ "list",
+ "-t", "snapshot",
+ "-o", "name",
+ "-H",
+ "-d", "1",
+ "-s", "creation",
+ "-r", fullPath)
+ if err != nil {
+ logger.Errorf("zfs list failed: %s", output)
+ return []string{}, fmt.Errorf("Failed to list ZFS snapshots: %s", output)
+ }
+
+ children := []string{}
+ for _, entry := range strings.Split(output, "\n") {
+ if entry == "" {
+ continue
+ }
+
+ if entry == fullPath {
+ continue
+ }
+
+ children = append(children, strings.SplitN(entry, "@", 2)[1])
+ }
+
+ return children, nil
+}
+
+func zfsPoolVolumeSnapshotRemovable(pool string, path string, name string) (bool, error) {
+ var snap string
+ if name == "" {
+ snap = path
+ } else {
+ snap = fmt.Sprintf("%s@%s", path, name)
+ }
+
+ clones, err := zfsFilesystemEntityPropertyGet(pool, snap, "clones")
+ if err != nil {
+ return false, err
+ }
+
+ if clones == "-" || clones == "" {
+ return true, nil
+ }
+
+ return false, nil
+}
+
+func zfsFilesystemEntityPropertyGet(pool string, path string, key string) (string, error) {
+ entity := pool
+ if path != "" {
+ entity = fmt.Sprintf("%s/%s", pool, path)
+ }
+ output, err := shared.RunCommand(
+ "zfs",
+ "get",
+ "-H",
+ "-p",
+ "-o", "value",
+ key,
+ entity)
+ if err != nil {
+ return "", fmt.Errorf("Failed to get ZFS config: %s", output)
+ }
+
+ return strings.TrimRight(output, "\n"), nil
+}
+
+func zfsPoolVolumeDestroy(pool string, path string) error {
+ mountpoint, err := zfsFilesystemEntityPropertyGet(pool, path, "mountpoint")
+ if err != nil {
+ return err
+ }
+
+ if mountpoint != "none" && shared.IsMountPoint(mountpoint) {
+ err := syscall.Unmount(mountpoint, syscall.MNT_DETACH)
+ if err != nil {
+ logger.Errorf("umount failed: %s", err)
+ return err
+ }
+ }
+
+ // Due to open fds or kernel refs, this may fail for a bit, give it 10s
+ output, err := shared.TryRunCommand(
+ "zfs",
+ "destroy",
+ "-r",
+ fmt.Sprintf("%s/%s", pool, path))
+
+ if err != nil {
+ logger.Errorf("zfs destroy failed: %s", output)
+ return fmt.Errorf("Failed to destroy ZFS filesystem: %s", output)
+ }
+
+ return nil
+}
+
+func zfsPoolVolumeCleanup(pool string, path string) error {
+ if strings.HasPrefix(path, "deleted/") {
+ // Cleanup of filesystems kept for refcount reason
+ removablePath, err := zfsPoolVolumeSnapshotRemovable(pool, path, "")
+ if err != nil {
+ return err
+ }
+
+ // Confirm that there are no more clones
+ if removablePath {
+ if strings.Contains(path, "@") {
+ // Cleanup snapshots
+ err = zfsPoolVolumeDestroy(pool, path)
+ if err != nil {
+ return err
+ }
+
+ // Check if the parent can now be deleted
+ subPath := strings.SplitN(path, "@", 2)[0]
+ snaps, err := ZfsPoolListSnapshots(pool, subPath)
+ if err != nil {
+ return err
+ }
+
+ if len(snaps) == 0 {
+ err := zfsPoolVolumeCleanup(pool, subPath)
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ // Cleanup filesystems
+ origin, err := zfsFilesystemEntityPropertyGet(pool, path, "origin")
+ if err != nil {
+ return err
+ }
+ origin = strings.TrimPrefix(origin, fmt.Sprintf("%s/", pool))
+
+ err = zfsPoolVolumeDestroy(pool, path)
+ if err != nil {
+ return err
+ }
+
+ // Attempt to remove its parent
+ if origin != "-" {
+ err := zfsPoolVolumeCleanup(pool, origin)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+ }
+ } else if strings.HasPrefix(path, "containers") && strings.Contains(path, "@copy-") {
+ // Just remove the copy- snapshot for copies of active containers
+ err := zfsPoolVolumeDestroy(pool, path)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func ZfsMount(poolName string, path string) error {
+ output, err := shared.TryRunCommand(
+ "zfs",
+ "mount",
+ fmt.Sprintf("%s/%s", poolName, path))
+ if err != nil {
+ return fmt.Errorf("Failed to mount ZFS filesystem: %s", output)
+ }
+
+ return nil
+}
+
+func ZfsUmount(poolName string, path string, mountpoint string) error {
+ output, err := shared.TryRunCommand(
+ "zfs",
+ "unmount",
+ fmt.Sprintf("%s/%s", poolName, path))
+ if err != nil {
+ logger.Warnf("Failed to unmount ZFS filesystem via zfs unmount: %s. Trying lazy umount (MNT_DETACH)...", output)
+ err := tryUnmount(mountpoint, syscall.MNT_DETACH)
+ if err != nil {
+ logger.Warnf("Failed to unmount ZFS filesystem via lazy umount (MNT_DETACH)...")
+ return err
+ }
+ }
+
+ return nil
+}
+
+func zfsPoolVolumeSnapshotRestore(pool string, path string, name string) error {
+ output, err := shared.TryRunCommand(
+ "zfs",
+ "rollback",
+ fmt.Sprintf("%s/%s@%s", pool, path, name))
+ if err != nil {
+ logger.Errorf("zfs rollback failed: %s", output)
+ return fmt.Errorf("Failed to restore ZFS snapshot: %s", output)
+ }
+
+ subvols, err := zfsPoolListSubvolumes(pool, fmt.Sprintf("%s/%s", pool, path))
+ if err != nil {
+ return err
+ }
+
+ for _, sub := range subvols {
+ snaps, err := ZfsPoolListSnapshots(pool, sub)
+ if err != nil {
+ return err
+ }
+
+ if !shared.StringInSlice(name, snaps) {
+ continue
+ }
+
+ output, err := shared.TryRunCommand(
+ "zfs",
+ "rollback",
+ fmt.Sprintf("%s/%s@%s", pool, sub, name))
+ if err != nil {
+ logger.Errorf("zfs rollback failed: %s", output)
+ return fmt.Errorf("Failed to restore ZFS sub-volume snapshot: %s", output)
+ }
+ }
+
+ return nil
+}
+
+func ZfsPoolVolumeSnapshotCreate(pool string, path string, name string) error {
+ output, err := shared.RunCommand(
+ "zfs",
+ "snapshot",
+ "-r",
+ fmt.Sprintf("%s/%s@%s", pool, path, name))
+ if err != nil {
+ logger.Errorf("zfs snapshot failed: %s", output)
+ return fmt.Errorf("Failed to create ZFS snapshot: %s", output)
+ }
+
+ return nil
+}
+
+func ZfsSnapshotDeleteInternal(project, poolName string, ctName string, onDiskPoolName string) error {
+ sourceContainerName, sourceContainerSnapOnlyName, _ := containerGetParentAndSnapshotName(ctName)
+ snapName := fmt.Sprintf("snapshot-%s", sourceContainerSnapOnlyName)
+
+ if ZfsFilesystemEntityExists(onDiskPoolName,
+ fmt.Sprintf("containers/%s@%s",
+ projectPrefix(project, sourceContainerName), snapName)) {
+ removable, err := zfsPoolVolumeSnapshotRemovable(onDiskPoolName,
+ fmt.Sprintf("containers/%s",
+ projectPrefix(project, sourceContainerName)),
+ snapName)
+ if err != nil {
+ return err
+ }
+
+ if removable {
+ err = ZfsPoolVolumeSnapshotDestroy(onDiskPoolName,
+ fmt.Sprintf("containers/%s",
+ projectPrefix(project, sourceContainerName)),
+ snapName)
+ } else {
+ err = zfsPoolVolumeSnapshotRename(onDiskPoolName,
+ fmt.Sprintf("containers/%s",
+ projectPrefix(project, sourceContainerName)),
+ snapName,
+ fmt.Sprintf("copy-%s", uuid.NewRandom().String()))
+ }
+ if err != nil {
+ return err
+ }
+ }
+
+ // Delete the snapshot on its storage pool:
+ // ${POOL}/snapshots/<snapshot_name>
+ snapshotContainerMntPoint := getSnapshotMountPoint(project, poolName, ctName)
+ if shared.PathExists(snapshotContainerMntPoint) {
+ err := os.RemoveAll(snapshotContainerMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Check if we can remove the snapshot symlink:
+ // ${LXD_DIR}/snapshots/<container_name> to ${POOL}/snapshots/<container_name>
+ // by checking if the directory is empty.
+ snapshotContainerPath := getSnapshotMountPoint(project, poolName, sourceContainerName)
+ empty, _ := shared.PathIsEmpty(snapshotContainerPath)
+ if empty == true {
+ // Remove the snapshot directory for the container:
+ // ${POOL}/snapshots/<source_container_name>
+ err := os.Remove(snapshotContainerPath)
+ if err != nil {
+ return err
+ }
+
+ snapshotSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceContainerName))
+ if shared.PathExists(snapshotSymlink) {
+ err := os.Remove(snapshotSymlink)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Legacy
+ snapPath := shared.VarPath(fmt.Sprintf("snapshots/%s/%s.zfs", projectPrefix(project, sourceContainerName), sourceContainerSnapOnlyName))
+ if shared.PathExists(snapPath) {
+ err := os.Remove(snapPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Legacy
+ parent := shared.VarPath(fmt.Sprintf("snapshots/%s", projectPrefix(project, sourceContainerName)))
+ if ok, _ := shared.PathIsEmpty(parent); ok {
+ err := os.Remove(parent)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func ZfsPoolVolumeSnapshotDestroy(pool, path string, name string) error {
+ output, err := shared.RunCommand(
+ "zfs",
+ "destroy",
+ "-r",
+ fmt.Sprintf("%s/%s@%s", pool, path, name))
+ if err != nil {
+ logger.Errorf("zfs destroy failed: %s", output)
+ return fmt.Errorf("Failed to destroy ZFS snapshot: %s", output)
+ }
+
+ return nil
+}
+
+func zfsPoolVolumeSnapshotRename(pool string, path string, oldName string, newName string) error {
+ output, err := shared.RunCommand(
+ "zfs",
+ "rename",
+ "-r",
+ fmt.Sprintf("%s/%s@%s", pool, path, oldName),
+ fmt.Sprintf("%s/%s@%s", pool, path, newName))
+ if err != nil {
+ logger.Errorf("zfs snapshot rename failed: %s", output)
+ return fmt.Errorf("Failed to rename ZFS snapshot: %s", output)
+ }
+
+ return nil
+}
+
+func zfsPoolVolumeClone(project, pool string, source string, name string, dest string, mountpoint string) error {
+ output, err := shared.RunCommand(
+ "zfs",
+ "clone",
+ "-p",
+ "-o", fmt.Sprintf("mountpoint=%s", mountpoint),
+ "-o", "canmount=noauto",
+ fmt.Sprintf("%s/%s@%s", pool, source, name),
+ fmt.Sprintf("%s/%s", pool, dest))
+ if err != nil {
+ logger.Errorf("zfs clone failed: %s", output)
+ return fmt.Errorf("Failed to clone the filesystem: %s", output)
+ }
+
+ subvols, err := zfsPoolListSubvolumes(pool, fmt.Sprintf("%s/%s", pool, source))
+ if err != nil {
+ return err
+ }
+
+ for _, sub := range subvols {
+ snaps, err := ZfsPoolListSnapshots(pool, sub)
+ if err != nil {
+ return err
+ }
+
+ if !shared.StringInSlice(name, snaps) {
+ continue
+ }
+
+ destSubvol := dest + strings.TrimPrefix(sub, source)
+ snapshotMntPoint := getSnapshotMountPoint(project, pool, destSubvol)
+
+ output, err := shared.RunCommand(
+ "zfs",
+ "clone",
+ "-p",
+ "-o", fmt.Sprintf("mountpoint=%s", snapshotMntPoint),
+ "-o", "canmount=noauto",
+ fmt.Sprintf("%s/%s@%s", pool, sub, name),
+ fmt.Sprintf("%s/%s", pool, destSubvol))
+ if err != nil {
+ logger.Errorf("zfs clone failed: %s", output)
+ return fmt.Errorf("Failed to clone the sub-volume: %s", output)
+ }
+ }
+
+ return nil
+}
+
+// zfsIsEnabled returns whether zfs backend is supported.
+func zfsIsEnabled() bool {
+ out, err := exec.LookPath("zfs")
+ if err != nil || len(out) == 0 {
+ return false
+ }
+
+ return true
+}
+
+// zfsToolVersionGet returns the ZFS tools version
+func zfsToolVersionGet() (string, error) {
+ // This function is only really ever relevant on Ubuntu as the only
+ // distro that ships out of sync tools and kernel modules
+ out, err := shared.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux")
+ if err != nil {
+ return "", err
+ }
+
+ return strings.TrimSpace(string(out)), nil
+}
+
+// zfsModuleVersionGet returns the ZFS module version
+func zfsModuleVersionGet() (string, error) {
+ var zfsVersion string
+
+ if shared.PathExists("/sys/module/zfs/version") {
+ out, err := ioutil.ReadFile("/sys/module/zfs/version")
+ if err != nil {
+ return "", fmt.Errorf("Could not determine ZFS module version")
+ }
+
+ zfsVersion = string(out)
+ } else {
+ out, err := shared.RunCommand("modinfo", "-F", "version", "zfs")
+ if err != nil {
+ return "", fmt.Errorf("Could not determine ZFS module version")
+ }
+
+ zfsVersion = out
+ }
+
+ return strings.TrimSpace(zfsVersion), nil
+}
+
+// ZfsPoolVolumeExists verifies if a specific ZFS pool or volume exists.
+func ZfsPoolVolumeExists(dataset string) (bool, error) {
+ output, err := shared.RunCommand(
+ "zfs", "list", "-Ho", "name")
+
+ if err != nil {
+ return false, err
+ }
+
+ for _, name := range strings.Split(output, "\n") {
+ if name == dataset {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+func ZfsIdmapSetSkipper(dir string, absPath string, fi os.FileInfo) bool {
+ strippedPath := absPath
+ if dir != "" {
+ strippedPath = absPath[len(dir):]
+ }
+
+ if fi.IsDir() && strippedPath == "/.zfs/snapshot" {
+ return true
+ }
+
+ return false
+}
diff --git a/lxd/storage_migration_zfs.go b/lxd/storage_migration_zfs.go
new file mode 100644
index 0000000000..19d024cb5b
--- /dev/null
+++ b/lxd/storage_migration_zfs.go
@@ -0,0 +1,372 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "strings"
+
+ "github.com/gorilla/websocket"
+ "github.com/pborman/uuid"
+
+ "github.com/lxc/lxd/lxd/state"
+ driver "github.com/lxc/lxd/lxd/storage"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+type zfsMigrationSourceDriver2 struct {
+ container container
+ snapshots []container
+ zfsSnapshotNames []string
+ runningSnapName string
+ stoppedSnapName string
+ zfsFeatures []string
+ onDiskPoolName string
+ pool *api.StoragePool
+ state *state.State
+}
+
+func zfsMigrationSource(s *state.State, pool *api.StoragePool, args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
+ onDiskPoolName := pool.Name
+
+ if pool.Config["zfs.pool_name"] != "" {
+ onDiskPoolName = pool.Config["zfs.pool_name"]
+ }
+
+ /* If the container is a snapshot, let's just send that; we don't need
+ * to send anything else, because that's all the user asked for.
+ */
+ if args.Container.IsSnapshot() {
+ return &zfsMigrationSourceDriver2{container: args.Container, zfsFeatures: args.ZfsFeatures, pool: pool, onDiskPoolName: onDiskPoolName}, nil
+ }
+
+ migrationDriver := zfsMigrationSourceDriver2{
+ container: args.Container,
+ snapshots: []container{},
+ zfsSnapshotNames: []string{},
+ zfsFeatures: args.ZfsFeatures,
+ pool: pool,
+ onDiskPoolName: onDiskPoolName,
+ }
+
+ if args.ContainerOnly {
+ return &migrationDriver, nil
+ }
+
+ /* List all the snapshots in order of reverse creation. The idea here
+ * is that we send the oldest to newest snapshot, hopefully saving on
+ * xfer costs. Then, after all that, we send the container itself.
+ */
+ snapshots, err := driver.ZfsPoolListSnapshots(migrationDriver.onDiskPoolName, fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name())))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, snap := range snapshots {
+ /* In the case of e.g. multiple copies running at the same
+ * time, we will have potentially multiple migration-send
+ * snapshots. (Or in the case of the test suite, sometimes one
+ * will take too long to delete.)
+ */
+ if !strings.HasPrefix(snap, "snapshot-") {
+ continue
+ }
+
+ lxdName := fmt.Sprintf("%s%s%s", args.Container.Name(), shared.SnapshotDelimiter, snap[len("snapshot-"):])
+ snapshot, err := containerLoadByProjectAndName(s, args.Container.Project(), lxdName)
+ if err != nil {
+ return nil, err
+ }
+
+ migrationDriver.snapshots = append(migrationDriver.snapshots, snapshot)
+ migrationDriver.zfsSnapshotNames = append(migrationDriver.zfsSnapshotNames, snap)
+ }
+
+ return &migrationDriver, nil
+}
+
+func (s *zfsMigrationSourceDriver2) send(conn *websocket.Conn, zfsName string, zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
+ sourceParentName, _, _ := containerGetParentAndSnapshotName(s.container.Name())
+ args := []string{"send"}
+
+ // Negotiated options
+ if s.zfsFeatures != nil && len(s.zfsFeatures) > 0 {
+ if shared.StringInSlice("compress", s.zfsFeatures) {
+ args = append(args, "-c")
+ args = append(args, "-L")
+ }
+ }
+
+ args = append(args, []string{fmt.Sprintf("%s/containers/%s@%s", s.onDiskPoolName, projectPrefix(s.container.Project(), sourceParentName), zfsName)}...)
+ if zfsParent != "" {
+ args = append(args, "-i", fmt.Sprintf("%s/containers/%s@%s", s.onDiskPoolName, projectPrefix(s.container.Project(), s.container.Name()), zfsParent))
+ }
+
+ cmd := exec.Command("zfs", args...)
+
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return err
+ }
+
+ readPipe := io.ReadCloser(stdout)
+ if readWrapper != nil {
+ readPipe = readWrapper(stdout)
+ }
+
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+
+ if err := cmd.Start(); err != nil {
+ return err
+ }
+
+ <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
+
+ output, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ logger.Errorf("Problem reading zfs send stderr: %s", err)
+ }
+
+ err = cmd.Wait()
+ if err != nil {
+ logger.Errorf("Problem with zfs send: %s", string(output))
+ }
+
+ return err
+}
+
+func (s *zfsMigrationSourceDriver2) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error {
+ if s.container.IsSnapshot() {
+ _, snapOnlyName, _ := containerGetParentAndSnapshotName(s.container.Name())
+ snapshotName := fmt.Sprintf("snapshot-%s", snapOnlyName)
+ wrapper := StorageProgressReader(op, "fs_progress", s.container.Name())
+ return s.send(conn, snapshotName, "", wrapper)
+ }
+
+ lastSnap := ""
+ if !containerOnly {
+ for i, snap := range s.zfsSnapshotNames {
+ prev := ""
+ if i > 0 {
+ prev = s.zfsSnapshotNames[i-1]
+ }
+
+ lastSnap = snap
+
+ wrapper := StorageProgressReader(op, "fs_progress", snap)
+ if err := s.send(conn, snap, prev, wrapper); err != nil {
+ return err
+ }
+ }
+ }
+
+ s.runningSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String())
+ if err := driver.ZfsPoolVolumeSnapshotCreate(s.onDiskPoolName, fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.runningSnapName); err != nil {
+ return err
+ }
+
+ wrapper := StorageProgressReader(op, "fs_progress", s.container.Name())
+ if err := s.send(conn, s.runningSnapName, lastSnap, wrapper); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *zfsMigrationSourceDriver2) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error {
+ s.stoppedSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String())
+ if err := driver.ZfsPoolVolumeSnapshotCreate(s.onDiskPoolName, fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.stoppedSnapName); err != nil {
+ return err
+ }
+
+ if err := s.send(conn, s.stoppedSnapName, s.runningSnapName, nil); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *zfsMigrationSourceDriver2) Cleanup() {
+ if s.stoppedSnapName != "" {
+ driver.ZfsPoolVolumeSnapshotDestroy(s.onDiskPoolName, fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.stoppedSnapName)
+ }
+ if s.runningSnapName != "" {
+ driver.ZfsPoolVolumeSnapshotDestroy(s.onDiskPoolName, fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.runningSnapName)
+ }
+}
+
+func (s *zfsMigrationSourceDriver2) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error {
+ msg := fmt.Sprintf("Function not implemented")
+ logger.Errorf(msg)
+ return fmt.Errorf(msg)
+}
+
+func zfsMigrationSink(pool *api.StoragePool, volume *api.StorageVolume, conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
+ var poolName string
+
+ if pool.Config["zfs.pool_name"] != "" {
+ poolName = pool.Config["zfs.pool_name"]
+ } else {
+ poolName = pool.Name
+ }
+
+ zfsRecv := func(zfsName string, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
+ zfsFsName := fmt.Sprintf("%s/%s", poolName, zfsName)
+ args := []string{"receive", "-F", "-u", zfsFsName}
+ cmd := exec.Command("zfs", args...)
+
+ stdin, err := cmd.StdinPipe()
+ if err != nil {
+ return err
+ }
+
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+
+ if err := cmd.Start(); err != nil {
+ return err
+ }
+
+ writePipe := io.WriteCloser(stdin)
+ if writeWrapper != nil {
+ writePipe = writeWrapper(stdin)
+ }
+
+ <-shared.WebsocketRecvStream(writePipe, conn)
+
+ output, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ logger.Debugf("Problem reading zfs recv stderr %s", err)
+ }
+
+ err = cmd.Wait()
+ if err != nil {
+ logger.Errorf("Problem with zfs recv: %s", string(output))
+ }
+ return err
+ }
+
+ /* In some versions of zfs we can write `zfs recv -F` to mounted
+ * filesystems, and in some versions we can't. So, let's always unmount
+ * this fs (it's empty anyway) before we zfs recv. N.B. that `zfs recv`
+ * of a snapshot also needs tha actual fs that it has snapshotted
+ * unmounted, so we do this before receiving anything.
+ */
+ zfsName := fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name()))
+ containerMntPoint := getContainerMountPoint(args.Container.Project(), pool.Name, args.Container.Name())
+ if shared.IsMountPoint(containerMntPoint) {
+ err := driver.ZfsUmount(poolName, zfsName, containerMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ if len(args.Snapshots) > 0 {
+ snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", pool.Name, "containers-snapshots", projectPrefix(args.Container.Project(), volume.Name))
+ snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(args.Container.Project(), args.Container.Name()))
+ if !shared.PathExists(snapshotMntPointSymlink) {
+ err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // At this point we have already figured out the parent
+ // container's root disk device so we can simply
+ // retrieve it from the expanded devices.
+ parentStoragePool := ""
+ parentExpandedDevices := args.Container.ExpandedDevices()
+ parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := shared.GetRootDiskDevice(parentExpandedDevices)
+ if parentLocalRootDiskDeviceKey != "" {
+ parentStoragePool = parentLocalRootDiskDevice["pool"]
+ }
+
+ // A little neuroticism.
+ if parentStoragePool == "" {
+ return fmt.Errorf("detected that the container's root device is missing the pool property during BTRFS migration")
+ }
+
+ for _, snap := range args.Snapshots {
+ ctArgs := snapshotProtobufToContainerArgs(args.Container.Project(), args.Container.Name(), snap)
+
+ // Ensure that snapshot and parent container have the
+ // same storage pool in their local root disk device.
+ // If the root disk device for the snapshot comes from a
+ // profile on the new instance as well we don't need to
+ // do anything.
+ if ctArgs.Devices != nil {
+ snapLocalRootDiskDeviceKey, _, _ := shared.GetRootDiskDevice(ctArgs.Devices)
+ if snapLocalRootDiskDeviceKey != "" {
+ ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
+ }
+ }
+ _, err := containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs)
+ if err != nil {
+ return err
+ }
+
+ wrapper := StorageProgressWriter(op, "fs_progress", snap.GetName())
+ name := fmt.Sprintf("containers/%s at snapshot-%s", projectPrefix(args.Container.Project(), args.Container.Name()), snap.GetName())
+ if err := zfsRecv(name, wrapper); err != nil {
+ return err
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(args.Container.Project(), poolName, fmt.Sprintf("%s/%s", args.Container.Name(), *snap.Name))
+ if !shared.PathExists(snapshotMntPoint) {
+ err := os.MkdirAll(snapshotMntPoint, 0700)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ defer func() {
+ /* clean up our migration-send snapshots that we got from recv. */
+ zfsSnapshots, err := driver.ZfsPoolListSnapshots(poolName, fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name())))
+ if err != nil {
+ logger.Errorf("Failed listing snapshots post migration: %s", err)
+ return
+ }
+
+ for _, snap := range zfsSnapshots {
+ // If we received a bunch of snapshots, remove the migration-send-* ones, if not, wipe any snapshot we got
+ if args.Snapshots != nil && len(args.Snapshots) > 0 && !strings.HasPrefix(snap, "migration-send") {
+ continue
+ }
+
+ driver.ZfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name())), snap)
+ }
+ }()
+
+ /* finally, do the real container */
+ wrapper := StorageProgressWriter(op, "fs_progress", args.Container.Name())
+ if err := zfsRecv(zfsName, wrapper); err != nil {
+ return err
+ }
+
+ if args.Live {
+ /* and again for the post-running snapshot if this was a live migration */
+ wrapper := StorageProgressWriter(op, "fs_progress", args.Container.Name())
+ if err := zfsRecv(zfsName, wrapper); err != nil {
+ return err
+ }
+ }
+
+ /* Sometimes, zfs recv mounts this anyway, even if we pass -u
+ * (https://forums.freebsd.org/threads/zfs-receive-u-shouldnt-mount-received-filesystem-right.36844/)
+ * but sometimes it doesn't. Let's try to mount, but not complain about
+ * failure.
+ */
+ driver.ZfsMount(poolName, zfsName)
+ return nil
+}
From 730ae1c18d5c78de3258a7b4c87567ddd9c4f521 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 16:02:11 +0200
Subject: [PATCH 12/15] lxd: Use new storage code
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/api_internal.go | 15 +-
lxd/container_lxc.go | 10 +-
lxd/main_init_interactive.go | 7 +-
lxd/migrate_container.go | 5 +-
lxd/migrate_storage_volumes.go | 5 +-
lxd/patches.go | 21 +-
lxd/storage.go | 2515 +++++++++++++++++++++++++++++++-
7 files changed, 2498 insertions(+), 80 deletions(-)
diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index fb47c4ef58..b8857001e4 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -20,12 +20,13 @@ import (
"github.com/lxc/lxd/lxd/db/cluster"
"github.com/lxc/lxd/lxd/db/node"
"github.com/lxc/lxd/lxd/db/query"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
+ log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/lxc/lxd/shared/osarch"
- log "github.com/lxc/lxd/shared/log15"
runtimeDebug "runtime/debug"
)
@@ -607,7 +608,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
}
case "zfs":
onDiskPoolName := backup.Pool.Config["zfs.pool_name"]
- snaps, err := zfsPoolListSnapshots(onDiskPoolName,
+ snaps, err := driver.ZfsPoolListSnapshots(onDiskPoolName,
fmt.Sprintf("containers/%s", req.Name))
if err != nil {
return InternalError(err)
@@ -663,10 +664,10 @@ func internalImport(d *Daemon, r *http.Request) Response {
switch backup.Pool.Driver {
case "btrfs":
snapName := fmt.Sprintf("%s/%s", req.Name, od)
- err = btrfsSnapshotDeleteInternal(project, poolName, snapName)
+ err = driver.BtrfsSnapshotDeleteInternal(project, poolName, snapName)
case "dir":
snapName := fmt.Sprintf("%s/%s", req.Name, od)
- err = dirSnapshotDeleteInternal(project, poolName, snapName)
+ err = driver.DirSnapshotDeleteInternal(project, poolName, snapName)
case "lvm":
onDiskPoolName := backup.Pool.Config["lvm.vg_name"]
if onDiskPoolName == "" {
@@ -698,7 +699,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
case "zfs":
onDiskPoolName := backup.Pool.Config["zfs.pool_name"]
snapName := fmt.Sprintf("%s/%s", req.Name, od)
- err = zfsSnapshotDeleteInternal(project, poolName, snapName,
+ err = driver.ZfsSnapshotDeleteInternal(project, poolName, snapName,
onDiskPoolName)
}
if err != nil {
@@ -710,7 +711,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
switch backup.Pool.Driver {
case "btrfs":
snpMntPt := getSnapshotMountPoint(project, backup.Pool.Name, snap.Name)
- if !shared.PathExists(snpMntPt) || !isBtrfsSubVolume(snpMntPt) {
+ if !shared.PathExists(snpMntPt) || !driver.IsBtrfsSubVolume(snpMntPt) {
if req.Force {
continue
}
@@ -771,7 +772,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
ctName, csName, _ := containerGetParentAndSnapshotName(snap.Name)
snapshotName := fmt.Sprintf("snapshot-%s", csName)
- exists := zfsFilesystemEntityExists(poolName,
+ exists := driver.ZfsFilesystemEntityExists(poolName,
fmt.Sprintf("containers/%s@%s", ctName,
snapshotName))
if !exists {
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 373e4ea0ba..1afff82b5b 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -2034,7 +2034,7 @@ func (c *containerLXC) startCommon() (string, error) {
if diskIdmap != nil {
if c.Storage().GetStorageType() == storageTypeZfs {
- err = diskIdmap.UnshiftRootfs(c.RootfsPath(), zfsIdmapSetSkipper)
+ err = diskIdmap.UnshiftRootfs(c.RootfsPath(), driver.ZfsIdmapSetSkipper)
} else {
err = diskIdmap.UnshiftRootfs(c.RootfsPath(), nil)
}
@@ -2048,7 +2048,7 @@ func (c *containerLXC) startCommon() (string, error) {
if nextIdmap != nil && !c.state.OS.Shiftfs {
if c.Storage().GetStorageType() == storageTypeZfs {
- err = nextIdmap.ShiftRootfs(c.RootfsPath(), zfsIdmapSetSkipper)
+ err = nextIdmap.ShiftRootfs(c.RootfsPath(), driver.ZfsIdmapSetSkipper)
} else {
err = nextIdmap.ShiftRootfs(c.RootfsPath(), nil)
}
@@ -5174,7 +5174,7 @@ func (c *containerLXC) Export(w io.Writer, properties map[string]string) error {
var err error
if c.Storage().GetStorageType() == storageTypeZfs {
- err = idmap.UnshiftRootfs(c.RootfsPath(), zfsIdmapSetSkipper)
+ err = idmap.UnshiftRootfs(c.RootfsPath(), driver.ZfsIdmapSetSkipper)
} else {
err = idmap.UnshiftRootfs(c.RootfsPath(), nil)
}
@@ -5184,7 +5184,7 @@ func (c *containerLXC) Export(w io.Writer, properties map[string]string) error {
}
if c.Storage().GetStorageType() == storageTypeZfs {
- defer idmap.ShiftRootfs(c.RootfsPath(), zfsIdmapSetSkipper)
+ defer idmap.ShiftRootfs(c.RootfsPath(), driver.ZfsIdmapSetSkipper)
} else {
defer idmap.ShiftRootfs(c.RootfsPath(), nil)
}
@@ -5499,7 +5499,7 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
}
if c.Storage().GetStorageType() == storageTypeZfs {
- err = idmapset.ShiftRootfs(args.stateDir, zfsIdmapSetSkipper)
+ err = idmapset.ShiftRootfs(args.stateDir, driver.ZfsIdmapSetSkipper)
} else {
err = idmapset.ShiftRootfs(args.stateDir, nil)
}
diff --git a/lxd/main_init_interactive.go b/lxd/main_init_interactive.go
index 0c179ef833..4b03bcb7c6 100644
--- a/lxd/main_init_interactive.go
+++ b/lxd/main_init_interactive.go
@@ -15,8 +15,9 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
- "github.com/lxc/lxd/client"
+ lxd "github.com/lxc/lxd/client"
"github.com/lxc/lxd/lxd/cluster"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -487,7 +488,7 @@ func (c *cmdInit) askStoragePool(config *cmdInitData, d lxd.ContainerServer, poo
if cli.AskBool(fmt.Sprintf("Create a new %s pool? (yes/no) [default=yes]: ", strings.ToUpper(pool.Driver)), "yes") {
if pool.Driver == "zfs" && os.Geteuid() == 0 {
- poolVolumeExists, err := zfsPoolVolumeExists(pool.Name)
+ poolVolumeExists, err := driver.ZfsPoolVolumeExists(pool.Name)
if err == nil && poolVolumeExists {
return fmt.Errorf("'%s' ZFS pool already exists", pool.Name)
}
@@ -564,7 +565,7 @@ func (c *cmdInit) askStoragePool(config *cmdInitData, d lxd.ContainerServer, poo
}
if pool.Driver == "zfs" && os.Geteuid() == 0 {
- poolVolumeExists, err := zfsPoolVolumeExists(pool.Config["source"])
+ poolVolumeExists, err := driver.ZfsPoolVolumeExists(pool.Config["source"])
if err == nil && !poolVolumeExists {
return fmt.Errorf("'%s' ZFS pool or dataset does not exist", pool.Config["source"])
}
diff --git a/lxd/migrate_container.go b/lxd/migrate_container.go
index c2bda84afb..6390507f56 100644
--- a/lxd/migrate_container.go
+++ b/lxd/migrate_container.go
@@ -17,6 +17,7 @@ import (
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/migration"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -394,7 +395,7 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
},
}
- if len(zfsVersion) >= 3 && zfsVersion[0:3] != "0.6" {
+ if len(driver.ZfsVersion) >= 3 && driver.ZfsVersion[0:3] != "0.6" {
header.ZfsFeatures = &migration.ZfsFeatures{
Compress: &hasFeature,
}
@@ -864,7 +865,7 @@ func (c *migrationSink) Do(migrateOp *operation) error {
}
// Return those ZFS features we know about (with the value sent by the remote)
- if len(zfsVersion) >= 3 && zfsVersion[0:3] != "0.6" {
+ if len(driver.ZfsVersion) >= 3 && driver.ZfsVersion[0:3] != "0.6" {
if header.ZfsFeatures != nil && header.ZfsFeatures.Compress != nil {
resp.ZfsFeatures = &migration.ZfsFeatures{
Compress: header.ZfsFeatures.Compress,
diff --git a/lxd/migrate_storage_volumes.go b/lxd/migrate_storage_volumes.go
index bb8748e420..6a287f35c6 100644
--- a/lxd/migrate_storage_volumes.go
+++ b/lxd/migrate_storage_volumes.go
@@ -7,6 +7,7 @@ import (
"github.com/gorilla/websocket"
"github.com/lxc/lxd/lxd/migration"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/logger"
@@ -91,7 +92,7 @@ func (s *migrationSourceWs) DoStorage(migrateOp *operation) error {
},
}
- if len(zfsVersion) >= 3 && zfsVersion[0:3] != "0.6" {
+ if len(driver.ZfsVersion) >= 3 && driver.ZfsVersion[0:3] != "0.6" {
header.ZfsFeatures = &migration.ZfsFeatures{
Compress: &hasFeature,
}
@@ -290,7 +291,7 @@ func (c *migrationSink) DoStorage(migrateOp *operation) error {
},
}
- if len(zfsVersion) >= 3 && zfsVersion[0:3] != "0.6" {
+ if len(driver.ZfsVersion) >= 3 && driver.ZfsVersion[0:3] != "0.6" {
resp.ZfsFeatures = &migration.ZfsFeatures{
Compress: &hasFeature,
}
diff --git a/lxd/patches.go b/lxd/patches.go
index d6cf113466..eaa7917e1c 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -12,15 +12,16 @@ import (
"github.com/boltdb/bolt"
"github.com/hashicorp/raft"
- "github.com/hashicorp/raft-boltdb"
+ raftboltdb "github.com/hashicorp/raft-boltdb"
+ "github.com/pkg/errors"
+
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/db/query"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/logger"
- "github.com/pkg/errors"
-
log "github.com/lxc/lxd/shared/log15"
+ "github.com/lxc/lxd/shared/logger"
)
/* Patches are one-time actions that are sometimes needed to update
@@ -599,7 +600,7 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, defaultPoolName string,
if shared.PathExists(oldContainerMntPoint) && !shared.PathExists(newContainerMntPoint) {
err = os.Rename(oldContainerMntPoint, newContainerMntPoint)
if err != nil {
- err := btrfsSubVolumeCreate(newContainerMntPoint)
+ err := driver.BtrfsSubVolumeCreate(newContainerMntPoint)
if err != nil {
return err
}
@@ -610,7 +611,7 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, defaultPoolName string,
return err
}
- btrfsSubVolumesDelete(oldContainerMntPoint)
+ driver.BtrfsSubVolumesDelete(oldContainerMntPoint)
if shared.PathExists(oldContainerMntPoint) {
err = os.RemoveAll(oldContainerMntPoint)
if err != nil {
@@ -684,9 +685,9 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, defaultPoolName string,
oldSnapshotMntPoint := shared.VarPath("snapshots", cs)
newSnapshotMntPoint := getSnapshotMountPoint("default", defaultPoolName, cs)
if shared.PathExists(oldSnapshotMntPoint) && !shared.PathExists(newSnapshotMntPoint) {
- err = btrfsSnapshot(oldSnapshotMntPoint, newSnapshotMntPoint, true)
+ err = driver.BtrfsSnapshot(oldSnapshotMntPoint, newSnapshotMntPoint, true)
if err != nil {
- err := btrfsSubVolumeCreate(newSnapshotMntPoint)
+ err := driver.BtrfsSubVolumeCreate(newSnapshotMntPoint)
if err != nil {
return err
}
@@ -697,7 +698,7 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, defaultPoolName string,
return err
}
- btrfsSubVolumesDelete(oldSnapshotMntPoint)
+ driver.BtrfsSubVolumesDelete(oldSnapshotMntPoint)
if shared.PathExists(oldSnapshotMntPoint) {
err = os.RemoveAll(oldSnapshotMntPoint)
if err != nil {
@@ -706,7 +707,7 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, defaultPoolName string,
}
} else {
// Delete the old subvolume.
- err = btrfsSubVolumesDelete(oldSnapshotMntPoint)
+ err = driver.BtrfsSubVolumesDelete(oldSnapshotMntPoint)
if err != nil {
return err
}
diff --git a/lxd/storage.go b/lxd/storage.go
index 9df1e51632..f882bc0419 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "io/ioutil"
"os"
"sync"
"sync/atomic"
@@ -14,10 +15,12 @@ import (
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/migration"
"github.com/lxc/lxd/lxd/state"
+ driver "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/idmap"
"github.com/lxc/lxd/shared/ioprogress"
+ log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/lxc/lxd/shared/version"
)
@@ -129,6 +132,2378 @@ func storageStringToType(sName string) (storageType, error) {
return -1, fmt.Errorf("invalid storage type name")
}
+type Storage struct {
+ sType storageType
+ sTypeName string
+
+ s *state.State
+
+ poolID int64
+ pool *api.StoragePool
+
+ volumeID int64
+ volume *api.StorageVolume
+
+ driver StorageDriver
+}
+
+func (s *Storage) GetStorageType() storageType {
+ return s.sType
+}
+
+func (s *Storage) GetStorageTypeName() string {
+ return s.sTypeName
+}
+
+func (s *Storage) GetStorageTypeVersion() string {
+ return s.driver.GetVersion()
+}
+
+func (s *Storage) GetState() *state.State {
+ return s.s
+}
+
+func (s *Storage) GetStoragePoolWritable() api.StoragePoolPut {
+ return s.pool.Writable()
+}
+
+func (s *Storage) SetStoragePoolWritable(writable *api.StoragePoolPut) {
+ s.pool.StoragePoolPut = *writable
+}
+
+func (s *Storage) GetStoragePool() *api.StoragePool {
+ return s.pool
+}
+
+func (s *Storage) GetStoragePoolVolumeWritable() api.StorageVolumePut {
+ return s.volume.Writable()
+}
+
+func (s *Storage) SetStoragePoolVolumeWritable(writable *api.StorageVolumePut) {
+ s.volume.StorageVolumePut = *writable
+}
+
+func (s *Storage) GetStoragePoolVolume() *api.StorageVolume {
+ return s.volume
+}
+
+func (s *Storage) GetContainerPoolInfo() (int64, string, string) {
+ return s.poolID, s.pool.Name, s.pool.Name
+}
+
+func (s *Storage) StorageCoreInit() error {
+ return s.driver.Init()
+}
+
+func (s *Storage) StoragePoolInit() error {
+ s.driver.SharedInit(s.s, s.pool, s.poolID, s.volume)
+ return s.StorageCoreInit()
+}
+
+func (s *Storage) StoragePoolCheck() error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Checking storage pool",
+ "Checked storage pool",
+ "Failed to check storage pool",
+ &ctx, &success, &err)()
+
+ err = s.driver.StoragePoolCheck()
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolCreate() error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Creating storage pool",
+ "Created storage pool",
+ "Failed to create storage pool",
+ &ctx, &success, &err)()
+
+ s.pool.Config["volatile.initial_source"] = s.pool.Config["source"]
+
+ err = s.driver.StoragePoolCreate()
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolDelete() error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Deleting storage pool",
+ "Deleted storage pool",
+ "Failed to delete storage pool",
+ &ctx, &success, &err)()
+
+ err = s.driver.StoragePoolDelete()
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolMount() (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Mounting storage pool",
+ "Mounted storage pool",
+ "Failed to mount storage pool",
+ &ctx, &success, &err)()
+
+ ok, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) StoragePoolUmount() (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Unmounting storage pool",
+ "Unmounted storage pool",
+ "Failed to unmount storage pool",
+ &ctx, &success, &err)()
+
+ ok, err := s.driver.StoragePoolUmount()
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) StoragePoolResources() (*api.ResourcesStoragePool, error) {
+ return s.driver.StoragePoolResources()
+}
+
+func (s *Storage) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Updating storage pool",
+ "Updated storage pool",
+ "Failed to update storage pool",
+ &ctx, &success, &err)()
+
+ changeable := changeableStoragePoolProperties[s.sTypeName]
+ unchangeable := []string{}
+
+ for _, change := range changedConfig {
+ if !shared.StringInSlice(change, changeable) {
+ unchangeable = append(unchangeable, change)
+ }
+ }
+
+ if len(unchangeable) > 0 {
+ err = updateStoragePoolError(unchangeable, s.sTypeName)
+ return err
+ }
+
+ err = s.driver.StoragePoolUpdate(writable, changedConfig)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeCreate() error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "volume": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Creating storage pool volume",
+ "Created storage pool volume",
+ "Failed to create storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ isSnapshot := shared.IsSnapshot(s.volume.Name)
+
+ // Create subvolume path on the storage pool.
+ var volumePath string
+
+ if isSnapshot {
+ volumePath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, "")
+ } else {
+ volumePath = getStoragePoolVolumeMountPoint(s.pool.Name, "")
+ }
+
+ if !shared.PathExists(volumePath) {
+ err = os.MkdirAll(volumePath, customDirMode)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = s.driver.VolumeCreate("default", s.volume.Name, driver.VolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeDelete() error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "volume": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Deleting storage pool volume",
+ "Deleted storage pool volume",
+ "Failed to delete storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ volumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
+
+ err = s.driver.VolumeDelete("default", s.volume.Name, true, driver.VolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+
+ err = os.RemoveAll(volumeMntPoint)
+ if err != nil {
+ return err
+ }
+
+ err = s.s.Cluster.StoragePoolVolumeDelete(
+ "default",
+ s.volume.Name,
+ storagePoolVolumeTypeCustom,
+ s.poolID)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeMount() (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "volume": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Mounting storage pool volume",
+ "Mounted storage pool volume",
+ "Failed to mount storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return ourMount, err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ ok, err := s.driver.VolumeMount("default", s.volume.Name, driver.VolumeTypeCustom)
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) StoragePoolVolumeUmount() (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "volume": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Unmounting storage pool volume",
+ "Unmounting storage pool volume",
+ "Failed to unmount storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return ourMount, err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ ok, err := s.driver.VolumeUmount("default", s.volume.Name, driver.VolumeTypeCustom)
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) StoragePoolVolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "volume": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Updating storage pool volume",
+ "Updated storage pool volume",
+ "Failed to update storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ if writable.Restore != "" {
+ err = s.driver.VolumeSnapshotRestore("default", fmt.Sprintf("%s/%s", s.volume.Name, writable.Restore), s.volume.Name, driver.VolumeTypeCustomSnapshot)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+ }
+
+ changeable := changeableStoragePoolVolumeProperties[s.sTypeName]
+ unchangeable := []string{}
+ for _, change := range changedConfig {
+ if !shared.StringInSlice(change, changeable) {
+ unchangeable = append(unchangeable, change)
+ }
+ }
+
+ if len(unchangeable) > 0 {
+ err = updateStoragePoolVolumeError(unchangeable, s.sTypeName)
+ return err
+ }
+
+ err = s.driver.VolumeUpdate(writable, changedConfig)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeRename(newName string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "old_name": s.volume.Name,
+ "new_name": newName,
+ }
+ success := false
+
+ defer logAction(
+ "Renaming storage pool volume",
+ "Renamed storage pool volume",
+ "Failed to rename storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ usedBy, err := storagePoolVolumeUsedByContainersGet(s.s, "default", s.volume.Name,
+ storagePoolVolumeTypeNameCustom)
+ if err != nil {
+ return err
+ }
+
+ if len(usedBy) > 0 {
+ err = fmt.Errorf(`storage volume "%s" on storage pool "%s" is attached to containers`,
+ s.volume.Name, s.pool.Name)
+ return err
+ }
+
+ err = s.driver.VolumeRename("default", s.volume.Name, newName, nil, driver.VolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+
+ err = s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, newName,
+ storagePoolVolumeTypeCustom, s.poolID)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeCopy(source *api.StorageVolumeSource) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": source.Name,
+ "target": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Copying storage pool volume",
+ "Copied storage pool volume",
+ "Failed to copy storage pool volume",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ if s.pool.Name != source.Pool {
+ err = s.doCrossPoolVolumeCopy(source)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+ }
+
+ isSnapshot := shared.IsSnapshot(source.Name)
+ volumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, "")
+
+ err = os.MkdirAll(volumeMntPoint, customDirMode)
+ if err != nil {
+ return err
+ }
+
+ if isSnapshot {
+ return s.driver.VolumeSnapshotCopy("default", source.Name, s.volume.Name, driver.VolumeTypeCustomSnapshot)
+ }
+
+ snapshots, err := s.s.Cluster.StoragePoolVolumeSnapshotsGetType(source.Name, storagePoolVolumeTypeCustom, s.poolID)
+ if err != nil {
+ return err
+ }
+
+ var snapOnlyNames []string
+
+ for _, snap := range snapshots {
+ snapOnlyNames = append(snapOnlyNames, shared.ExtractSnapshotName(snap))
+ }
+
+ err = s.driver.VolumeCopy("default", source.Name, s.volume.Name, snapOnlyNames, driver.VolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) doCrossPoolVolumeCopy(source *api.StorageVolumeSource) error {
+ // setup storage for the source volume
+ srcStorage, err := storagePoolVolumeInit(s.s, "default", source.Pool, source.Name,
+ storagePoolVolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+
+ ourMount, err := srcStorage.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+ if ourMount {
+ defer srcStorage.StoragePoolUmount()
+ }
+
+ err = s.StoragePoolVolumeCreate()
+ if err != nil {
+ return err
+ }
+
+ ourMount, err = s.StoragePoolVolumeMount()
+ if err != nil {
+ return err
+ }
+ if ourMount {
+ defer s.StoragePoolVolumeUmount()
+ }
+
+ dstMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
+ bwlimit := s.pool.Config["rsync.bwlimit"]
+
+ if !source.VolumeOnly {
+ snapshots, err := storagePoolVolumeSnapshotsGet(s.s, source.Pool, source.Name, storagePoolVolumeTypeCustom)
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snapshots {
+ srcMountPoint := getStoragePoolVolumeSnapshotMountPoint(source.Pool, snap)
+
+ _, err = rsyncLocalCopy(srcMountPoint, dstMountPoint, bwlimit)
+ if err != nil {
+ logger.Errorf("Failed to rsync into ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
+ return err
+ }
+
+ _, snapOnlyName, _ := containerGetParentAndSnapshotName(source.Name)
+
+ s.StoragePoolVolumeSnapshotCreate(&api.StorageVolumeSnapshotsPost{Name: fmt.Sprintf("%s/%s", s.volume.Name, snapOnlyName)})
+ }
+ }
+
+ var srcMountPoint string
+
+ if shared.IsSnapshot(source.Name) {
+ srcMountPoint = getStoragePoolVolumeSnapshotMountPoint(source.Pool, source.Name)
+ } else {
+ srcMountPoint = getStoragePoolVolumeMountPoint(source.Pool, source.Name)
+ }
+
+ _, err = rsyncLocalCopy(srcMountPoint, dstMountPoint, bwlimit)
+ if err != nil {
+ os.RemoveAll(dstMountPoint)
+ return err
+ }
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": s.volume.Name,
+ "target": target.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Creating storage pool volume snapshot",
+ "Created storage pool volume snapshot",
+ "Failed to create storage pool volume snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ _, _, ok := containerGetParentAndSnapshotName(target.Name)
+ if !ok {
+ err = fmt.Errorf("Not a snapshot name")
+ return err
+ }
+
+ targetPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
+
+ err = os.MkdirAll(targetPath, snapshotsDirMode)
+ if err != nil {
+ return err
+ }
+
+ err = s.driver.VolumeSnapshotCreate("default", s.volume.Name, target.Name,
+ driver.VolumeTypeCustomSnapshot)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeSnapshotDelete() error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "volume": s.volume.Name,
+ }
+ success := false
+
+ defer logAction(
+ "Deleting storage pool volume snapshot",
+ "Deleted storage pool volume snapshot",
+ "Failed to delete storage pool volume snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ err = s.driver.VolumeSnapshotDelete("default", s.volume.Name, true, driver.VolumeTypeCustomSnapshot)
+ if err != nil {
+ return err
+ }
+
+ snapshotMntPoint := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
+
+ err = os.RemoveAll(snapshotMntPoint)
+ if err != nil {
+ return err
+ }
+
+ sourceVolumeName, _, _ := containerGetParentAndSnapshotName(s.volume.Name)
+ snapshotVolumePath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceVolumeName)
+
+ empty, _ := shared.PathIsEmpty(snapshotVolumePath)
+ if empty {
+ err = os.Remove(snapshotVolumePath)
+ if err != nil {
+ return err
+ }
+
+ snapshotSymlink := shared.VarPath("custom-snapshots", sourceVolumeName)
+ if shared.PathExists(snapshotSymlink) {
+ err = os.Remove(snapshotSymlink)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ err = s.s.Cluster.StoragePoolVolumeDelete(
+ "default",
+ s.volume.Name,
+ storagePoolVolumeTypeCustom,
+ s.poolID)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StoragePoolVolumeSnapshotRename(newName string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "old_name": s.volume.Name,
+ "new_name": newName,
+ }
+ success := false
+
+ defer logAction(
+ "Renaming storage pool volume snapshot",
+ "Renamed storage pool volume snapshot",
+ "Failed to rename storage pool volume snapshot",
+ &ctx, &success, &err)()
+
+ sourceName, _, _ := containerGetParentAndSnapshotName(s.volume.Name)
+ fullSnapshotName := fmt.Sprintf("%s%s%s", sourceName, shared.SnapshotDelimiter, newName)
+
+ oldPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
+ newPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, fullSnapshotName)
+
+ err = os.MkdirAll(newPath, customDirMode)
+ if err != nil {
+ return err
+ }
+
+ err = s.driver.VolumeSnapshotRename("default", s.volume.Name, fullSnapshotName, driver.VolumeTypeCustomSnapshot)
+ if err != nil {
+ return err
+ }
+
+ // It might be, that the driver already renamed the path.
+ if shared.PathExists(oldPath) {
+ err = os.Rename(oldPath, newPath)
+ }
+
+ err = s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, fullSnapshotName, storagePoolVolumeTypeCustom, s.poolID)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerCreate(container container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "name": container.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Creating container",
+ "Created container",
+ "Failed to create container",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ err = errors.Wrapf(err, "Mount storage pool '%s'", s.pool.Name)
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ containerPath := getContainerMountPoint("default", s.pool.Name, "")
+
+ err = os.MkdirAll(containerPath, containersDirMode)
+ if err != nil {
+ err = errors.Wrapf(err, "Create containers mountpoint '%s'", containerPath)
+ return err
+ }
+
+ // Create container volume
+ err = s.driver.VolumeCreate(container.Project(), container.Name(),
+ driver.VolumeTypeContainer)
+ if err != nil {
+ err = errors.Wrapf(err, "Create container '%s'", container.Name())
+ return err
+ }
+
+ // Create directories
+ containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
+
+ err = createContainerMountpoint(containerMntPoint, container.Path(), container.IsPrivileged())
+ if err != nil {
+ err = errors.Wrapf(err, "Create container mountpoint '%s'", containerMntPoint)
+ return err
+ }
+
+ revert := false
+
+ defer func() {
+ if revert {
+ deleteContainerMountpoint(containerMntPoint, container.Path(), s.GetStorageTypeName())
+ }
+ }()
+
+ success = true
+
+ return container.TemplateApply("create")
+}
+
+func (s *Storage) ContainerCreateFromImage(container container, fingerprint string, tracker *ioprogress.ProgressTracker) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "name": container.Name(),
+ "fingerprint": fingerprint,
+ }
+ success := false
+
+ defer logAction(
+ "Creating container from image",
+ "Created from image",
+ "Failed to create from image",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ err = errors.Wrapf(err, "Mount storage pool '%s'", s.pool.Name)
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ containerPath := getContainerMountPoint("default", s.pool.Name, "")
+
+ err = os.MkdirAll(containerPath, containersDirMode)
+ if err != nil {
+ err = errors.Wrapf(err, "Create containers mountpoint '%s'", containerPath)
+ return err
+ }
+
+ // ImageCreate / VolumeCreate
+ if s.sType == storageTypeBtrfs || s.sType == storageTypeZfs {
+ err = s.ImageCreate(fingerprint, tracker)
+ if err != nil {
+ err = errors.Wrapf(err, "Create image '%s'", fingerprint)
+ return err
+ }
+ }
+
+ // Create directories
+ containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
+
+ imageMntPoint := shared.VarPath("images", fingerprint)
+ revert := true
+
+ // Create container volume
+ if s.sType == storageTypeBtrfs || s.sType == storageTypeZfs {
+ err = s.driver.VolumeCopy(container.Project(), fingerprint, container.Name(), nil, driver.VolumeTypeImage)
+ if err != nil {
+ err = errors.Wrapf(err, "Copy volume")
+ return err
+ }
+
+ // For btrfs, it is important to create the container mountpoint _after_
+ // the subvolume has been created.
+ err = createContainerMountpoint(containerMntPoint, container.Path(), container.IsPrivileged())
+ if err != nil {
+ err = errors.Wrapf(err, "Create container mountpoint '%s'", containerMntPoint)
+ return err
+ }
+
+ defer func() {
+ if revert {
+ deleteContainerMountpoint(containerMntPoint, container.Path(), s.GetStorageTypeName())
+ }
+ }()
+
+ } else {
+ err = createContainerMountpoint(containerMntPoint, container.Path(), container.IsPrivileged())
+ if err != nil {
+ err = errors.Wrapf(err, "Create container mountpoint '%s'", containerMntPoint)
+ return err
+ }
+
+ defer func() {
+ if revert {
+ deleteContainerMountpoint(containerMntPoint, container.Path(), s.GetStorageTypeName())
+ }
+ }()
+
+ err = unpackImage(imageMntPoint, containerMntPoint, s.sType, s.s.OS.RunningInUserNS,
+ tracker)
+ if err != nil {
+ err = errors.Wrap(err, "Unpack image")
+ return err
+ }
+ }
+
+ revert = false
+ success = true
+
+ return container.TemplateApply("create")
+}
+
+func (s *Storage) ContainerDelete(c container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "container": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Deleting container",
+ "Deleted container",
+ "Failed to delete container",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ containerMntPoint := getContainerMountPoint(c.Project(), s.pool.Name, c.Name())
+ snapshotMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, c.Name())
+ // ${LXD_DIR}/snapshots/<container_name> to ${POOL}/snapshots/<container_name>
+ snapshotSymlink := shared.VarPath("snapshots", projectPrefix(c.Project(), c.Name()))
+
+ err = s.driver.VolumeDelete(c.Project(), c.Name(), true, driver.VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ err = deleteContainerMountpoint(containerMntPoint, c.Path(), s.GetStorageTypeName())
+ if err != nil {
+ return err
+ }
+
+ snapshots, err := c.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snapshots {
+ err = s.driver.VolumeSnapshotDelete(snap.Project(), snap.Name(), true, driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+
+ err = deleteSnapshotMountpoint(snapshotMntPoint, snapshotSymlink, snapshotMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerRename(c container, newName string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "old_name": c.Name(),
+ "new_name": newName,
+ }
+ success := false
+
+ logAction(
+ "Renaming container",
+ "Renamed container",
+ "Failed to rename container",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ oldContainerMntPoint := getContainerMountPoint(c.Project(), s.pool.Name, c.Name())
+ oldContainerSymlink := containerPath(c.Project(), c.Name(), false)
+ newContainerMntPoint := getContainerMountPoint(c.Project(), s.pool.Name, newName)
+ newContainerSymlink := containerPath(c.Project(), newName, false)
+
+ var snapshotNames []string
+
+ snapshots, err := c.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snapshots {
+ snapshotNames = append(snapshotNames, shared.ExtractSnapshotName(snap.Name()))
+ }
+
+ // Snapshots are renamed here as well as they're tied to a volume/containers.
+ // There's no need to call VolumeSnapshotRename for each snapshot.
+ err = s.driver.VolumeRename(c.Project(), c.Name(), newName, snapshotNames,
+ driver.VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ err = renameContainerMountpoint(oldContainerMntPoint, oldContainerSymlink,
+ newContainerMntPoint, newContainerSymlink)
+ if err != nil {
+ return err
+ }
+
+ if c.IsSnapshot() {
+ success = true
+ return nil
+ }
+
+ oldSnapshotsMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, c.Name())
+ newSnapshotsMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, newName)
+ oldSnapshotSymlink := containerPath(c.Project(), c.Name(), true)
+ newSnapshotSymlink := containerPath(c.Project(), newName, true)
+
+ err = renameContainerMountpoint(oldSnapshotsMntPoint, oldSnapshotSymlink, newSnapshotsMntPoint, newSnapshotSymlink)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerMount(c container) (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "container": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Mounting container",
+ "Mounted container",
+ "Failed to mount container",
+ &ctx, &success, &err)()
+
+ _, err = s.driver.StoragePoolMount()
+ if err != nil {
+ return false, err
+ }
+
+ ok, err := s.driver.VolumeMount(c.Project(), c.Name(), driver.VolumeTypeContainer)
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) ContainerUmount(c container, path string) (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "container": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Unmounting container",
+ "Unmounted container",
+ "Failed to unmount container",
+ &ctx, &success, &err)()
+
+ ok, err := s.driver.VolumeUmount(c.Project(), c.Name(), driver.VolumeTypeContainer)
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) ContainerCopy(target container, source container, containerOnly bool) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": source.Name(),
+ "target": target.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Copying container",
+ "Copied container",
+ "Failed to copy container",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ ourStart, err := source.StorageStart()
+ if err != nil {
+ return err
+ }
+
+ if ourStart {
+ defer source.StorageStop()
+ }
+
+ sourcePool, err := source.StoragePool()
+ if err != nil {
+ return err
+ }
+
+ targetPool, err := target.StoragePool()
+ if err != nil {
+ return err
+ }
+
+ var snapshots []container
+
+ if !containerOnly {
+ snapshots, err = source.Snapshots()
+ if err != nil {
+ return err
+ }
+ }
+
+ if sourcePool != targetPool {
+ err = s.doCrossPoolContainerCopy(target, source, containerOnly, false, snapshots)
+ if err != nil {
+ return err
+ }
+
+ success = true
+ return nil
+ }
+
+ containerMntPoint := getContainerMountPoint("default", s.pool.Name, "")
+
+ err = os.MkdirAll(containerMntPoint, containersDirMode)
+ if err != nil {
+ return err
+ }
+
+ targetMntPoint := getContainerMountPoint(target.Project(), s.pool.Name, target.Name())
+
+ var snapshotNames []string
+
+ if !containerOnly {
+ for _, c := range snapshots {
+ snapshotNames = append(snapshotNames, shared.ExtractSnapshotName(c.Name()))
+ }
+
+ snapshotParentMntPoint := getSnapshotMountPoint(target.Project(), s.pool.Name,
+ target.Name())
+ snapshotParentMntPointSymlink := shared.VarPath("snapshots",
+ projectPrefix(target.Project(), target.Name()))
+
+ err = createSnapshotMountpoint(snapshotParentMntPoint, snapshotParentMntPoint,
+ snapshotParentMntPointSymlink)
+ if err != nil {
+ return err
+ }
+ }
+
+ if shared.IsSnapshot(source.Name()) {
+ err = s.driver.VolumeSnapshotCopy(source.Project(), source.Name(), target.Name(), driver.VolumeTypeContainerSnapshot)
+ } else {
+ err = s.driver.VolumeCopy(source.Project(), source.Name(), target.Name(), snapshotNames, driver.VolumeTypeContainer)
+ }
+ if err != nil {
+ return err
+ }
+
+ err = createContainerMountpoint(targetMntPoint, target.Path(), target.IsPrivileged())
+ if err != nil {
+ return err
+ }
+
+ err = target.TemplateApply("copy")
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerGetUsage(container container) (int64, error) {
+ return s.driver.VolumeGetUsage(container.Project(), container.Name(), container.Path())
+}
+
+func (s *Storage) ContainerRefresh(target container, source container, snapshots []container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": source.Name(),
+ "target": target.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Refreshing container",
+ "Refreshed container",
+ "Failed to refresh container",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ err = s.doCrossPoolContainerCopy(target, source, len(snapshots) == 0, true, snapshots)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) doCrossPoolContainerCopy(target container, source container, containerOnly bool,
+ refresh bool, refreshSnapshots []container) error {
+ sourcePool, err := source.StoragePool()
+ if err != nil {
+ return err
+ }
+
+ targetPool, err := target.StoragePool()
+ if err != nil {
+ return err
+ }
+
+ // setup storage for the source volume
+ srcStorage, err := storagePoolVolumeInit(s.s, "default", sourcePool, source.Name(),
+ storagePoolVolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ ourMount, err := srcStorage.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+ if ourMount {
+ defer srcStorage.StoragePoolUmount()
+ }
+
+ var snapshots []container
+
+ if refresh {
+ snapshots = refreshSnapshots
+ } else {
+ snapshots, err = source.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ // create the main container
+ err = s.ContainerCreate(target)
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = s.ContainerMount(target)
+ if err != nil {
+ return err
+ }
+ defer s.ContainerUmount(target, shared.VarPath("containers", projectPrefix(target.Project(), target.Name())))
+
+ destContainerMntPoint := getContainerMountPoint(target.Project(), targetPool, target.Name())
+ bwlimit := s.pool.Config["rsync.bwlimit"]
+
+ if !containerOnly {
+ snapshotSubvolumePath := getSnapshotMountPoint(target.Project(), s.pool.Name, target.Name())
+ if !shared.PathExists(snapshotSubvolumePath) {
+ err := os.MkdirAll(snapshotSubvolumePath, containersDirMode)
+ if err != nil {
+ return err
+ }
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(target.Project(), s.pool.Name, s.volume.Name)
+ snapshotMntPointSymlink := containerPath(target.Project(), target.Name(), true)
+
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPoint, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snapshots {
+ srcSnapshotMntPoint := getSnapshotMountPoint(source.Project(), sourcePool, snap.Name())
+ targetParentName, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
+ destSnapshotMntPoint := getSnapshotMountPoint(target.Project(), targetPool,
+ fmt.Sprintf("%s%s%s", target.Name(), shared.SnapshotDelimiter, snapOnlyName))
+
+ switch s.sType {
+ case storageTypeZfs:
+ fallthrough
+ case storageTypeBtrfs:
+ _, err = rsyncLocalCopy(srcSnapshotMntPoint, destContainerMntPoint, bwlimit)
+ if err != nil {
+ return err
+ }
+
+ // create snapshot
+ err = s.driver.VolumeSnapshotCreate(target.Project(), target.Name(),
+ fmt.Sprintf("%s%s%s", target.Name(), shared.SnapshotDelimiter, snapOnlyName),
+ driver.VolumeTypeContainerSnapshot)
+ case storageTypeDir:
+ _, err = rsyncLocalCopy(srcSnapshotMntPoint, destSnapshotMntPoint, bwlimit)
+ default:
+ return fmt.Errorf("Cross pool copy not implemented for '%s'", s.sTypeName)
+ }
+ if err != nil {
+ return err
+ }
+
+ err := createSnapshotMountpoint(destSnapshotMntPoint, destSnapshotMntPoint,
+ shared.VarPath("snapshots",
+ projectPrefix(target.Project(), targetParentName)))
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ srcContainerMntPoint := getContainerMountPoint(source.Project(), sourcePool, source.Name())
+
+ _, err = rsyncLocalCopy(srcContainerMntPoint, destContainerMntPoint, bwlimit)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Storage) ContainerRestore(targetContainer container, sourceContainer container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": sourceContainer.Name(),
+ "target": targetContainer.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Restoring container",
+ "Restored container",
+ "Failed to restore container",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ snapshots, err := targetContainer.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ var snapshotNames []string
+
+ for _, snap := range snapshots {
+ snapshotNames = append(snapshotNames, snap.Name())
+ }
+
+ deleteSnapshots := func() error {
+ for i := len(snapshots) - 1; i != 0; i-- {
+ if snapshots[i].Name() == sourceContainer.Name() {
+ break
+ }
+
+ err := snapshots[i].Delete()
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
+
+ err = s.driver.VolumePrepareRestore(sourceContainer.Name(), targetContainer.Name(), snapshotNames, deleteSnapshots)
+ if err != nil {
+ return err
+ }
+
+ err = s.driver.VolumeSnapshotRestore(sourceContainer.Project(), sourceContainer.Name(),
+ targetContainer.Name(), driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerStorageReady(c container) bool {
+ return s.driver.VolumeReady(c.Project(), c.Name())
+}
+
+func (s *Storage) ContainerSnapshotCreate(target container, source container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": source.Name(),
+ "target": target.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Creating container snapshot",
+ "Created container snapshot",
+ "Failed to create container snapshot",
+ &ctx, &success, &err)()
+
+ _, err = s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ // We can only create the btrfs subvolume under the mounted storage
+ // pool. The on-disk layout for snapshots on a btrfs storage pool will
+ // thus be
+ // ${LXD_DIR}/storage-pools/<pool>/snapshots/. The btrfs tool will
+ // complain if the intermediate path does not exist, so create it if it
+ // doesn't already.
+ snapshotSubvolumePath := getSnapshotMountPoint(source.Project(), s.pool.Name, source.Name())
+ err = os.MkdirAll(snapshotSubvolumePath, containersDirMode)
+ if err != nil {
+ return err
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(source.Project(), s.pool.Name, source.Name())
+ snapshotMntPointSymlink := containerPath(source.Project(), source.Name(), target.IsSnapshot())
+
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPoint, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+
+ err = s.driver.VolumeSnapshotCreate(source.Project(), source.Name(), target.Name(), driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return s.ContainerDelete(target)
+ }
+
+ // This is used only in Dir
+ if s.sType == storageTypeDir && source.IsRunning() {
+ err = source.Freeze()
+ if err != nil {
+ // Don't just fail here
+ success = true
+ return nil
+ }
+
+ defer source.Unfreeze()
+
+ err = s.driver.VolumeSnapshotCreate(source.Project(), source.Name(), target.Name(), driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerSnapshotCreateEmpty(c container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "snapshot": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Creating empty container snapshot",
+ "Created empty container snapshot",
+ "Failed to create empty container snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ parentName, _, _ := containerGetParentAndSnapshotName(c.Name())
+ snapshotMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, parentName)
+
+ err = os.MkdirAll(snapshotMntPoint, containersDirMode)
+ if err != nil {
+ return err
+ }
+
+ err = s.driver.VolumeSnapshotCreate(c.Project(), "", c.Name(), driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+
+ sourceName, _, _ := containerGetParentAndSnapshotName(c.Name())
+ snapshotMntPointSymlinkTarget := getSnapshotMountPoint(c.Project(), s.pool.Name, sourceName)
+ snapshotMntPointSymlink := containerPath(c.Project(), sourceName, true)
+
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerSnapshotDelete(c container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "container": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Deleting container snapshot",
+ "Deleted container snapshot",
+ "Failed to delete container snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ sourceContainerName, _, _ := containerGetParentAndSnapshotName(c.Name())
+ snapshotMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, c.Name())
+ snapshotSymlink := shared.VarPath("snapshots", projectPrefix(c.Project(), sourceContainerName))
+
+ err = s.driver.VolumeSnapshotDelete(c.Project(), c.Name(), true, driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+
+ deleteSnapshotMountpoint(snapshotMntPoint, snapshotMntPoint, snapshotSymlink)
+
+ if shared.PathExists(snapshotMntPoint) {
+ err := os.RemoveAll(snapshotMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ snapshotContainerPath := getSnapshotMountPoint(c.Project(), s.pool.Name, sourceContainerName)
+
+ empty, _ := shared.PathIsEmpty(snapshotContainerPath)
+ if empty {
+ err = os.Remove(snapshotContainerPath)
+ if err != nil {
+ return err
+ }
+
+ snapshotSymlink := shared.VarPath("snapshots", projectPrefix(c.Project(), sourceContainerName))
+ if shared.PathExists(snapshotSymlink) {
+ err = os.Remove(snapshotSymlink)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerSnapshotRename(c container, newName string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "old_name": c.Name(),
+ "new_name": newName,
+ }
+ success := false
+
+ defer logAction(
+ "Renaming container snapshot",
+ "Renamed container snapshot",
+ "Failed to rename container snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ oldSnapshotMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, c.Name())
+ newSnapshotMntPoint := getSnapshotMountPoint(c.Project(), s.pool.Name, newName)
+
+ err = s.driver.VolumeSnapshotRename(c.Project(), c.Name(), newName,
+ driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+
+ // It might be, that the driver already renamed the path.
+ if shared.PathExists(oldSnapshotMntPoint) {
+ err = os.Rename(oldSnapshotMntPoint, newSnapshotMntPoint)
+ if err != nil {
+ return err
+ }
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerSnapshotStart(c container) (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "container": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Starting container snapshot",
+ "Started container snapshot",
+ "Failed to start container snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return false, errors.Wrap(err, "Mount storage pool")
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ ok, err := s.driver.VolumeMount(c.Project(), c.Name(), driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) ContainerSnapshotStop(c container) (bool, error) {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "container": c.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Stopping container snapshot",
+ "Stopped container snapshot",
+ "Failed to stop container snapshot",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return false, errors.Wrap(err, "Mount storage pool")
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ ok, err := s.driver.VolumeUmount(c.Project(), c.Name(), driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return ok, err
+ }
+
+ success = true
+
+ return ok, nil
+}
+
+func (s *Storage) ImageCreate(fingerprint string, tracker *ioprogress.ProgressTracker) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "fingerprint": fingerprint,
+ }
+ success := false
+
+ defer logAction(
+ "Creating image",
+ "Created image",
+ "Failed to create image",
+ &ctx, &success, &err)()
+
+ cleanupFunc := driver.LockImageCreate(s.pool.Name, fingerprint)
+ if cleanupFunc == nil {
+ return nil
+ }
+ defer cleanupFunc()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return errors.Wrap(err, "Mount storage pool")
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ // Don't create image if it already exists
+ if shared.PathExists(getImageMountPoint(s.pool.Name, fingerprint)) {
+ return nil
+ }
+
+ err = s.createImageDbPoolVolume(fingerprint)
+ if err != nil {
+ return errors.Wrap(err, "Create image db pool volume")
+ }
+
+ undo := true
+ defer func() {
+ if undo {
+ s.deleteImageDbPoolVolume(fingerprint)
+ }
+ }()
+
+ imageSourcePath := shared.VarPath("images", fingerprint)
+ imageVolumePath := getImageMountPoint(s.pool.Name, "")
+
+ if !shared.PathExists(imageVolumePath) {
+ err = os.MkdirAll(imageVolumePath, imagesDirMode)
+ if err != nil {
+ return errors.Wrap(err, "Create image mount point")
+ }
+ }
+
+ volumeName := fingerprint
+
+ if s.sType == storageTypeBtrfs {
+ volumeName = fmt.Sprintf("%s_tmp", fingerprint)
+ }
+
+ imageTargetPath := getImageMountPoint(s.pool.Name, volumeName)
+
+ err = s.driver.VolumeCreate("default", volumeName, driver.VolumeTypeImage)
+ if err != nil {
+ return errors.Wrap(err, "Create volume")
+ }
+
+ if s.sType == storageTypeZfs {
+ undo = false
+ success = true
+ return nil
+ }
+
+ if s.sType == storageTypeBtrfs {
+ defer func() {
+ s.driver.VolumeDelete("default", volumeName, false, driver.VolumeTypeImage)
+ }()
+ }
+
+ err = unpackImage(imageSourcePath, imageTargetPath, s.sType, s.s.OS.RunningInUserNS, tracker)
+ if err != nil {
+ return errors.Wrap(err, "Unpack image")
+ }
+
+ if s.sType == storageTypeBtrfs {
+ // Create read-only snapshot of the image volume
+ err = s.driver.VolumeSnapshotCreate("default", volumeName,
+ fingerprint, driver.VolumeTypeImageSnapshot)
+ if err != nil {
+ return errors.Wrap(err, "Create volume snapshot")
+ }
+ }
+
+ undo = false
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ImageDelete(fingerprint string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "fingerprint": fingerprint,
+ }
+ success := false
+
+ defer logAction(
+ "Deleting image",
+ "Deleted image",
+ "Failed to delete image",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if err != nil {
+ return err
+ }
+
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ err = s.deleteImageDbPoolVolume(fingerprint)
+ if err != nil {
+ return err
+ }
+
+ imageMntPoint := getImageMountPoint(s.pool.Name, fingerprint)
+
+ err = s.driver.VolumeDelete("default", fingerprint, false, driver.VolumeTypeImage)
+ if err != nil {
+ return err
+ }
+
+ // Now delete the mountpoint for the image:
+ // ${LXD_DIR}/images/<fingerprint>.
+ if shared.PathExists(imageMntPoint) {
+ err := os.RemoveAll(imageMntPoint)
+ if err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) StorageEntitySetQuota(volumeType int, size int64, data interface{}) error {
+ if !shared.IntInSlice(volumeType, supportedVolumeTypes) {
+ return fmt.Errorf("Invalid storage type")
+ }
+
+ var c container
+ var subvol string
+ var volType driver.VolumeType
+
+ project := "default"
+
+ switch volumeType {
+ case storagePoolVolumeTypeContainer:
+ c = data.(container)
+ subvol = c.Name()
+ volType = driver.VolumeTypeContainer
+ project = c.Project()
+ case storagePoolVolumeTypeCustom:
+ subvol = s.volume.Name
+ volType = driver.VolumeTypeCustom
+ }
+
+ return s.driver.VolumeSetQuota(project, subvol, size, s.s.OS.RunningInUserNS, volType)
+}
+
+func (s *Storage) ContainerBackupCreate(backup backup, source container) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "source": source.Name(),
+ }
+ success := false
+
+ defer logAction(
+ "Creating container backup",
+ "Created container backup",
+ "Failed to create container backup",
+ &ctx, &success, &err)()
+
+ // Start storage
+ ourStart, err := source.StorageStart()
+ if err != nil {
+ return err
+ }
+
+ if ourStart {
+ defer source.StorageStop()
+ }
+
+ // Create a temporary path for the backup
+ tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpPath)
+
+ var snapshots []string
+
+ if !backup.containerOnly {
+ var snaps []container
+
+ snaps, err = source.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range snaps {
+ snapshots = append(snapshots, shared.ExtractSnapshotName(snap.Name()))
+ }
+ }
+
+ err = s.driver.VolumeBackupCreate(tmpPath, source.Project(), source.Name(), snapshots, backup.optimizedStorage)
+ if err != nil {
+ return err
+ }
+
+ // Pack the backup
+ err = backupCreateTarball(s.s, tmpPath, backup)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
+ var err error
+ ctx := log.Ctx{
+ "driver": s.sTypeName,
+ "pool": s.pool.Name,
+ "backup": info,
+ }
+ success := false
+
+ defer logAction(
+ "Loading container backup",
+ "Loaded container backup",
+ "Failed to load container backup",
+ &ctx, &success, &err)()
+
+ ourMount, err := s.driver.StoragePoolMount()
+ if ourMount {
+ defer s.driver.StoragePoolUmount()
+ }
+
+ if info.HasBinaryFormat {
+ containerName, _, _ := containerGetParentAndSnapshotName(info.Name)
+ containerMntPoint := getContainerMountPoint("default", s.pool.Name, "")
+
+ /*
+ err := createContainerMountpoint(containerMntPoint, containerPath(info.Project, info.Name, false), info.Privileged)
+ if err != nil {
+ return err
+ }
+ */
+
+ var unpackDir string
+
+ unpackDir, err = ioutil.TempDir(containerMntPoint, containerName)
+ if err != nil {
+ return err
+ }
+ // TODO: Check whether this is OK when using ZFS regarding the remove-mount-order.
+ // Alternatively, have the callee clean up the directory.
+ defer os.RemoveAll(unpackDir)
+
+ err = os.Chmod(unpackDir, 0700)
+ if err != nil {
+ return err
+ }
+
+ // ${LXD_DIR}/storage-pools/<pool>/containers/<container_name>.XXX/.backup_unpack
+ unpackPath := fmt.Sprintf("%s/.backup_unpack", unpackDir)
+ err = os.MkdirAll(unpackPath, 0711)
+ if err != nil {
+ return err
+ }
+
+ // Prepare tar arguments
+ args := append(tarArgs, []string{
+ "-",
+ "--strip-components=1",
+ "-C", unpackPath, "backup",
+ }...)
+
+ // Extract container
+ data.Seek(0, 0)
+ err = shared.RunCommandWithFds(data, nil, "tar", args...)
+ if err != nil {
+ logger.Errorf("Failed to untar \"%s\" into \"%s\": %s", "backup", unpackPath, err)
+ return err
+ }
+
+ err = s.driver.VolumeBackupLoad(unpackDir, info.Project, info.Name,
+ info.Snapshots, info.Privileged, info.HasBinaryFormat)
+ if err != nil {
+ return err
+ }
+
+ _, err = s.driver.VolumeMount(info.Project, info.Name, driver.VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ success = true
+
+ return nil
+ }
+
+ containersPath := getContainerMountPoint("default", s.pool.Name, "")
+
+ if !shared.PathExists(containersPath) {
+ err = os.MkdirAll(containersPath, containersDirMode)
+ if err != nil {
+ return err
+ }
+ }
+
+ // create the main container
+ err = s.driver.VolumeCreate(info.Project, info.Name,
+ driver.VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ _, err = s.driver.VolumeMount(info.Project, info.Name, driver.VolumeTypeContainer)
+ if err != nil {
+ return err
+ }
+
+ containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, info.Name)
+
+ if s.sType != storageTypeZfs {
+ // Create the mountpoint for the container at:
+ // ${LXD_DIR}/containers/<name>
+ err = createContainerMountpoint(containerMntPoint,
+ containerPath(info.Project, info.Name, false),
+ info.Privileged)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Extract container
+ for _, snap := range info.Snapshots {
+ cur := fmt.Sprintf("backup/snapshots/%s", snap)
+
+ // Prepare tar arguments
+ args := append(tarArgs, []string{
+ "-",
+ "--recursive-unlink",
+ "--xattrs-include=*",
+ "--strip-components=3",
+ "-C", containerMntPoint, cur,
+ }...)
+
+ // Extract snapshots
+ data.Seek(0, 0)
+ err = shared.RunCommandWithFds(data, nil, "tar", args...)
+ if err != nil {
+ logger.Errorf("Failed to untar \"%s\" into \"%s\": %s", cur, containerMntPoint, err)
+ return err
+ }
+
+ // create snapshot
+ fullSnapshotName := fmt.Sprintf("%s/%s", info.Name, snap)
+
+ snapshotPath := getSnapshotMountPoint(info.Project, s.pool.Name, info.Name)
+ if !shared.PathExists(snapshotPath) {
+ err = os.MkdirAll(snapshotPath, containersDirMode)
+ if err != nil {
+ return err
+ }
+ }
+
+ snapshotMntPoint := getSnapshotMountPoint(info.Project, s.pool.Name, info.Name)
+ snapshotMntPointSymlink := shared.VarPath("snapshots",
+ projectPrefix(info.Project, info.Name))
+
+ err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPoint, snapshotMntPointSymlink)
+ if err != nil {
+ return err
+ }
+
+ err = s.driver.VolumeSnapshotCreate(info.Project, info.Name, fullSnapshotName,
+ driver.VolumeTypeContainerSnapshot)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Prepare tar arguments
+ args := append(tarArgs, []string{
+ "-",
+ "--strip-components=2",
+ "--xattrs-include=*",
+ "-C", containerMntPoint, "backup/container",
+ }...)
+
+ // Extract container
+ data.Seek(0, 0)
+ err = shared.RunCommandWithFds(data, nil, "tar", args...)
+ if err != nil {
+ logger.Errorf("Failed to untar \"backup/container\" into \"%s\": %s", containerMntPoint, err)
+ return err
+ }
+
+ success = true
+
+ return nil
+}
+
+func (s *Storage) MigrationType() migration.MigrationFSType {
+ switch s.sType {
+ case storageTypeBtrfs:
+ if !s.s.OS.RunningInUserNS {
+ return migration.MigrationFSType_BTRFS
+ }
+ case storageTypeZfs:
+ return migration.MigrationFSType_ZFS
+ }
+
+ return migration.MigrationFSType_RSYNC
+}
+
+func (s *Storage) PreservesInodes() bool {
+ switch s.sType {
+ case storageTypeBtrfs:
+ return !s.s.OS.RunningInUserNS
+ case storageTypeZfs:
+ return true
+ }
+
+ // storageTypeDir, storageTypeLvm, storageTypeCeph, storageTypeMock
+ return false
+}
+
+func (s *Storage) MigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
+ switch s.sType {
+ case storageTypeBtrfs:
+ if s.s.OS.RunningInUserNS {
+ return rsyncMigrationSource(args)
+ }
+
+ // Implement in the main package. Driver specific code needs to be exported
+ // and may not be part of the StorageDriver code. The reason for it being
+ // part of the main package is that it needs to be aware of containers, and
+ // reorganizing the container code will be a PITA.
+ return btrfsMigrationSource(args, s.pool)
+ case storageTypeDir:
+ return rsyncMigrationSource(args)
+ case storageTypeZfs:
+ return zfsMigrationSource(s.s, s.pool, args)
+ }
+
+ return nil, nil
+}
+
+func (s *Storage) MigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
+ switch s.sType {
+ case storageTypeDir:
+ return rsyncMigrationSink(conn, op, args)
+ case storageTypeBtrfs:
+ if s.s.OS.RunningInUserNS {
+ return rsyncMigrationSink(conn, op, args)
+ }
+
+ return btrfsMigrationSink(s.pool, conn, op, args)
+ case storageTypeZfs:
+ return zfsMigrationSink(s.pool, s.volume, conn, op, args)
+ }
+
+ return nil
+}
+
+func (s *Storage) StorageMigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
+ return rsyncStorageMigrationSource(args)
+}
+
+func (s *Storage) StorageMigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
+ return rsyncStorageMigrationSink(conn, op, args)
+}
+
+
+func (s *Storage) createImageDbPoolVolume(fingerprint string) error {
+ // Fill in any default volume config.
+ volumeConfig := map[string]string{}
+ err := storageVolumeFillDefault(fingerprint, volumeConfig, s.pool)
+ if err != nil {
+ return err
+ }
+
+ // Create a db entry for the storage volume of the image.
+ _, err = s.s.Cluster.StoragePoolVolumeCreate("default", fingerprint, "", storagePoolVolumeTypeImage, false, s.poolID, volumeConfig)
+ if err != nil {
+ // Try to delete the db entry on error.
+ s.deleteImageDbPoolVolume(fingerprint)
+ return err
+ }
+
+ return nil
+}
+
+func (s *Storage) deleteImageDbPoolVolume(fingerprint string) error {
+ err := s.s.Cluster.StoragePoolVolumeDelete("default", fingerprint, storagePoolVolumeTypeImage, s.poolID)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+type StorageDriver interface {
+ Init() error
+ SharedInit(s *state.State, pool *api.StoragePool, poolID int64, volume *api.StorageVolume)
+ GetVersion() string
+
+ StoragePoolCheck() error
+ StoragePoolCreate() error
+ StoragePoolDelete() error
+ StoragePoolMount() (bool, error)
+ StoragePoolUmount() (bool, error)
+ StoragePoolResources() (*api.ResourcesStoragePool, error)
+ StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error
+
+ VolumeCreate(project string, volumeName string, volumeType driver.VolumeType) error
+ VolumeCopy(project, source string, target string, snapshots []string, volumeType driver.VolumeType) error
+ VolumeDelete(project string, volumeName string, recursive bool, volumeType driver.VolumeType) error
+ VolumeRename(project string, oldName string, newName string, snapshots []string, volumeType driver.VolumeType) error
+ VolumeMount(project string, name string, volumeType driver.VolumeType) (bool, error)
+ VolumeUmount(project string, name string, volumeType driver.VolumeType) (bool, error)
+ VolumeGetUsage(project, name, path string) (int64, error)
+ VolumeSetQuota(project, name string, size int64, userns bool, volumeType driver.VolumeType) error
+ VolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error
+ VolumeReady(project string, name string) bool
+ VolumePrepareRestore(sourceName string, targetName string, targetSnapshots []string, f func() error) error
+ // TODO: remove in favour of VolumeSnapshotRestore, or remove VolumeSnapshotRestore in favour of this
+ VolumeRestore(project string, sourceName string, targetName string, volumeType driver.VolumeType) error
+ VolumeSnapshotCreate(project string, source string, target string, volumeType driver.VolumeType) error
+ VolumeSnapshotCopy(project, source string, target string, volumeType driver.VolumeType) error
+ VolumeSnapshotDelete(project string, volumeName string, recursive bool, volumeType driver.VolumeType) error
+ VolumeSnapshotRestore(project string, sourceName string, targetName string, volumeType driver.VolumeType) error
+ VolumeSnapshotRename(project string, oldName string, newName string, volumeType driver.VolumeType) error
+ VolumeBackupCreate(path string, project string, source string, snapshots []string, optimized bool) error
+ VolumeBackupLoad(backupDir string, project string, containerName string, snapshots []string, privileged bool, optimized bool) error
+}
+
// The storage interface defines the functions needed to implement a storage
// backend for a given storage driver.
type storage interface {
@@ -243,19 +2618,9 @@ func storageCoreInit(driver string) (storage, error) {
switch sType {
case storageTypeBtrfs:
- btrfs := storageBtrfs{}
- err = btrfs.StorageCoreInit()
- if err != nil {
- return nil, err
- }
- return &btrfs, nil
+ return storageCoreInit2(driver)
case storageTypeDir:
- dir := storageDir{}
- err = dir.StorageCoreInit()
- if err != nil {
- return nil, err
- }
- return &dir, nil
+ return storageCoreInit2(driver)
case storageTypeCeph:
ceph := storageCeph{}
err = ceph.StorageCoreInit()
@@ -278,17 +2643,36 @@ func storageCoreInit(driver string) (storage, error) {
}
return &mock, nil
case storageTypeZfs:
- zfs := storageZfs{}
- err = zfs.StorageCoreInit()
- if err != nil {
- return nil, err
- }
- return &zfs, nil
+ return storageCoreInit2(driver)
}
return nil, fmt.Errorf("invalid storage type")
}
+func storageCoreInit2(storageDriver string) (storage, error) {
+ sType, err := storageStringToType(storageDriver)
+ if err != nil {
+ return nil, err
+ }
+
+ st := Storage{}
+
+ switch sType {
+ case storageTypeDir:
+ st.driver = &driver.Dir{}
+ case storageTypeBtrfs:
+ st.driver = &driver.Btrfs{}
+ case storageTypeZfs:
+ st.driver = &driver.Zfs{}
+ default:
+ return nil, fmt.Errorf("invalid storage type")
+ }
+
+ st.driver.Init()
+
+ return &st, nil
+}
+
func storageInit(s *state.State, project, poolName, volumeName string, volumeType int) (storage, error) {
// Load the storage pool.
poolID, pool, err := s.Cluster.StoragePoolGet(poolName)
@@ -305,9 +2689,8 @@ func storageInit(s *state.State, project, poolName, volumeName string, volumeTyp
// Load the storage volume.
volume := &api.StorageVolume{}
- volumeID := int64(-1)
if volumeName != "" {
- volumeID, volume, err = s.Cluster.StoragePoolNodeVolumeGetTypeByProject(project, volumeName, volumeType, poolID)
+ _, volume, err = s.Cluster.StoragePoolNodeVolumeGetTypeByProject(project, volumeName, volumeType, poolID)
if err != nil {
return nil, err
}
@@ -320,28 +2703,9 @@ func storageInit(s *state.State, project, poolName, volumeName string, volumeTyp
switch sType {
case storageTypeBtrfs:
- btrfs := storageBtrfs{}
- btrfs.poolID = poolID
- btrfs.pool = pool
- btrfs.volume = volume
- btrfs.s = s
- err = btrfs.StoragePoolInit()
- if err != nil {
- return nil, err
- }
- return &btrfs, nil
+ return storageInit2(s, project, poolName, volumeName, volumeType)
case storageTypeDir:
- dir := storageDir{}
- dir.poolID = poolID
- dir.pool = pool
- dir.volume = volume
- dir.volumeID = volumeID
- dir.s = s
- err = dir.StoragePoolInit()
- if err != nil {
- return nil, err
- }
- return &dir, nil
+ return storageInit2(s, project, poolName, volumeName, volumeType)
case storageTypeCeph:
ceph := storageCeph{}
ceph.poolID = poolID
@@ -376,19 +2740,68 @@ func storageInit(s *state.State, project, poolName, volumeName string, volumeTyp
}
return &mock, nil
case storageTypeZfs:
- zfs := storageZfs{}
- zfs.poolID = poolID
- zfs.pool = pool
- zfs.volume = volume
- zfs.s = s
- err = zfs.StoragePoolInit()
+ return storageInit2(s, project, poolName, volumeName, volumeType)
+ }
+
+ return nil, fmt.Errorf("invalid storage type")
+}
+
+func storageInit2(s *state.State, project, poolName, volumeName string, volumeType int) (storage, error) {
+ // Load the storage pool.
+ poolID, pool, err := s.Cluster.StoragePoolGet(poolName)
+ if err != nil {
+ return nil, errors.Wrapf(err, "Load storage pool %q", poolName)
+ }
+
+ if pool.Driver == "" {
+ // This shouldn't actually be possible but better safe than
+ // sorry.
+ return nil, fmt.Errorf("no storage driver was provided")
+ }
+
+ // Load the storage volume.
+ volume := &api.StorageVolume{}
+ volumeID := int64(-1)
+ if volumeName != "" {
+ volumeID, volume, err = s.Cluster.StoragePoolNodeVolumeGetTypeByProject(project, volumeName, volumeType, poolID)
if err != nil {
return nil, err
}
- return &zfs, nil
}
- return nil, fmt.Errorf("invalid storage type")
+ sType, err := storageStringToType(pool.Driver)
+ if err != nil {
+ return nil, err
+ }
+
+ st := Storage{}
+ st.poolID = poolID
+ st.pool = pool
+ st.volumeID = volumeID
+ st.volume = volume
+ st.s = s
+ st.sType = sType
+ st.sTypeName = pool.Driver
+
+ switch sType {
+ case storageTypeDir:
+ st.driver = &driver.Dir{}
+ case storageTypeBtrfs:
+ st.driver = &driver.Btrfs{}
+ case storageTypeZfs:
+ st.driver = &driver.Zfs{}
+ default:
+ return nil, fmt.Errorf("invalid storage type")
+ }
+
+ st.driver.SharedInit(s, pool, poolID, volume)
+
+ err = st.driver.Init()
+ if err != nil {
+ return nil, err
+ }
+
+ return &st, nil
}
func storagePoolInit(s *state.State, poolName string) (storage, error) {
@@ -505,7 +2918,7 @@ func storagePoolVolumeAttachInit(s *state.State, poolName string, volumeName str
var err error
if st.GetStorageType() == storageTypeZfs {
- err = lastIdmap.UnshiftRootfs(remapPath, zfsIdmapSetSkipper)
+ err = lastIdmap.UnshiftRootfs(remapPath, driver.ZfsIdmapSetSkipper)
} else {
err = lastIdmap.UnshiftRootfs(remapPath, nil)
}
@@ -521,7 +2934,7 @@ func storagePoolVolumeAttachInit(s *state.State, poolName string, volumeName str
var err error
if st.GetStorageType() == storageTypeZfs {
- err = nextIdmap.ShiftRootfs(remapPath, zfsIdmapSetSkipper)
+ err = nextIdmap.ShiftRootfs(remapPath, driver.ZfsIdmapSetSkipper)
} else {
err = nextIdmap.ShiftRootfs(remapPath, nil)
}
From fa6c22745bb878c924099ca83397b8e50e952e13 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 16:51:53 +0200
Subject: [PATCH 13/15] lxd: Remove old btrfs storage code
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage_btrfs.go | 3195 ------------------------------------------
1 file changed, 3195 deletions(-)
delete mode 100644 lxd/storage_btrfs.go
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
deleted file mode 100644
index 7fcab12506..0000000000
--- a/lxd/storage_btrfs.go
+++ /dev/null
@@ -1,3195 +0,0 @@
-package main
-
-import (
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "os/exec"
- "path"
- "path/filepath"
- "sort"
- "strconv"
- "strings"
- "syscall"
-
- "github.com/gorilla/websocket"
- "github.com/pkg/errors"
-
- "github.com/lxc/lxd/lxd/db"
- "github.com/lxc/lxd/lxd/migration"
- driver "github.com/lxc/lxd/lxd/storage"
- "github.com/lxc/lxd/lxd/util"
- "github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/api"
- "github.com/lxc/lxd/shared/ioprogress"
- "github.com/lxc/lxd/shared/logger"
-)
-
-type storageBtrfs struct {
- remount uintptr
- storageShared
-}
-
-var btrfsVersion = ""
-
-func (s *storageBtrfs) getBtrfsMountOptions() string {
- if s.pool.Config["btrfs.mount_options"] != "" {
- return s.pool.Config["btrfs.mount_options"]
- }
-
- return "user_subvol_rm_allowed"
-}
-
-func (s *storageBtrfs) setBtrfsMountOptions(mountOptions string) {
- s.pool.Config["btrfs.mount_options"] = mountOptions
-}
-
-// ${LXD_DIR}/storage-pools/<pool>/containers
-func (s *storageBtrfs) getContainerSubvolumePath(poolName string) string {
- return shared.VarPath("storage-pools", poolName, "containers")
-}
-
-// ${LXD_DIR}/storage-pools/<pool>/containers-snapshots
-func getSnapshotSubvolumePath(project, poolName string, containerName string) string {
- return shared.VarPath("storage-pools", poolName, "containers-snapshots", projectPrefix(project, containerName))
-}
-
-// ${LXD_DIR}/storage-pools/<pool>/images
-func (s *storageBtrfs) getImageSubvolumePath(poolName string) string {
- return shared.VarPath("storage-pools", poolName, "images")
-}
-
-// ${LXD_DIR}/storage-pools/<pool>/custom
-func (s *storageBtrfs) getCustomSubvolumePath(poolName string) string {
- return shared.VarPath("storage-pools", poolName, "custom")
-}
-
-// ${LXD_DIR}/storage-pools/<pool>/custom-snapshots
-func (s *storageBtrfs) getCustomSnapshotSubvolumePath(poolName string) string {
- return shared.VarPath("storage-pools", poolName, "custom-snapshots")
-}
-
-func (s *storageBtrfs) StorageCoreInit() error {
- s.sType = storageTypeBtrfs
- typeName, err := storageTypeToString(s.sType)
- if err != nil {
- return err
- }
- s.sTypeName = typeName
-
- if btrfsVersion != "" {
- s.sTypeVersion = btrfsVersion
- return nil
- }
-
- out, err := exec.LookPath("btrfs")
- if err != nil || len(out) == 0 {
- return fmt.Errorf("The 'btrfs' tool isn't available")
- }
-
- output, err := shared.RunCommand("btrfs", "version")
- if err != nil {
- return fmt.Errorf("The 'btrfs' tool isn't working properly")
- }
-
- count, err := fmt.Sscanf(strings.SplitN(output, " ", 2)[1], "v%s\n", &s.sTypeVersion)
- if err != nil || count != 1 {
- return fmt.Errorf("The 'btrfs' tool isn't working properly")
- }
-
- btrfsVersion = s.sTypeVersion
-
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolInit() error {
- err := s.StorageCoreInit()
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolCheck() error {
- // FIXEM(brauner): Think of something smart or useful (And then think
- // again if it is worth implementing it. :)).
- logger.Debugf("Checking BTRFS storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolCreate() error {
- logger.Infof("Creating BTRFS storage pool \"%s\"", s.pool.Name)
- s.pool.Config["volatile.initial_source"] = s.pool.Config["source"]
-
- isBlockDev := false
-
- source := s.pool.Config["source"]
- if strings.HasPrefix(source, "/") {
- source = shared.HostPath(s.pool.Config["source"])
- }
-
- defaultSource := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", s.pool.Name))
- if source == "" || source == defaultSource {
- source = defaultSource
- s.pool.Config["source"] = source
-
- f, err := os.Create(source)
- if err != nil {
- return fmt.Errorf("Failed to open %s: %s", source, err)
- }
- defer f.Close()
-
- err = f.Chmod(0600)
- if err != nil {
- return fmt.Errorf("Failed to chmod %s: %s", source, err)
- }
-
- size, err := shared.ParseByteSizeString(s.pool.Config["size"])
- if err != nil {
- return err
- }
- err = f.Truncate(size)
- if err != nil {
- return fmt.Errorf("Failed to create sparse file %s: %s", source, err)
- }
-
- output, err := makeFSType(source, "btrfs", &mkfsOptions{label: s.pool.Name})
- if err != nil {
- return fmt.Errorf("Failed to create the BTRFS pool: %s", output)
- }
- } else {
- // Unset size property since it doesn't make sense.
- s.pool.Config["size"] = ""
-
- if filepath.IsAbs(source) {
- isBlockDev = shared.IsBlockdevPath(source)
- if isBlockDev {
- output, err := makeFSType(source, "btrfs", &mkfsOptions{label: s.pool.Name})
- if err != nil {
- return fmt.Errorf("Failed to create the BTRFS pool: %s", output)
- }
- } else {
- if isBtrfsSubVolume(source) {
- subvols, err := btrfsSubVolumesGet(source)
- if err != nil {
- return fmt.Errorf("Could not determine if existing BTRFS subvolume ist empty: %s", err)
- }
- if len(subvols) > 0 {
- return fmt.Errorf("Requested BTRFS subvolume exists but is not empty")
- }
- } else {
- cleanSource := filepath.Clean(source)
- lxdDir := shared.VarPath()
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- if shared.PathExists(source) && !isOnBtrfs(source) {
- return fmt.Errorf("Existing path is neither a BTRFS subvolume nor does it reside on a BTRFS filesystem")
- } else if strings.HasPrefix(cleanSource, lxdDir) {
- if cleanSource != poolMntPoint {
- return fmt.Errorf("BTRFS subvolumes requests in LXD directory \"%s\" are only valid under \"%s\"\n(e.g. source=%s)", shared.VarPath(), shared.VarPath("storage-pools"), poolMntPoint)
- } else if s.s.OS.BackingFS != "btrfs" {
- return fmt.Errorf("Creation of BTRFS subvolume requested but \"%s\" does not reside on BTRFS filesystem", source)
- }
- }
-
- err := btrfsSubVolumeCreate(source)
- if err != nil {
- return err
- }
- }
- }
- } else {
- return fmt.Errorf("Invalid \"source\" property")
- }
- }
-
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- if !shared.PathExists(poolMntPoint) {
- err := os.MkdirAll(poolMntPoint, storagePoolsDirMode)
- if err != nil {
- return err
- }
- }
-
- var err1 error
- var devUUID string
- mountFlags, mountOptions := lxdResolveMountoptions(s.getBtrfsMountOptions())
- mountFlags |= s.remount
- if isBlockDev && filepath.IsAbs(source) {
- devUUID, _ = shared.LookupUUIDByBlockDevPath(source)
- // The symlink might not have been created even with the delay
- // we granted it above. So try to call btrfs filesystem show and
- // parse it out. (I __hate__ this!)
- if devUUID == "" {
- logger.Warnf("Failed to detect UUID by looking at /dev/disk/by-uuid")
- devUUID, err1 = s.btrfsLookupFsUUID(source)
- if err1 != nil {
- logger.Errorf("Failed to detect UUID by parsing filesystem info")
- return err1
- }
- }
- s.pool.Config["source"] = devUUID
-
- // If the symlink in /dev/disk/by-uuid hasn't been created yet
- // aka we only detected it by parsing btrfs filesystem show, we
- // cannot call StoragePoolMount() since it will try to do the
- // reverse operation. So instead we shamelessly mount using the
- // block device path at the time of pool creation.
- err1 = syscall.Mount(source, poolMntPoint, "btrfs", mountFlags, mountOptions)
- } else {
- _, err1 = s.StoragePoolMount()
- }
- if err1 != nil {
- return err1
- }
-
- // Create default subvolumes.
- dummyDir := getContainerMountPoint("default", s.pool.Name, "")
- err := btrfsSubVolumeCreate(dummyDir)
- if err != nil {
- return fmt.Errorf("Could not create btrfs subvolume: %s", dummyDir)
- }
-
- dummyDir = getSnapshotMountPoint("default", s.pool.Name, "")
- err = btrfsSubVolumeCreate(dummyDir)
- if err != nil {
- return fmt.Errorf("Could not create btrfs subvolume: %s", dummyDir)
- }
-
- dummyDir = getImageMountPoint(s.pool.Name, "")
- err = btrfsSubVolumeCreate(dummyDir)
- if err != nil {
- return fmt.Errorf("Could not create btrfs subvolume: %s", dummyDir)
- }
-
- dummyDir = getStoragePoolVolumeMountPoint(s.pool.Name, "")
- err = btrfsSubVolumeCreate(dummyDir)
- if err != nil {
- return fmt.Errorf("Could not create btrfs subvolume: %s", dummyDir)
- }
-
- dummyDir = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, "")
- err = btrfsSubVolumeCreate(dummyDir)
- if err != nil {
- return fmt.Errorf("Could not create btrfs subvolume: %s", dummyDir)
- }
-
- err = s.StoragePoolCheck()
- if err != nil {
- return err
- }
-
- logger.Infof("Created BTRFS storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolDelete() error {
- logger.Infof("Deleting BTRFS storage pool \"%s\"", s.pool.Name)
-
- source := s.pool.Config["source"]
- if strings.HasPrefix(source, "/") {
- source = shared.HostPath(s.pool.Config["source"])
- }
-
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- // Delete default subvolumes.
- dummyDir := getContainerMountPoint("default", s.pool.Name, "")
- btrfsSubVolumesDelete(dummyDir)
-
- dummyDir = getSnapshotMountPoint("default", s.pool.Name, "")
- btrfsSubVolumesDelete(dummyDir)
-
- dummyDir = getImageMountPoint(s.pool.Name, "")
- btrfsSubVolumesDelete(dummyDir)
-
- dummyDir = getStoragePoolVolumeMountPoint(s.pool.Name, "")
- btrfsSubVolumesDelete(dummyDir)
-
- _, err := s.StoragePoolUmount()
- if err != nil {
- return err
- }
-
- // This is a UUID. Check whether we can find the block device.
- if !filepath.IsAbs(source) {
- // Try to lookup the disk device by UUID but don't fail. If we
- // don't find one this might just mean we have been given the
- // UUID of a subvolume.
- byUUID := fmt.Sprintf("/dev/disk/by-uuid/%s", source)
- diskPath, err := os.Readlink(byUUID)
- msg := ""
- if err == nil {
- msg = fmt.Sprintf("Removing disk device %s with UUID: %s.", diskPath, source)
- } else {
- msg = fmt.Sprintf("Failed to lookup disk device with UUID: %s: %s.", source, err)
- }
- logger.Debugf(msg)
- } else {
- var err error
- cleanSource := filepath.Clean(source)
- sourcePath := shared.VarPath("disks", s.pool.Name)
- loopFilePath := sourcePath + ".img"
- if cleanSource == loopFilePath {
- // This is a loop file so simply remove it.
- err = os.Remove(source)
- } else {
- if !isBtrfsFilesystem(source) && isBtrfsSubVolume(source) {
- err = btrfsSubVolumesDelete(source)
- }
- }
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- }
-
- // Remove the mountpoint for the storage pool.
- err = os.RemoveAll(getStoragePoolMountPoint(s.pool.Name))
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- logger.Infof("Deleted BTRFS storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolMount() (bool, error) {
- logger.Debugf("Mounting BTRFS storage pool \"%s\"", s.pool.Name)
-
- source := s.pool.Config["source"]
- if strings.HasPrefix(source, "/") {
- source = shared.HostPath(s.pool.Config["source"])
- }
-
- if source == "" {
- return false, fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
-
- poolMountLockID := getPoolMountLockID(s.pool.Name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolMountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in mounting the storage pool.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[poolMountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- removeLockFromMap := func() {
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolMountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, poolMountLockID)
- }
- lxdStorageMapLock.Unlock()
- }
- defer removeLockFromMap()
-
- // Check whether the mount poolMntPoint exits.
- if !shared.PathExists(poolMntPoint) {
- err := os.MkdirAll(poolMntPoint, storagePoolsDirMode)
- if err != nil {
- return false, err
- }
- }
-
- if shared.IsMountPoint(poolMntPoint) && (s.remount&syscall.MS_REMOUNT) == 0 {
- return false, nil
- }
-
- mountFlags, mountOptions := lxdResolveMountoptions(s.getBtrfsMountOptions())
- mountSource := source
- isBlockDev := shared.IsBlockdevPath(source)
- if filepath.IsAbs(source) {
- cleanSource := filepath.Clean(source)
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- loopFilePath := shared.VarPath("disks", s.pool.Name+".img")
- if !isBlockDev && cleanSource == loopFilePath {
- // If source == "${LXD_DIR}"/disks/{pool_name} it is a
- // loop file we're dealing with.
- //
- // Since we mount the loop device LO_FLAGS_AUTOCLEAR is
- // fine since the loop device will be kept around for as
- // long as the mount exists.
- loopF, loopErr := driver.PrepareLoopDev(source, driver.LoFlagsAutoclear)
- if loopErr != nil {
- return false, loopErr
- }
- mountSource = loopF.Name()
- defer loopF.Close()
- } else if !isBlockDev && cleanSource != poolMntPoint {
- mountSource = source
- mountFlags |= syscall.MS_BIND
- } else if !isBlockDev && cleanSource == poolMntPoint && s.s.OS.BackingFS == "btrfs" {
- return false, nil
- }
- // User is using block device path.
- } else {
- // Try to lookup the disk device by UUID but don't fail. If we
- // don't find one this might just mean we have been given the
- // UUID of a subvolume.
- byUUID := fmt.Sprintf("/dev/disk/by-uuid/%s", source)
- diskPath, err := os.Readlink(byUUID)
- if err == nil {
- mountSource = fmt.Sprintf("/dev/%s", strings.Trim(diskPath, "../../"))
- } else {
- // We have very likely been given a subvolume UUID. In
- // this case we should simply assume that the user has
- // mounted the parent of the subvolume or the subvolume
- // itself. Otherwise this becomes a really messy
- // detection task.
- return false, nil
- }
- }
-
- mountFlags |= s.remount
- err := syscall.Mount(mountSource, poolMntPoint, "btrfs", mountFlags, mountOptions)
- if err != nil {
- logger.Errorf("Failed to mount BTRFS storage pool \"%s\" onto \"%s\" with mountoptions \"%s\": %s", mountSource, poolMntPoint, mountOptions, err)
- return false, err
- }
-
- logger.Debugf("Mounted BTRFS storage pool \"%s\"", s.pool.Name)
- return true, nil
-}
-
-func (s *storageBtrfs) StoragePoolUmount() (bool, error) {
- logger.Debugf("Unmounting BTRFS storage pool \"%s\"", s.pool.Name)
-
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
-
- poolUmountLockID := getPoolUmountLockID(s.pool.Name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolUmountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in unmounting the storage pool.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[poolUmountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- removeLockFromMap := func() {
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolUmountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, poolUmountLockID)
- }
- lxdStorageMapLock.Unlock()
- }
-
- defer removeLockFromMap()
-
- if shared.IsMountPoint(poolMntPoint) {
- err := syscall.Unmount(poolMntPoint, 0)
- if err != nil {
- return false, err
- }
- }
-
- logger.Debugf("Unmounted BTRFS storage pool \"%s\"", s.pool.Name)
- return true, nil
-}
-
-func (s *storageBtrfs) StoragePoolUpdate(writable *api.StoragePoolPut,
- changedConfig []string) error {
- logger.Infof(`Updating BTRFS storage pool "%s"`, s.pool.Name)
-
- changeable := changeableStoragePoolProperties["btrfs"]
- unchangeable := []string{}
- for _, change := range changedConfig {
- if !shared.StringInSlice(change, changeable) {
- unchangeable = append(unchangeable, change)
- }
- }
-
- if len(unchangeable) > 0 {
- return updateStoragePoolError(unchangeable, "btrfs")
- }
-
- // "rsync.bwlimit" requires no on-disk modifications.
-
- if shared.StringInSlice("btrfs.mount_options", changedConfig) {
- s.setBtrfsMountOptions(writable.Config["btrfs.mount_options"])
- s.remount |= syscall.MS_REMOUNT
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
- }
-
- logger.Infof(`Updated BTRFS storage pool "%s"`, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) GetContainerPoolInfo() (int64, string, string) {
- return s.poolID, s.pool.Name, s.pool.Name
-}
-
-// Functions dealing with storage volumes.
-func (s *storageBtrfs) StoragePoolVolumeCreate() error {
- logger.Infof("Creating BTRFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- isSnapshot := shared.IsSnapshot(s.volume.Name)
-
- // Create subvolume path on the storage pool.
- var customSubvolumePath string
-
- if isSnapshot {
- customSubvolumePath = s.getCustomSnapshotSubvolumePath(s.pool.Name)
- } else {
- customSubvolumePath = s.getCustomSubvolumePath(s.pool.Name)
- }
-
- if !shared.PathExists(customSubvolumePath) {
- err := os.MkdirAll(customSubvolumePath, 0700)
- if err != nil {
- return err
- }
- }
-
- // Create subvolume.
- var customSubvolumeName string
-
- if isSnapshot {
- customSubvolumeName = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- } else {
- customSubvolumeName = getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- }
-
- err = btrfsSubVolumeCreate(customSubvolumeName)
- if err != nil {
- return err
- }
-
- // apply quota
- if s.volume.Config["size"] != "" {
- size, err := shared.ParseByteSizeString(s.volume.Config["size"])
- if err != nil {
- return err
- }
-
- err = s.StorageEntitySetQuota(storagePoolVolumeTypeCustom, size, nil)
- if err != nil {
- return err
- }
- }
-
- logger.Infof("Created BTRFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolVolumeDelete() error {
- logger.Infof("Deleting BTRFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Delete subvolume.
- customSubvolumeName := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- if shared.PathExists(customSubvolumeName) && isBtrfsSubVolume(customSubvolumeName) {
- err = btrfsSubVolumesDelete(customSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- // Delete the mountpoint.
- if shared.PathExists(customSubvolumeName) {
- err = os.Remove(customSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- err = s.s.Cluster.StoragePoolVolumeDelete(
- "default",
- s.volume.Name,
- storagePoolVolumeTypeCustom,
- s.poolID)
- if err != nil {
- logger.Errorf(`Failed to delete database entry for BTRFS storage volume "%s" on storage pool "%s"`, s.volume.Name, s.pool.Name)
- }
-
- logger.Infof("Deleted BTRFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolVolumeMount() (bool, error) {
- logger.Debugf("Mounting BTRFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Mounted BTRFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return true, nil
-}
-
-func (s *storageBtrfs) StoragePoolVolumeUmount() (bool, error) {
- return true, nil
-}
-
-func (s *storageBtrfs) StoragePoolVolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error {
- if writable.Restore != "" {
- logger.Debugf(`Restoring BTRFS storage volume "%s" from snapshot "%s"`,
- s.volume.Name, writable.Restore)
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Create a backup so we can revert.
- targetVolumeSubvolumeName := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- backupTargetVolumeSubvolumeName := fmt.Sprintf("%s.tmp", targetVolumeSubvolumeName)
- err = os.Rename(targetVolumeSubvolumeName, backupTargetVolumeSubvolumeName)
- if err != nil {
- return err
- }
- undo := true
- defer func() {
- if undo {
- os.Rename(backupTargetVolumeSubvolumeName, targetVolumeSubvolumeName)
- }
- }()
-
- sourceVolumeSubvolumeName := getStoragePoolVolumeSnapshotMountPoint(
- s.pool.Name, fmt.Sprintf("%s/%s", s.volume.Name, writable.Restore))
- err = s.btrfsPoolVolumesSnapshot(sourceVolumeSubvolumeName,
- targetVolumeSubvolumeName, false, true)
- if err != nil {
- return err
- }
-
- undo = false
- err = btrfsSubVolumesDelete(backupTargetVolumeSubvolumeName)
- if err != nil {
- return err
- }
-
- logger.Debugf(`Restored BTRFS storage volume "%s" from snapshot "%s"`,
- s.volume.Name, writable.Restore)
- return nil
- }
-
- logger.Infof(`Updating BTRFS storage volume "%s"`, s.volume.Name)
-
- changeable := changeableStoragePoolVolumeProperties["btrfs"]
- unchangeable := []string{}
- for _, change := range changedConfig {
- if !shared.StringInSlice(change, changeable) {
- unchangeable = append(unchangeable, change)
- }
- }
-
- if len(unchangeable) > 0 {
- return updateStoragePoolVolumeError(unchangeable, "btrfs")
- }
-
- if shared.StringInSlice("size", changedConfig) {
- if s.volume.Type != storagePoolVolumeTypeNameCustom {
- return updateStoragePoolVolumeError([]string{"size"}, "btrfs")
- }
-
- if s.volume.Config["size"] != writable.Config["size"] {
- size, err := shared.ParseByteSizeString(writable.Config["size"])
- if err != nil {
- return err
- }
-
- err = s.StorageEntitySetQuota(storagePoolVolumeTypeCustom, size, nil)
- if err != nil {
- return err
- }
- }
- }
-
- logger.Infof(`Updated BTRFS storage volume "%s"`, s.volume.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolVolumeRename(newName string) error {
- logger.Infof(`Renaming BTRFS storage volume on storage pool "%s" from "%s" to "%s`,
- s.pool.Name, s.volume.Name, newName)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- usedBy, err := storagePoolVolumeUsedByContainersGet(s.s, "default", s.volume.Name, storagePoolVolumeTypeNameCustom)
- if err != nil {
- return err
- }
- if len(usedBy) > 0 {
- return fmt.Errorf(`BTRFS storage volume "%s" on storage pool "%s" is attached to containers`,
- s.volume.Name, s.pool.Name)
- }
-
- oldPath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- newPath := getStoragePoolVolumeMountPoint(s.pool.Name, newName)
- err = os.Rename(oldPath, newPath)
- if err != nil {
- return err
- }
-
- logger.Infof(`Renamed BTRFS storage volume on storage pool "%s" from "%s" to "%s`,
- s.pool.Name, s.volume.Name, newName)
-
- err = s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, newName,
- storagePoolVolumeTypeCustom, s.poolID)
- if err != nil {
- return err
- }
-
- // Get volumes attached to source storage volume
- volumes, err := s.s.Cluster.StoragePoolVolumeSnapshotsGetType(s.volume.Name,
- storagePoolVolumeTypeCustom, s.poolID)
- if err != nil {
- return err
- }
-
- for _, vol := range volumes {
- _, snapshotName, _ := containerGetParentAndSnapshotName(vol)
- oldVolumeName := fmt.Sprintf("%s%s%s", s.volume.Name, shared.SnapshotDelimiter, snapshotName)
- newVolumeName := fmt.Sprintf("%s%s%s", newName, shared.SnapshotDelimiter, snapshotName)
-
- // Rename volume snapshots
- oldPath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, oldVolumeName)
- newPath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, newVolumeName)
- err = os.Rename(oldPath, newPath)
- if err != nil {
- return err
- }
-
- err = s.s.Cluster.StoragePoolVolumeRename("default", oldVolumeName, newVolumeName,
- storagePoolVolumeTypeCustom, s.poolID)
- if err != nil {
- return nil
- }
- }
-
- return nil
-}
-
-// Functions dealing with container storage.
-func (s *storageBtrfs) ContainerStorageReady(container container) bool {
- containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- return isBtrfsSubVolume(containerMntPoint)
-}
-
-func (s *storageBtrfs) doContainerCreate(project, name string, privileged bool) error {
- logger.Debugf("Creating empty BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // We can only create the btrfs subvolume under the mounted storage
- // pool. The on-disk layout for containers on a btrfs storage pool will
- // thus be
- // ${LXD_DIR}/storage-pools/<pool>/containers/. The btrfs tool will
- // complain if the intermediate path does not exist, so create it if it
- // doesn't already.
- containerSubvolumePath := s.getContainerSubvolumePath(s.pool.Name)
- if !shared.PathExists(containerSubvolumePath) {
- err := os.MkdirAll(containerSubvolumePath, containersDirMode)
- if err != nil {
- return err
- }
- }
-
- // Create empty subvolume for container.
- containerSubvolumeName := getContainerMountPoint(project, s.pool.Name, name)
- err = btrfsSubVolumeCreate(containerSubvolumeName)
- if err != nil {
- return err
- }
-
- // Create the mountpoint for the container at:
- // ${LXD_DIR}/containers/<name>
- err = createContainerMountpoint(containerSubvolumeName, shared.VarPath("containers", projectPrefix(project, name)), privileged)
- if err != nil {
- return err
- }
-
- logger.Debugf("Created empty BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) ContainerCreate(container container) error {
- err := s.doContainerCreate(container.Project(), container.Name(), container.IsPrivileged())
- if err != nil {
- return err
- }
-
- return container.TemplateApply("create")
-}
-
-// And this function is why I started hating on btrfs...
-func (s *storageBtrfs) ContainerCreateFromImage(container container, fingerprint string, tracker *ioprogress.ProgressTracker) error {
- logger.Debugf("Creating BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return errors.Wrap(err, "Failed to mount storage pool")
- }
-
- // We can only create the btrfs subvolume under the mounted storage
- // pool. The on-disk layout for containers on a btrfs storage pool will
- // thus be
- // ${LXD_DIR}/storage-pools/<pool>/containers/. The btrfs tool will
- // complain if the intermediate path does not exist, so create it if it
- // doesn't already.
- containerSubvolumePath := s.getContainerSubvolumePath(s.pool.Name)
- if !shared.PathExists(containerSubvolumePath) {
- err := os.MkdirAll(containerSubvolumePath, containersDirMode)
- if err != nil {
- return errors.Wrap(err, "Failed to create volume directory")
- }
- }
-
- // Mountpoint of the image:
- // ${LXD_DIR}/images/<fingerprint>
- imageMntPoint := getImageMountPoint(s.pool.Name, fingerprint)
- imageStoragePoolLockID := getImageCreateLockID(s.pool.Name, fingerprint)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[imageStoragePoolLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- } else {
- lxdStorageOngoingOperationMap[imageStoragePoolLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- var imgerr error
- if !shared.PathExists(imageMntPoint) || !isBtrfsSubVolume(imageMntPoint) {
- imgerr = s.ImageCreate(fingerprint, tracker)
- }
-
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[imageStoragePoolLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, imageStoragePoolLockID)
- }
- lxdStorageMapLock.Unlock()
-
- if imgerr != nil {
- return errors.Wrap(imgerr, "Failed to create image volume")
- }
- }
-
- // Create a rw snapshot at
- // ${LXD_DIR}/storage-pools/<pool>/containers/<name>
- // from the mounted ro image snapshot mounted at
- // ${LXD_DIR}/storage-pools/<pool>/images/<fingerprint>
- containerSubvolumeName := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- err = s.btrfsPoolVolumesSnapshot(imageMntPoint, containerSubvolumeName, false, false)
- if err != nil {
- return errors.Wrap(err, "Failed to storage pool volume snapshot")
- }
-
- // Create the mountpoint for the container at:
- // ${LXD_DIR}/containers/<name>
- err = createContainerMountpoint(containerSubvolumeName, container.Path(), container.IsPrivileged())
- if err != nil {
- return errors.Wrap(err, "Failed to create container mountpoint")
- }
-
- logger.Debugf("Created BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- err = container.TemplateApply("create")
- if err != nil {
- return errors.Wrap(err, "Failed to apply container template")
- }
- return nil
-}
-
-func (s *storageBtrfs) ContainerDelete(container container) error {
- logger.Debugf("Deleting BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- // The storage pool needs to be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Delete the subvolume.
- containerSubvolumeName := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- if shared.PathExists(containerSubvolumeName) && isBtrfsSubVolume(containerSubvolumeName) {
- err = btrfsSubVolumesDelete(containerSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- // Delete the container's symlink to the subvolume.
- err = deleteContainerMountpoint(containerSubvolumeName, container.Path(), s.GetStorageTypeName())
- if err != nil {
- return err
- }
-
- // Delete potential snapshot mountpoints.
- snapshotMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- if shared.PathExists(snapshotMntPoint) {
- err := os.RemoveAll(snapshotMntPoint)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- }
-
- // Delete potential symlink
- // ${LXD_DIR}/snapshots/<container_name> to ${POOL}/snapshots/<container_name>
- snapshotSymlink := shared.VarPath("snapshots", projectPrefix(container.Project(), container.Name()))
- if shared.PathExists(snapshotSymlink) {
- err := os.Remove(snapshotSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Deleted BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) copyContainer(target container, source container) error {
- sourceContainerSubvolumeName := getContainerMountPoint(source.Project(), s.pool.Name, source.Name())
- if source.IsSnapshot() {
- sourceContainerSubvolumeName = getSnapshotMountPoint(source.Project(), s.pool.Name, source.Name())
- }
- targetContainerSubvolumeName := getContainerMountPoint(target.Project(), s.pool.Name, target.Name())
-
- containersPath := getContainerMountPoint("default", s.pool.Name, "")
- // Ensure that the directories immediately preceding the subvolume directory exist.
- if !shared.PathExists(containersPath) {
- err := os.MkdirAll(containersPath, containersDirMode)
- if err != nil {
- return err
- }
- }
-
- err := s.btrfsPoolVolumesSnapshot(sourceContainerSubvolumeName, targetContainerSubvolumeName, false, true)
- if err != nil {
- return err
- }
-
- err = createContainerMountpoint(targetContainerSubvolumeName, target.Path(), target.IsPrivileged())
- if err != nil {
- return err
- }
-
- err = target.TemplateApply("copy")
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) copySnapshot(target container, source container) error {
- sourceName := source.Name()
- targetName := target.Name()
- sourceContainerSubvolumeName := getSnapshotMountPoint(source.Project(), s.pool.Name, sourceName)
- targetContainerSubvolumeName := getSnapshotMountPoint(target.Project(), s.pool.Name, targetName)
-
- targetParentName, _, _ := containerGetParentAndSnapshotName(target.Name())
- containersPath := getSnapshotMountPoint(target.Project(), s.pool.Name, targetParentName)
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(target.Project(), targetParentName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(target.Project(), targetParentName))
- err := createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
-
- // Ensure that the directories immediately preceding the subvolume directory exist.
- if !shared.PathExists(containersPath) {
- err := os.MkdirAll(containersPath, containersDirMode)
- if err != nil {
- return err
- }
- }
-
- err = s.btrfsPoolVolumesSnapshot(sourceContainerSubvolumeName, targetContainerSubvolumeName, false, true)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) doCrossPoolContainerCopy(target container, source container, containerOnly bool, refresh bool, refreshSnapshots []container) error {
- sourcePool, err := source.StoragePool()
- if err != nil {
- return err
- }
-
- // setup storage for the source volume
- srcStorage, err := storagePoolVolumeInit(s.s, "default", sourcePool, source.Name(), storagePoolVolumeTypeContainer)
- if err != nil {
- return err
- }
-
- ourMount, err := srcStorage.StoragePoolMount()
- if err != nil {
- return err
- }
- if ourMount {
- defer srcStorage.StoragePoolUmount()
- }
-
- targetPool, err := target.StoragePool()
- if err != nil {
- return err
- }
-
- var snapshots []container
-
- if refresh {
- snapshots = refreshSnapshots
- } else {
- snapshots, err = source.Snapshots()
- if err != nil {
- return err
- }
-
- // create the main container
- err = s.doContainerCreate(target.Project(), target.Name(), target.IsPrivileged())
- if err != nil {
- return err
- }
- }
-
- destContainerMntPoint := getContainerMountPoint(target.Project(), targetPool, target.Name())
- bwlimit := s.pool.Config["rsync.bwlimit"]
- if !containerOnly {
- for _, snap := range snapshots {
- srcSnapshotMntPoint := getSnapshotMountPoint(target.Project(), sourcePool, snap.Name())
- _, err = rsyncLocalCopy(srcSnapshotMntPoint, destContainerMntPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into BTRFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- // create snapshot
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
- err = s.doContainerSnapshotCreate(target.Project(), fmt.Sprintf("%s/%s", target.Name(), snapOnlyName), target.Name())
- if err != nil {
- return err
- }
- }
- }
-
- srcContainerMntPoint := getContainerMountPoint(source.Project(), sourcePool, source.Name())
- _, err = rsyncLocalCopy(srcContainerMntPoint, destContainerMntPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into BTRFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) ContainerCopy(target container, source container, containerOnly bool) error {
- logger.Debugf("Copying BTRFS container storage %s to %s", source.Name(), target.Name())
-
- // The storage pool needs to be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- _, sourcePool, _ := source.Storage().GetContainerPoolInfo()
- _, targetPool, _ := target.Storage().GetContainerPoolInfo()
- if sourcePool != targetPool {
- return s.doCrossPoolContainerCopy(target, source, containerOnly, false, nil)
- }
-
- err = s.copyContainer(target, source)
- if err != nil {
- return err
- }
-
- if containerOnly {
- logger.Debugf("Copied BTRFS container storage %s to %s", source.Name(), target.Name())
- return nil
- }
-
- snapshots, err := source.Snapshots()
- if err != nil {
- return err
- }
-
- if len(snapshots) == 0 {
- logger.Debugf("Copied BTRFS container storage %s to %s", source.Name(), target.Name())
- return nil
- }
-
- for _, snap := range snapshots {
- sourceSnapshot, err := containerLoadByProjectAndName(s.s, source.Project(), snap.Name())
- if err != nil {
- return err
- }
-
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
- newSnapName := fmt.Sprintf("%s/%s", target.Name(), snapOnlyName)
- targetSnapshot, err := containerLoadByProjectAndName(s.s, target.Project(), newSnapName)
- if err != nil {
- return err
- }
-
- err = s.copySnapshot(targetSnapshot, sourceSnapshot)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Copied BTRFS container storage %s to %s", source.Name(), target.Name())
- return nil
-}
-
-func (s *storageBtrfs) ContainerRefresh(target container, source container, snapshots []container) error {
- logger.Debugf("Refreshing BTRFS container storage for %s from %s", target.Name(), source.Name())
-
- // The storage pool needs to be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- return s.doCrossPoolContainerCopy(target, source, len(snapshots) == 0, true, snapshots)
-}
-
-func (s *storageBtrfs) ContainerMount(c container) (bool, error) {
- logger.Debugf("Mounting BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Mounted BTRFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return true, nil
-}
-
-func (s *storageBtrfs) ContainerUmount(c container, path string) (bool, error) {
- return true, nil
-}
-
-func (s *storageBtrfs) ContainerRename(container container, newName string) error {
- logger.Debugf("Renaming BTRFS storage volume for container \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- oldContainerSubvolumeName := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- newContainerSubvolumeName := getContainerMountPoint(container.Project(), s.pool.Name, newName)
- err = os.Rename(oldContainerSubvolumeName, newContainerSubvolumeName)
- if err != nil {
- return err
- }
-
- newSymlink := shared.VarPath("containers", projectPrefix(container.Project(), newName))
- err = renameContainerMountpoint(oldContainerSubvolumeName, container.Path(), newContainerSubvolumeName, newSymlink)
- if err != nil {
- return err
- }
-
- oldSnapshotSubvolumeName := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- newSnapshotSubvolumeName := getSnapshotMountPoint(container.Project(), s.pool.Name, newName)
- if shared.PathExists(oldSnapshotSubvolumeName) {
- err = os.Rename(oldSnapshotSubvolumeName, newSnapshotSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- oldSnapshotSymlink := shared.VarPath("snapshots", projectPrefix(container.Project(), container.Name()))
- newSnapshotSymlink := shared.VarPath("snapshots", projectPrefix(container.Project(), newName))
- if shared.PathExists(oldSnapshotSymlink) {
- err := os.Remove(oldSnapshotSymlink)
- if err != nil {
- return err
- }
-
- err = os.Symlink(newSnapshotSubvolumeName, newSnapshotSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Renamed BTRFS storage volume for container \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
- return nil
-}
-
-func (s *storageBtrfs) ContainerRestore(container container, sourceContainer container) error {
- logger.Debugf("Restoring BTRFS storage volume for container \"%s\" from %s to %s", s.volume.Name, sourceContainer.Name(), container.Name())
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Create a backup so we can revert.
- targetContainerSubvolumeName := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- backupTargetContainerSubvolumeName := fmt.Sprintf("%s.tmp", targetContainerSubvolumeName)
- err = os.Rename(targetContainerSubvolumeName, backupTargetContainerSubvolumeName)
- if err != nil {
- return err
- }
- undo := true
- defer func() {
- if undo {
- os.Rename(backupTargetContainerSubvolumeName, targetContainerSubvolumeName)
- }
- }()
-
- ourStart, err := sourceContainer.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer sourceContainer.StorageStop()
- }
-
- // Mount the source container.
- srcContainerStorage := sourceContainer.Storage()
- _, sourcePool, _ := srcContainerStorage.GetContainerPoolInfo()
- sourceContainerSubvolumeName := ""
- if sourceContainer.IsSnapshot() {
- sourceContainerSubvolumeName = getSnapshotMountPoint(sourceContainer.Project(), sourcePool, sourceContainer.Name())
- } else {
- sourceContainerSubvolumeName = getContainerMountPoint(container.Project(), sourcePool, sourceContainer.Name())
- }
-
- var failure error
- _, targetPool, _ := s.GetContainerPoolInfo()
- if targetPool == sourcePool {
- // They are on the same storage pool, so we can simply snapshot.
- err := s.btrfsPoolVolumesSnapshot(sourceContainerSubvolumeName, targetContainerSubvolumeName, false, true)
- if err != nil {
- failure = err
- }
- } else {
- err := btrfsSubVolumeCreate(targetContainerSubvolumeName)
- if err == nil {
- // Use rsync to fill the empty volume. Sync by using
- // the subvolume name.
- bwlimit := s.pool.Config["rsync.bwlimit"]
- output, err := rsyncLocalCopy(sourceContainerSubvolumeName, targetContainerSubvolumeName, bwlimit)
- if err != nil {
- s.ContainerDelete(container)
- logger.Errorf("ContainerRestore: rsync failed: %s", string(output))
- failure = err
- }
- } else {
- failure = err
- }
- }
-
- if failure == nil {
- undo = false
- _, sourcePool, _ := srcContainerStorage.GetContainerPoolInfo()
- _, targetPool, _ := s.GetContainerPoolInfo()
- if targetPool == sourcePool {
- // Remove the backup, we made
- return btrfsSubVolumesDelete(backupTargetContainerSubvolumeName)
- }
-
- err = os.RemoveAll(backupTargetContainerSubvolumeName)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- }
-
- logger.Debugf("Restored BTRFS storage volume for container \"%s\" from %s to %s", s.volume.Name, sourceContainer.Name(), container.Name())
- return failure
-}
-
-func (s *storageBtrfs) ContainerGetUsage(container container) (int64, error) {
- return s.btrfsPoolVolumeQGroupUsage(container.Path())
-}
-
-func (s *storageBtrfs) doContainerSnapshotCreate(project string, targetName string, sourceName string) error {
- logger.Debugf("Creating BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // We can only create the btrfs subvolume under the mounted storage
- // pool. The on-disk layout for snapshots on a btrfs storage pool will
- // thus be
- // ${LXD_DIR}/storage-pools/<pool>/snapshots/. The btrfs tool will
- // complain if the intermediate path does not exist, so create it if it
- // doesn't already.
- snapshotSubvolumePath := getSnapshotSubvolumePath(project, s.pool.Name, sourceName)
- if !shared.PathExists(snapshotSubvolumePath) {
- err := os.MkdirAll(snapshotSubvolumePath, containersDirMode)
- if err != nil {
- return err
- }
- }
-
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(project, s.volume.Name))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceName))
- if !shared.PathExists(snapshotMntPointSymlink) {
- if !shared.PathExists(snapshotMntPointSymlinkTarget) {
- err = os.MkdirAll(snapshotMntPointSymlinkTarget, snapshotsDirMode)
- if err != nil {
- return err
- }
- }
-
- err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
- }
-
- srcContainerSubvolumeName := getContainerMountPoint(project, s.pool.Name, sourceName)
- snapshotSubvolumeName := getSnapshotMountPoint(project, s.pool.Name, targetName)
- err = s.btrfsPoolVolumesSnapshot(srcContainerSubvolumeName, snapshotSubvolumeName, true, true)
- if err != nil {
- return err
- }
-
- logger.Debugf("Created BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) ContainerSnapshotCreate(snapshotContainer container, sourceContainer container) error {
- err := s.doContainerSnapshotCreate(sourceContainer.Project(), snapshotContainer.Name(), sourceContainer.Name())
- if err != nil {
- s.ContainerSnapshotDelete(snapshotContainer)
- return err
- }
-
- return nil
-}
-
-func btrfsSnapshotDeleteInternal(project, poolName string, snapshotName string) error {
- snapshotSubvolumeName := getSnapshotMountPoint(project, poolName, snapshotName)
- if shared.PathExists(snapshotSubvolumeName) && isBtrfsSubVolume(snapshotSubvolumeName) {
- err := btrfsSubVolumesDelete(snapshotSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- sourceSnapshotMntPoint := shared.VarPath("snapshots", projectPrefix(project, snapshotName))
- os.Remove(sourceSnapshotMntPoint)
- os.Remove(snapshotSubvolumeName)
-
- sourceName, _, _ := containerGetParentAndSnapshotName(snapshotName)
- snapshotSubvolumePath := getSnapshotSubvolumePath(project, poolName, sourceName)
- os.Remove(snapshotSubvolumePath)
- if !shared.PathExists(snapshotSubvolumePath) {
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceName))
- os.Remove(snapshotMntPointSymlink)
- }
-
- return nil
-}
-
-func (s *storageBtrfs) ContainerSnapshotDelete(snapshotContainer container) error {
- logger.Debugf("Deleting BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- err = btrfsSnapshotDeleteInternal(snapshotContainer.Project(), s.pool.Name, snapshotContainer.Name())
- if err != nil {
- return err
- }
-
- logger.Debugf("Deleted BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) ContainerSnapshotStart(container container) (bool, error) {
- logger.Debugf("Initializing BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return false, err
- }
-
- snapshotSubvolumeName := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- roSnapshotSubvolumeName := fmt.Sprintf("%s.ro", snapshotSubvolumeName)
- if shared.PathExists(roSnapshotSubvolumeName) {
- logger.Debugf("The BTRFS snapshot is already mounted read-write")
- return false, nil
- }
-
- err = os.Rename(snapshotSubvolumeName, roSnapshotSubvolumeName)
- if err != nil {
- return false, err
- }
-
- err = s.btrfsPoolVolumesSnapshot(roSnapshotSubvolumeName, snapshotSubvolumeName, false, true)
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Initialized BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return true, nil
-}
-
-func (s *storageBtrfs) ContainerSnapshotStop(container container) (bool, error) {
- logger.Debugf("Stopping BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return false, err
- }
-
- snapshotSubvolumeName := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- roSnapshotSubvolumeName := fmt.Sprintf("%s.ro", snapshotSubvolumeName)
- if !shared.PathExists(roSnapshotSubvolumeName) {
- logger.Debugf("The BTRFS snapshot is currently not mounted read-write")
- return false, nil
- }
-
- if shared.PathExists(snapshotSubvolumeName) && isBtrfsSubVolume(snapshotSubvolumeName) {
- err = btrfsSubVolumesDelete(snapshotSubvolumeName)
- if err != nil {
- return false, err
- }
- }
-
- err = os.Rename(roSnapshotSubvolumeName, snapshotSubvolumeName)
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Stopped BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return true, nil
-}
-
-// ContainerSnapshotRename renames a snapshot of a container.
-func (s *storageBtrfs) ContainerSnapshotRename(snapshotContainer container, newName string) error {
- logger.Debugf("Renaming BTRFS storage volume for snapshot \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Unmount the snapshot if it is mounted otherwise we'll get EBUSY.
- // Rename the subvolume on the storage pool.
- oldSnapshotSubvolumeName := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, snapshotContainer.Name())
- newSnapshotSubvolumeName := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, newName)
- err = os.Rename(oldSnapshotSubvolumeName, newSnapshotSubvolumeName)
- if err != nil {
- return err
- }
-
- logger.Debugf("Renamed BTRFS storage volume for snapshot \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
- return nil
-}
-
-// Needed for live migration where an empty snapshot needs to be created before
-// rsyncing into it.
-func (s *storageBtrfs) ContainerSnapshotCreateEmpty(snapshotContainer container) error {
- logger.Debugf("Creating empty BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- // Mount the storage pool.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Create the snapshot subvole path on the storage pool.
- sourceName, _, _ := containerGetParentAndSnapshotName(snapshotContainer.Name())
- snapshotSubvolumePath := getSnapshotSubvolumePath(snapshotContainer.Project(), s.pool.Name, sourceName)
- snapshotSubvolumeName := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, snapshotContainer.Name())
- if !shared.PathExists(snapshotSubvolumePath) {
- err := os.MkdirAll(snapshotSubvolumePath, containersDirMode)
- if err != nil {
- return err
- }
- }
-
- err = btrfsSubVolumeCreate(snapshotSubvolumeName)
- if err != nil {
- return err
- }
-
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(snapshotContainer.Project(), sourceName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(snapshotContainer.Project(), sourceName))
- if !shared.PathExists(snapshotMntPointSymlink) {
- err := createContainerMountpoint(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink, snapshotContainer.IsPrivileged())
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Created empty BTRFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) doBtrfsBackup(cur string, prev string, target string) error {
- args := []string{"send"}
- if prev != "" {
- args = append(args, "-p", prev)
- }
- args = append(args, cur)
-
- eater, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE, 0644)
- if err != nil {
- return err
- }
- defer eater.Close()
-
- btrfsSendCmd := exec.Command("btrfs", args...)
- btrfsSendCmd.Stdout = eater
-
- err = btrfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- return err
-}
-
-func (s *storageBtrfs) doContainerBackupCreateOptimized(tmpPath string, backup backup, source container) error {
- // Handle snapshots
- finalParent := ""
- if !backup.containerOnly {
- snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath)
-
- // Retrieve the snapshots
- snapshots, err := source.Snapshots()
- if err != nil {
- return err
- }
-
- // Create the snapshot path
- if len(snapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return err
- }
- }
-
- for i, snap := range snapshots {
- _, snapName, _ := containerGetParentAndSnapshotName(snap.Name())
-
- // Figure out previous and current subvolumes
- prev := ""
- if i > 0 {
- // /var/lib/lxd/storage-pools/<pool>/containers-snapshots/<container>/<snapshot>
- prev = getSnapshotMountPoint(source.Project(), s.pool.Name, snapshots[i-1].Name())
- }
- cur := getSnapshotMountPoint(source.Project(), s.pool.Name, snap.Name())
-
- // Make a binary btrfs backup
- target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snapName)
- err := s.doBtrfsBackup(cur, prev, target)
- if err != nil {
- return err
- }
-
- finalParent = cur
- }
- }
-
- // Make a temporary copy of the container
- sourceVolume := getContainerMountPoint(source.Project(), s.pool.Name, source.Name())
- containersPath := getContainerMountPoint("default", s.pool.Name, "")
- tmpContainerMntPoint, err := ioutil.TempDir(containersPath, source.Name())
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpContainerMntPoint)
-
- err = os.Chmod(tmpContainerMntPoint, 0700)
- if err != nil {
- return err
- }
-
- targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
- err = s.btrfsPoolVolumesSnapshot(sourceVolume, targetVolume, true, true)
- if err != nil {
- return err
- }
- defer btrfsSubVolumesDelete(targetVolume)
-
- // Dump the container to a file
- fsDump := fmt.Sprintf("%s/container.bin", tmpPath)
- err = s.doBtrfsBackup(targetVolume, finalParent, fsDump)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) doContainerBackupCreateVanilla(tmpPath string, backup backup, source container) error {
- // Prepare for rsync
- rsync := func(oldPath string, newPath string, bwlimit string) error {
- output, err := rsyncLocalCopy(oldPath, newPath, bwlimit)
- if err != nil {
- return fmt.Errorf("Failed to rsync: %s: %s", string(output), err)
- }
-
- return nil
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- // Handle snapshots
- if !backup.containerOnly {
- snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath)
-
- // Retrieve the snapshots
- snapshots, err := source.Snapshots()
- if err != nil {
- return err
- }
-
- // Create the snapshot path
- if len(snapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return err
- }
- }
-
- for _, snap := range snapshots {
- _, snapName, _ := containerGetParentAndSnapshotName(snap.Name())
-
- // Mount the snapshot to a usable path
- _, err := s.ContainerSnapshotStart(snap)
- if err != nil {
- return err
- }
-
- snapshotMntPoint := getSnapshotMountPoint(snap.Project(), s.pool.Name, snap.Name())
- target := fmt.Sprintf("%s/%s", snapshotsPath, snapName)
-
- // Copy the snapshot
- err = rsync(snapshotMntPoint, target, bwlimit)
- s.ContainerSnapshotStop(snap)
- if err != nil {
- return err
- }
- }
- }
-
- // Make a temporary copy of the container
- sourceVolume := getContainerMountPoint(source.Project(), s.pool.Name, source.Name())
- containersPath := getContainerMountPoint("default", s.pool.Name, "")
- tmpContainerMntPoint, err := ioutil.TempDir(containersPath, source.Name())
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpContainerMntPoint)
-
- err = os.Chmod(tmpContainerMntPoint, 0700)
- if err != nil {
- return err
- }
-
- targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
- err = s.btrfsPoolVolumesSnapshot(sourceVolume, targetVolume, true, true)
- if err != nil {
- return err
- }
- defer btrfsSubVolumesDelete(targetVolume)
-
- // Copy the container
- containerPath := fmt.Sprintf("%s/container", tmpPath)
- err = rsync(targetVolume, containerPath, bwlimit)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) ContainerBackupCreate(backup backup, source container) error {
- // Start storage
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- // Create a temporary path for the backup
- tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_")
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpPath)
-
- // Generate the actual backup
- if backup.optimizedStorage {
- err = s.doContainerBackupCreateOptimized(tmpPath, backup, source)
- if err != nil {
- return err
- }
- } else {
- err := s.doContainerBackupCreateVanilla(tmpPath, backup, source)
- if err != nil {
- return err
- }
- }
-
- // Pack the backup
- err = backupCreateTarball(s.s, tmpPath, backup)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) doContainerBackupLoadOptimized(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- containerName, _, _ := containerGetParentAndSnapshotName(info.Name)
-
- containerMntPoint := getContainerMountPoint("default", s.pool.Name, "")
- unpackDir, err := ioutil.TempDir(containerMntPoint, containerName)
- if err != nil {
- return err
- }
- defer os.RemoveAll(unpackDir)
-
- err = os.Chmod(unpackDir, 0700)
- if err != nil {
- return err
- }
-
- unpackPath := fmt.Sprintf("%s/.backup_unpack", unpackDir)
- err = os.MkdirAll(unpackPath, 0711)
- if err != nil {
- return err
- }
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--strip-components=1",
- "-C", unpackPath, "backup",
- }...)
-
- // Extract container
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- logger.Errorf("Failed to untar \"%s\" into \"%s\": %s", "backup", unpackPath, err)
- return err
- }
-
- for _, snapshotOnlyName := range info.Snapshots {
- snapshotBackup := fmt.Sprintf("%s/snapshots/%s.bin", unpackPath, snapshotOnlyName)
- feeder, err := os.Open(snapshotBackup)
- if err != nil {
- return err
- }
-
- // create mountpoint
- snapshotMntPoint := getSnapshotMountPoint(info.Project, s.pool.Name, containerName)
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(info.Project, containerName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(info.Project, containerName))
- err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- feeder.Close()
- return err
- }
-
- // /var/lib/lxd/storage-pools/<pool>/snapshots/<container>/
- btrfsRecvCmd := exec.Command("btrfs", "receive", "-e", snapshotMntPoint)
- btrfsRecvCmd.Stdin = feeder
- msg, err := btrfsRecvCmd.CombinedOutput()
- feeder.Close()
- if err != nil {
- logger.Errorf("Failed to receive contents of btrfs backup \"%s\": %s", snapshotBackup, string(msg))
- return err
- }
- }
-
- containerBackupFile := fmt.Sprintf("%s/container.bin", unpackPath)
- feeder, err := os.Open(containerBackupFile)
- if err != nil {
- return err
- }
- defer feeder.Close()
-
- // /var/lib/lxd/storage-pools/<pool>/containers/
- btrfsRecvCmd := exec.Command("btrfs", "receive", "-vv", "-e", unpackDir)
- btrfsRecvCmd.Stdin = feeder
- msg, err := btrfsRecvCmd.CombinedOutput()
- if err != nil {
- logger.Errorf("Failed to receive contents of btrfs backup \"%s\": %s", containerBackupFile, string(msg))
- return err
- }
- tmpContainerMntPoint := fmt.Sprintf("%s/.backup", unpackDir)
- defer btrfsSubVolumesDelete(tmpContainerMntPoint)
-
- containerMntPoint = getContainerMountPoint(info.Project, s.pool.Name, info.Name)
- err = s.btrfsPoolVolumesSnapshot(tmpContainerMntPoint, containerMntPoint, false, true)
- if err != nil {
- logger.Errorf("Failed to create btrfs snapshot \"%s\" of \"%s\": %s", tmpContainerMntPoint, containerMntPoint, err)
- return err
- }
-
- // Create mountpoints
- err = createContainerMountpoint(containerMntPoint, shared.VarPath("containers", projectPrefix(info.Project, info.Name)), info.Privileged)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) doContainerBackupLoadVanilla(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- // create the main container
- err := s.doContainerCreate(info.Project, info.Name, info.Privileged)
- if err != nil {
- return err
- }
-
- containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, info.Name)
- // Extract container
- for _, snap := range info.Snapshots {
- cur := fmt.Sprintf("backup/snapshots/%s", snap)
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--recursive-unlink",
- "--xattrs-include=*",
- "--strip-components=3",
- "-C", containerMntPoint, cur,
- }...)
-
- // Extract snapshots
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- logger.Errorf("Failed to untar \"%s\" into \"%s\": %s", cur, containerMntPoint, err)
- return err
- }
-
- // create snapshot
- err = s.doContainerSnapshotCreate(info.Project, fmt.Sprintf("%s/%s", info.Name, snap), info.Name)
- if err != nil {
- return err
- }
- }
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--strip-components=2",
- "--xattrs-include=*",
- "-C", containerMntPoint, "backup/container",
- }...)
-
- // Extract container
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- logger.Errorf("Failed to untar \"backup/container\" into \"%s\": %s", containerMntPoint, err)
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- logger.Debugf("Loading BTRFS storage volume for backup \"%s\" on storage pool \"%s\"", info.Name, s.pool.Name)
-
- if info.HasBinaryFormat {
- return s.doContainerBackupLoadOptimized(info, data, tarArgs)
- }
-
- return s.doContainerBackupLoadVanilla(info, data, tarArgs)
-}
-
-func (s *storageBtrfs) ImageCreate(fingerprint string, tracker *ioprogress.ProgressTracker) error {
- logger.Debugf("Creating BTRFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
-
- // Create the subvolume.
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- err = s.createImageDbPoolVolume(fingerprint)
- if err != nil {
- return err
- }
-
- // We can only create the btrfs subvolume under the mounted storage
- // pool. The on-disk layout for images on a btrfs storage pool will thus
- // be
- // ${LXD_DIR}/storage-pools/<pool>/images/. The btrfs tool will
- // complain if the intermediate path does not exist, so create it if it
- // doesn't already.
- imageSubvolumePath := s.getImageSubvolumePath(s.pool.Name)
- if !shared.PathExists(imageSubvolumePath) {
- err := os.MkdirAll(imageSubvolumePath, imagesDirMode)
- if err != nil {
- return err
- }
- }
-
- // Create a temporary rw btrfs subvolume. From this rw subvolume we'll
- // create a ro snapshot below. The path with which we do this is
- // ${LXD_DIR}/storage-pools/<pool>/images/<fingerprint>@<pool>_tmp.
- imageSubvolumeName := getImageMountPoint(s.pool.Name, fingerprint)
- tmpImageSubvolumeName := fmt.Sprintf("%s_tmp", imageSubvolumeName)
- err = btrfsSubVolumeCreate(tmpImageSubvolumeName)
- if err != nil {
- return err
- }
- // Delete volume on error.
- undo := true
- defer func() {
- if undo {
- btrfsSubVolumesDelete(tmpImageSubvolumeName)
- }
- }()
-
- // Unpack the image in imageMntPoint.
- imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, tmpImageSubvolumeName, storageTypeBtrfs, s.s.OS.RunningInUserNS, tracker)
- if err != nil {
- return err
- }
-
- // Now create a read-only snapshot of the subvolume.
- // The path with which we do this is
- // ${LXD_DIR}/storage-pools/<pool>/images/<fingerprint>.
- err = s.btrfsPoolVolumesSnapshot(tmpImageSubvolumeName, imageSubvolumeName, true, true)
- if err != nil {
- return err
- }
-
- defer func() {
- if undo {
- btrfsSubVolumesDelete(imageSubvolumeName)
- }
- }()
-
- err = btrfsSubVolumesDelete(tmpImageSubvolumeName)
- if err != nil {
- return err
- }
-
- undo = false
-
- logger.Debugf("Created BTRFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) ImageDelete(fingerprint string) error {
- logger.Debugf("Deleting BTRFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Delete the btrfs subvolume. The path with which we
- // do this is ${LXD_DIR}/storage-pools/<pool>/images/<fingerprint>.
- imageSubvolumeName := getImageMountPoint(s.pool.Name, fingerprint)
- if shared.PathExists(imageSubvolumeName) && isBtrfsSubVolume(imageSubvolumeName) {
- err = btrfsSubVolumesDelete(imageSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- err = s.deleteImageDbPoolVolume(fingerprint)
- if err != nil {
- return err
- }
-
- // Now delete the mountpoint for the image:
- // ${LXD_DIR}/images/<fingerprint>.
- if shared.PathExists(imageSubvolumeName) {
- err := os.RemoveAll(imageSubvolumeName)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- }
-
- logger.Debugf("Deleted BTRFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) ImageMount(fingerprint string) (bool, error) {
- logger.Debugf("Mounting BTRFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
-
- // The storage pool must be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Mounted BTRFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
- return true, nil
-}
-
-func (s *storageBtrfs) ImageUmount(fingerprint string) (bool, error) {
- return true, nil
-}
-
-func btrfsSubVolumeCreate(subvol string) error {
- parentDestPath := filepath.Dir(subvol)
- if !shared.PathExists(parentDestPath) {
- err := os.MkdirAll(parentDestPath, 0711)
- if err != nil {
- return err
- }
- }
-
- output, err := shared.RunCommand(
- "btrfs",
- "subvolume",
- "create",
- subvol)
- if err != nil {
- logger.Errorf("Failed to create BTRFS subvolume \"%s\": %s", subvol, output)
- return err
- }
-
- return nil
-}
-
-func btrfsSubVolumeQGroup(subvol string) (string, error) {
- output, err := shared.RunCommand(
- "btrfs",
- "qgroup",
- "show",
- subvol,
- "-e",
- "-f")
-
- if err != nil {
- return "", db.ErrNoSuchObject
- }
-
- var qgroup string
- for _, line := range strings.Split(output, "\n") {
- if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
- continue
- }
-
- fields := strings.Fields(line)
- if len(fields) != 4 {
- continue
- }
-
- qgroup = fields[0]
- }
-
- if qgroup == "" {
- return "", fmt.Errorf("Unable to find quota group")
- }
-
- return qgroup, nil
-}
-
-func (s *storageBtrfs) btrfsPoolVolumeQGroupUsage(subvol string) (int64, error) {
- output, err := shared.RunCommand(
- "btrfs",
- "qgroup",
- "show",
- subvol,
- "-e",
- "-f")
-
- if err != nil {
- return -1, fmt.Errorf("BTRFS quotas not supported. Try enabling them with \"btrfs quota enable\"")
- }
-
- for _, line := range strings.Split(output, "\n") {
- if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
- continue
- }
-
- fields := strings.Fields(line)
- if len(fields) != 4 {
- continue
- }
-
- usage, err := strconv.ParseInt(fields[2], 10, 64)
- if err != nil {
- continue
- }
-
- return usage, nil
- }
-
- return -1, fmt.Errorf("Unable to find current qgroup usage")
-}
-
-func btrfsSubVolumeDelete(subvol string) error {
- // Attempt (but don't fail on) to delete any qgroup on the subvolume
- qgroup, err := btrfsSubVolumeQGroup(subvol)
- if err == nil {
- shared.RunCommand(
- "btrfs",
- "qgroup",
- "destroy",
- qgroup,
- subvol)
- }
-
- // Attempt to make the subvolume writable
- shared.RunCommand("btrfs", "property", "set", subvol, "ro", "false")
-
- // Delete the subvolume itself
- _, err = shared.RunCommand(
- "btrfs",
- "subvolume",
- "delete",
- subvol)
-
- return err
-}
-
-// btrfsPoolVolumesDelete is the recursive variant on btrfsPoolVolumeDelete,
-// it first deletes subvolumes of the subvolume and then the
-// subvolume itself.
-func btrfsSubVolumesDelete(subvol string) error {
- // Delete subsubvols.
- subsubvols, err := btrfsSubVolumesGet(subvol)
- if err != nil {
- return err
- }
- sort.Sort(sort.Reverse(sort.StringSlice(subsubvols)))
-
- for _, subsubvol := range subsubvols {
- err := btrfsSubVolumeDelete(path.Join(subvol, subsubvol))
- if err != nil {
- return err
- }
- }
-
- // Delete the subvol itself
- err = btrfsSubVolumeDelete(subvol)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-/*
- * btrfsSnapshot creates a snapshot of "source" to "dest"
- * the result will be readonly if "readonly" is True.
- */
-func btrfsSnapshot(source string, dest string, readonly bool) error {
- var output string
- var err error
- if readonly {
- output, err = shared.RunCommand(
- "btrfs",
- "subvolume",
- "snapshot",
- "-r",
- source,
- dest)
- } else {
- output, err = shared.RunCommand(
- "btrfs",
- "subvolume",
- "snapshot",
- source,
- dest)
- }
- if err != nil {
- return fmt.Errorf(
- "subvolume snapshot failed, source=%s, dest=%s, output=%s",
- source,
- dest,
- output,
- )
- }
-
- return err
-}
-
-func (s *storageBtrfs) btrfsPoolVolumeSnapshot(source string, dest string, readonly bool) error {
- return btrfsSnapshot(source, dest, readonly)
-}
-
-func (s *storageBtrfs) btrfsPoolVolumesSnapshot(source string, dest string, readonly bool, recursive bool) error {
- // Now snapshot all subvolumes of the root.
- if recursive {
- // Get a list of subvolumes of the root
- subsubvols, err := btrfsSubVolumesGet(source)
- if err != nil {
- return err
- }
- sort.Sort(sort.StringSlice(subsubvols))
-
- if len(subsubvols) > 0 && readonly {
- // A root with subvolumes can never be readonly,
- // also don't make subvolumes readonly.
- readonly = false
-
- logger.Warnf("Subvolumes detected, ignoring ro flag")
- }
-
- // First snapshot the root
- err = s.btrfsPoolVolumeSnapshot(source, dest, readonly)
- if err != nil {
- return err
- }
-
- for _, subsubvol := range subsubvols {
- // Clear the target for the subvol to use
- os.Remove(path.Join(dest, subsubvol))
-
- err := s.btrfsPoolVolumeSnapshot(path.Join(source, subsubvol), path.Join(dest, subsubvol), readonly)
- if err != nil {
- return err
- }
- }
- } else {
- err := s.btrfsPoolVolumeSnapshot(source, dest, readonly)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// isBtrfsSubVolume returns true if the given Path is a btrfs subvolume else
-// false.
-func isBtrfsSubVolume(subvolPath string) bool {
- fs := syscall.Stat_t{}
- err := syscall.Lstat(subvolPath, &fs)
- if err != nil {
- return false
- }
-
- // Check if BTRFS_FIRST_FREE_OBJECTID
- if fs.Ino != 256 {
- return false
- }
-
- return true
-}
-
-func isBtrfsFilesystem(path string) bool {
- _, err := shared.RunCommand("btrfs", "filesystem", "show", path)
- if err != nil {
- return false
- }
-
- return true
-}
-
-func isOnBtrfs(path string) bool {
- fs := syscall.Statfs_t{}
-
- err := syscall.Statfs(path, &fs)
- if err != nil {
- return false
- }
-
- if fs.Type != util.FilesystemSuperMagicBtrfs {
- return false
- }
-
- return true
-}
-
-func btrfsSubVolumesGet(path string) ([]string, error) {
- result := []string{}
-
- if !strings.HasSuffix(path, "/") {
- path = path + "/"
- }
-
- // Unprivileged users can't get to fs internals
- filepath.Walk(path, func(fpath string, fi os.FileInfo, err error) error {
- // Skip walk errors
- if err != nil {
- return nil
- }
-
- // Ignore the base path
- if strings.TrimRight(fpath, "/") == strings.TrimRight(path, "/") {
- return nil
- }
-
- // Subvolumes can only be directories
- if !fi.IsDir() {
- return nil
- }
-
- // Check if a btrfs subvolume
- if isBtrfsSubVolume(fpath) {
- result = append(result, strings.TrimPrefix(fpath, path))
- }
-
- return nil
- })
-
- return result, nil
-}
-
-type btrfsMigrationSourceDriver struct {
- container container
- snapshots []container
- btrfsSnapshotNames []string
- btrfs *storageBtrfs
- runningSnapName string
- stoppedSnapName string
-}
-
-func (s *btrfsMigrationSourceDriver) send(conn *websocket.Conn, btrfsPath string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
- args := []string{"send"}
- if btrfsParent != "" {
- args = append(args, "-p", btrfsParent)
- }
- args = append(args, btrfsPath)
-
- cmd := exec.Command("btrfs", args...)
-
- stdout, err := cmd.StdoutPipe()
- if err != nil {
- return err
- }
-
- readPipe := io.ReadCloser(stdout)
- if readWrapper != nil {
- readPipe = readWrapper(stdout)
- }
-
- stderr, err := cmd.StderrPipe()
- if err != nil {
- return err
- }
-
- err = cmd.Start()
- if err != nil {
- return err
- }
-
- <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
-
- output, err := ioutil.ReadAll(stderr)
- if err != nil {
- logger.Errorf("Problem reading btrfs send stderr: %s", err)
- }
-
- err = cmd.Wait()
- if err != nil {
- logger.Errorf("Problem with btrfs send: %s", string(output))
- }
-
- return err
-}
-
-func (s *btrfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error {
- _, containerPool, _ := s.container.Storage().GetContainerPoolInfo()
- containerName := s.container.Name()
- containersPath := getContainerMountPoint("default", containerPool, "")
- sourceName := containerName
-
- // Deal with sending a snapshot to create a container on another LXD
- // instance.
- if s.container.IsSnapshot() {
- sourceName, _, _ := containerGetParentAndSnapshotName(containerName)
- snapshotsPath := getSnapshotMountPoint(s.container.Project(), containerPool, sourceName)
- tmpContainerMntPoint, err := ioutil.TempDir(snapshotsPath, sourceName)
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpContainerMntPoint)
-
- err = os.Chmod(tmpContainerMntPoint, 0700)
- if err != nil {
- return err
- }
-
- migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint)
- snapshotMntPoint := getSnapshotMountPoint(s.container.Project(), containerPool, containerName)
- err = s.btrfs.btrfsPoolVolumesSnapshot(snapshotMntPoint, migrationSendSnapshot, true, true)
- if err != nil {
- return err
- }
- defer btrfsSubVolumesDelete(migrationSendSnapshot)
-
- wrapper := StorageProgressReader(op, "fs_progress", containerName)
- return s.send(conn, migrationSendSnapshot, "", wrapper)
- }
-
- if !containerOnly {
- for i, snap := range s.snapshots {
- prev := ""
- if i > 0 {
- prev = getSnapshotMountPoint(snap.Project(), containerPool, s.snapshots[i-1].Name())
- }
-
- snapMntPoint := getSnapshotMountPoint(snap.Project(), containerPool, snap.Name())
- wrapper := StorageProgressReader(op, "fs_progress", snap.Name())
- if err := s.send(conn, snapMntPoint, prev, wrapper); err != nil {
- return err
- }
- }
- }
-
- tmpContainerMntPoint, err := ioutil.TempDir(containersPath, containerName)
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpContainerMntPoint)
-
- err = os.Chmod(tmpContainerMntPoint, 0700)
- if err != nil {
- return err
- }
-
- migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint)
- containerMntPoint := getContainerMountPoint(s.container.Project(), containerPool, sourceName)
- err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPoint, migrationSendSnapshot, true, true)
- if err != nil {
- return err
- }
- defer btrfsSubVolumesDelete(migrationSendSnapshot)
-
- btrfsParent := ""
- if len(s.btrfsSnapshotNames) > 0 {
- btrfsParent = s.btrfsSnapshotNames[len(s.btrfsSnapshotNames)-1]
- }
-
- wrapper := StorageProgressReader(op, "fs_progress", containerName)
- return s.send(conn, migrationSendSnapshot, btrfsParent, wrapper)
-}
-
-func (s *btrfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error {
- tmpPath := getSnapshotMountPoint(s.container.Project(), s.btrfs.pool.Name,
- fmt.Sprintf("%s/.migration-send", s.container.Name()))
- err := os.MkdirAll(tmpPath, 0711)
- if err != nil {
- return err
- }
-
- err = os.Chmod(tmpPath, 0700)
- if err != nil {
- return err
- }
-
- s.stoppedSnapName = fmt.Sprintf("%s/.root", tmpPath)
- parentName, _, _ := containerGetParentAndSnapshotName(s.container.Name())
- containerMntPt := getContainerMountPoint(s.container.Project(), s.btrfs.pool.Name, parentName)
- err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPt, s.stoppedSnapName, true, true)
- if err != nil {
- return err
- }
-
- return s.send(conn, s.stoppedSnapName, s.runningSnapName, nil)
-}
-
-func (s *btrfsMigrationSourceDriver) Cleanup() {
- if s.stoppedSnapName != "" {
- btrfsSubVolumesDelete(s.stoppedSnapName)
- }
-
- if s.runningSnapName != "" {
- btrfsSubVolumesDelete(s.runningSnapName)
- }
-}
-
-func (s *storageBtrfs) MigrationType() migration.MigrationFSType {
- if s.s.OS.RunningInUserNS {
- return migration.MigrationFSType_RSYNC
- }
-
- return migration.MigrationFSType_BTRFS
-}
-
-func (s *storageBtrfs) PreservesInodes() bool {
- if s.s.OS.RunningInUserNS {
- return false
- }
-
- return true
-}
-
-func (s *storageBtrfs) MigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
- if s.s.OS.RunningInUserNS {
- return rsyncMigrationSource(args)
- }
-
- /* List all the snapshots in order of reverse creation. The idea here
- * is that we send the oldest to newest snapshot, hopefully saving on
- * xfer costs. Then, after all that, we send the container itself.
- */
- var err error
- var snapshots = []container{}
- if !args.ContainerOnly {
- snapshots, err = args.Container.Snapshots()
- if err != nil {
- return nil, err
- }
- }
-
- driver := &btrfsMigrationSourceDriver{
- container: args.Container,
- snapshots: snapshots,
- btrfsSnapshotNames: []string{},
- btrfs: s,
- }
-
- if !args.ContainerOnly {
- for _, snap := range snapshots {
- btrfsPath := getSnapshotMountPoint(snap.Project(), s.pool.Name, snap.Name())
- driver.btrfsSnapshotNames = append(driver.btrfsSnapshotNames, btrfsPath)
- }
- }
-
- return driver, nil
-}
-
-func (s *storageBtrfs) MigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
- if s.s.OS.RunningInUserNS {
- return rsyncMigrationSink(conn, op, args)
- }
-
- btrfsRecv := func(snapName string, btrfsPath string, targetPath string, isSnapshot bool, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
- args := []string{"receive", "-e", btrfsPath}
- cmd := exec.Command("btrfs", args...)
-
- // Remove the existing pre-created subvolume
- err := btrfsSubVolumesDelete(targetPath)
- if err != nil {
- logger.Errorf("Failed to delete pre-created BTRFS subvolume: %s: %v", btrfsPath, err)
- return err
- }
-
- stdin, err := cmd.StdinPipe()
- if err != nil {
- return err
- }
-
- stderr, err := cmd.StderrPipe()
- if err != nil {
- return err
- }
-
- err = cmd.Start()
- if err != nil {
- return err
- }
-
- writePipe := io.WriteCloser(stdin)
- if writeWrapper != nil {
- writePipe = writeWrapper(stdin)
- }
-
- <-shared.WebsocketRecvStream(writePipe, conn)
-
- output, err := ioutil.ReadAll(stderr)
- if err != nil {
- logger.Debugf("Problem reading btrfs receive stderr %s", err)
- }
-
- err = cmd.Wait()
- if err != nil {
- logger.Errorf("Problem with btrfs receive: %s", string(output))
- return err
- }
-
- receivedSnapshot := fmt.Sprintf("%s/.migration-send", btrfsPath)
- // handle older lxd versions
- if !shared.PathExists(receivedSnapshot) {
- receivedSnapshot = fmt.Sprintf("%s/.root", btrfsPath)
- }
- if isSnapshot {
- receivedSnapshot = fmt.Sprintf("%s/%s", btrfsPath, snapName)
- err = s.btrfsPoolVolumesSnapshot(receivedSnapshot, targetPath, true, true)
- } else {
- err = s.btrfsPoolVolumesSnapshot(receivedSnapshot, targetPath, false, true)
- }
- if err != nil {
- logger.Errorf("Problem with btrfs snapshot: %s", err)
- return err
- }
-
- err = btrfsSubVolumesDelete(receivedSnapshot)
- if err != nil {
- logger.Errorf("Failed to delete BTRFS subvolume \"%s\": %s", btrfsPath, err)
- return err
- }
-
- return nil
- }
-
- containerName := args.Container.Name()
- _, containerPool, _ := args.Container.Storage().GetContainerPoolInfo()
- containersPath := getSnapshotMountPoint(args.Container.Project(), containerPool, containerName)
- if !args.ContainerOnly && len(args.Snapshots) > 0 {
- err := os.MkdirAll(containersPath, containersDirMode)
- if err != nil {
- return err
- }
-
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", containerPool, "containers-snapshots", projectPrefix(args.Container.Project(), containerName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(args.Container.Project(), containerName))
- if !shared.PathExists(snapshotMntPointSymlink) {
- err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
- }
- }
-
- // At this point we have already figured out the parent
- // container's root disk device so we can simply
- // retrieve it from the expanded devices.
- parentStoragePool := ""
- parentExpandedDevices := args.Container.ExpandedDevices()
- parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := shared.GetRootDiskDevice(parentExpandedDevices)
- if parentLocalRootDiskDeviceKey != "" {
- parentStoragePool = parentLocalRootDiskDevice["pool"]
- }
-
- // A little neuroticism.
- if parentStoragePool == "" {
- return fmt.Errorf("Detected that the container's root device is missing the pool property during BTRFS migration")
- }
-
- if !args.ContainerOnly {
- for _, snap := range args.Snapshots {
- ctArgs := snapshotProtobufToContainerArgs(args.Container.Project(), containerName, snap)
-
- // Ensure that snapshot and parent container have the
- // same storage pool in their local root disk device.
- // If the root disk device for the snapshot comes from a
- // profile on the new instance as well we don't need to
- // do anything.
- if ctArgs.Devices != nil {
- snapLocalRootDiskDeviceKey, _, _ := shared.GetRootDiskDevice(ctArgs.Devices)
- if snapLocalRootDiskDeviceKey != "" {
- ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
- }
- }
-
- snapshotMntPoint := getSnapshotMountPoint(args.Container.Project(), containerPool, ctArgs.Name)
- _, err := containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs)
- if err != nil {
- return err
- }
-
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(args.Container.Project(), containerName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(args.Container.Project(), containerName))
- err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
-
- tmpSnapshotMntPoint, err := ioutil.TempDir(containersPath, projectPrefix(args.Container.Project(), containerName))
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpSnapshotMntPoint)
-
- err = os.Chmod(tmpSnapshotMntPoint, 0700)
- if err != nil {
- return err
- }
-
- wrapper := StorageProgressWriter(op, "fs_progress", *snap.Name)
- err = btrfsRecv(*(snap.Name), tmpSnapshotMntPoint, snapshotMntPoint, true, wrapper)
- if err != nil {
- return err
- }
- }
- }
-
- containersMntPoint := getContainerMountPoint(args.Container.Project(), s.pool.Name, "")
- err := createContainerMountpoint(containersMntPoint, args.Container.Path(), args.Container.IsPrivileged())
- if err != nil {
- return err
- }
-
- /* finally, do the real container */
- wrapper := StorageProgressWriter(op, "fs_progress", containerName)
- tmpContainerMntPoint, err := ioutil.TempDir(containersMntPoint, projectPrefix(args.Container.Project(), containerName))
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpContainerMntPoint)
-
- err = os.Chmod(tmpContainerMntPoint, 0700)
- if err != nil {
- return err
- }
-
- containerMntPoint := getContainerMountPoint(args.Container.Project(), s.pool.Name, containerName)
- err = btrfsRecv("", tmpContainerMntPoint, containerMntPoint, false, wrapper)
- if err != nil {
- return err
- }
-
- if args.Live {
- err = btrfsRecv("", tmpContainerMntPoint, containerMntPoint, false, wrapper)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func (s *storageBtrfs) btrfsLookupFsUUID(fs string) (string, error) {
- output, err := shared.RunCommand(
- "btrfs",
- "filesystem",
- "show",
- "--raw",
- fs)
- if err != nil {
- return "", fmt.Errorf("failed to detect UUID")
- }
-
- outputString := output
- idx := strings.Index(outputString, "uuid: ")
- outputString = outputString[idx+6:]
- outputString = strings.TrimSpace(outputString)
- idx = strings.Index(outputString, "\t")
- outputString = outputString[:idx]
- outputString = strings.Trim(outputString, "\n")
-
- return outputString, nil
-}
-
-func (s *storageBtrfs) StorageEntitySetQuota(volumeType int, size int64, data interface{}) error {
- logger.Debugf(`Setting BTRFS quota for "%s"`, s.volume.Name)
-
- var c container
- var subvol string
- switch volumeType {
- case storagePoolVolumeTypeContainer:
- c = data.(container)
- subvol = getContainerMountPoint(c.Project(), s.pool.Name, c.Name())
- case storagePoolVolumeTypeCustom:
- subvol = getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- }
-
- qgroup, err := btrfsSubVolumeQGroup(subvol)
- if err != nil {
- if err != db.ErrNoSuchObject {
- return err
- }
-
- // Enable quotas
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- output, err := shared.RunCommand(
- "btrfs", "quota", "enable", poolMntPoint)
- if err != nil && !s.s.OS.RunningInUserNS {
- return fmt.Errorf("Failed to enable quotas on BTRFS pool: %s", output)
- }
- }
-
- // Attempt to make the subvolume writable
- shared.RunCommand("btrfs", "property", "set", subvol, "ro", "false")
- if size > 0 {
- output, err := shared.RunCommand(
- "btrfs",
- "qgroup",
- "limit",
- "-e", fmt.Sprintf("%d", size),
- subvol)
-
- if err != nil {
- return fmt.Errorf("Failed to set btrfs quota: %s", output)
- }
- } else if qgroup != "" {
- output, err := shared.RunCommand(
- "btrfs",
- "qgroup",
- "destroy",
- qgroup,
- subvol)
-
- if err != nil {
- return fmt.Errorf("Failed to set btrfs quota: %s", output)
- }
- }
-
- logger.Debugf(`Set BTRFS quota for "%s"`, s.volume.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolResources() (*api.ResourcesStoragePool, error) {
- ourMount, err := s.StoragePoolMount()
- if err != nil {
- return nil, err
- }
- if ourMount {
- defer s.StoragePoolUmount()
- }
-
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
-
- // Inode allocation is dynamic so no use in reporting them.
-
- return storageResource(poolMntPoint)
-}
-
-func (s *storageBtrfs) StoragePoolVolumeCopy(source *api.StorageVolumeSource) error {
- logger.Infof("Copying BTRFS storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
- successMsg := fmt.Sprintf("Copied BTRFS storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
-
- // The storage pool needs to be mounted.
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- if s.pool.Name != source.Pool {
- return s.doCrossPoolVolumeCopy(source.Pool, source.Name, source.VolumeOnly)
- }
-
- err = s.copyVolume(source.Pool, source.Name, s.volume.Name, source.VolumeOnly)
- if err != nil {
- logger.Errorf("Failed to create BTRFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- if source.VolumeOnly {
- logger.Infof(successMsg)
- return nil
- }
-
- subvols, err := btrfsSubVolumesGet(s.getCustomSnapshotSubvolumePath(source.Pool))
- if err != nil {
- return err
- }
-
- for _, snapOnlyName := range subvols {
- snap := fmt.Sprintf("%s/%s", source.Name, snapOnlyName)
-
- err := s.copyVolume(source.Pool, snap, fmt.Sprintf("%s/%s", s.volume.Name, snapOnlyName), false)
- if err != nil {
- logger.Errorf("Failed to create BTRFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
- }
-
- logger.Infof(successMsg)
- return nil
-}
-
-func (s *storageBtrfs) copyVolume(sourcePool string, sourceName string, targetName string, volumeOnly bool) error {
- var customDir string
- var srcMountPoint string
- var dstMountPoint string
-
- isSrcSnapshot := shared.IsSnapshot(sourceName)
- isDstSnapshot := shared.IsSnapshot(targetName)
-
- if isSrcSnapshot {
- srcMountPoint = getStoragePoolVolumeSnapshotMountPoint(sourcePool, sourceName)
- } else {
- srcMountPoint = getStoragePoolVolumeMountPoint(sourcePool, sourceName)
- }
-
- if isDstSnapshot {
- dstMountPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, targetName)
- } else {
- dstMountPoint = getStoragePoolVolumeMountPoint(s.pool.Name, targetName)
- }
-
- // Ensure that the directories immediately preceding the subvolume directory exist.
- if isDstSnapshot {
- volName, _, _ := containerGetParentAndSnapshotName(targetName)
- customDir = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, volName)
- } else {
- customDir = getStoragePoolVolumeMountPoint(s.pool.Name, "")
- }
-
- if !shared.PathExists(customDir) {
- err := os.MkdirAll(customDir, customDirMode)
- if err != nil {
- logger.Errorf("Failed to create directory \"%s\" for storage volume \"%s\" on storage pool \"%s\": %s", customDir, s.volume.Name, s.pool.Name, err)
- return err
- }
- }
-
- err := s.btrfsPoolVolumesSnapshot(srcMountPoint, dstMountPoint, false, true)
- if err != nil {
- logger.Errorf("Failed to create BTRFS snapshot for storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- return nil
-}
-
-func (s *storageBtrfs) doCrossPoolVolumeCopy(sourcePool string, sourceName string, volumeOnly bool) error {
- // setup storage for the source volume
- srcStorage, err := storagePoolVolumeInit(s.s, "default", sourcePool, sourceName, storagePoolVolumeTypeCustom)
- if err != nil {
- return err
- }
-
- ourMount, err := srcStorage.StoragePoolMount()
- if err != nil {
- return err
- }
- if ourMount {
- defer srcStorage.StoragePoolUmount()
- }
-
- err = s.StoragePoolVolumeCreate()
- if err != nil {
- return err
- }
-
- destVolumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- if !volumeOnly {
- // Handle snapshots
- snapshots, err := storagePoolVolumeSnapshotsGet(s.s, sourcePool, sourceName, storagePoolVolumeTypeCustom)
- if err != nil {
- return err
- }
-
- for _, snap := range snapshots {
- srcSnapshotMntPoint := getStoragePoolVolumeSnapshotMountPoint(sourcePool, snap)
-
- _, err = rsyncLocalCopy(srcSnapshotMntPoint, destVolumeMntPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into BTRFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- // create snapshot
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap)
-
- err = s.doVolumeSnapshotCreate(s.pool.Name, s.volume.Name, fmt.Sprintf("%s/%s", s.volume.Name, snapOnlyName))
- if err != nil {
- return err
- }
- }
- }
-
- var srcVolumeMntPoint string
-
- if shared.IsSnapshot(sourceName) {
- // copy snapshot to volume
- srcVolumeMntPoint = getStoragePoolVolumeSnapshotMountPoint(sourcePool, sourceName)
- } else {
- // copy volume to volume
- srcVolumeMntPoint = getStoragePoolVolumeMountPoint(sourcePool, sourceName)
- }
-
- _, err = rsyncLocalCopy(srcVolumeMntPoint, destVolumeMntPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into BTRFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- return nil
-}
-
-func (s *btrfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error {
- msg := fmt.Sprintf("Function not implemented")
- logger.Errorf(msg)
- return fmt.Errorf(msg)
-}
-
-func (s *storageBtrfs) StorageMigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
- return rsyncStorageMigrationSource(args)
-}
-
-func (s *storageBtrfs) StorageMigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
- return rsyncStorageMigrationSink(conn, op, args)
-}
-
-func (s *storageBtrfs) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
- logger.Infof("Creating BTRFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- err := s.doVolumeSnapshotCreate(s.pool.Name, s.volume.Name, target.Name)
- if err != nil {
- return err
- }
-
- logger.Infof("Created BTRFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) doVolumeSnapshotCreate(sourcePool string, sourceName string, targetName string) error {
- // Create subvolume path on the storage pool.
- customSubvolumePath := s.getCustomSubvolumePath(s.pool.Name)
-
- err := os.MkdirAll(customSubvolumePath, 0700)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- _, _, ok := containerGetParentAndSnapshotName(targetName)
- if !ok {
- return err
- }
-
- customSnapshotSubvolumeName := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
-
- err = os.MkdirAll(customSnapshotSubvolumeName, snapshotsDirMode)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- sourcePath := getStoragePoolVolumeMountPoint(sourcePool, sourceName)
- targetPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, targetName)
-
- return s.btrfsPoolVolumesSnapshot(sourcePath, targetPath, true, true)
-}
-
-func (s *storageBtrfs) StoragePoolVolumeSnapshotDelete() error {
- logger.Infof("Deleting BTRFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- snapshotSubvolumeName := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- if shared.PathExists(snapshotSubvolumeName) && isBtrfsSubVolume(snapshotSubvolumeName) {
- err := btrfsSubVolumesDelete(snapshotSubvolumeName)
- if err != nil {
- return err
- }
- }
-
- err := os.RemoveAll(snapshotSubvolumeName)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- sourceName, _, _ := containerGetParentAndSnapshotName(s.volume.Name)
- storageVolumeSnapshotPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceName)
- empty, err := shared.PathIsEmpty(storageVolumeSnapshotPath)
- if err == nil && empty {
- err := os.RemoveAll(storageVolumeSnapshotPath)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- }
-
- err = s.s.Cluster.StoragePoolVolumeDelete(
- "default",
- s.volume.Name,
- storagePoolVolumeTypeCustom,
- s.poolID)
- if err != nil {
- logger.Errorf(`Failed to delete database entry for BTRFS storage volume "%s" on storage pool "%s"`,
- s.volume.Name, s.pool.Name)
- }
-
- logger.Infof("Deleted BTRFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageBtrfs) StoragePoolVolumeSnapshotRename(newName string) error {
- logger.Infof("Renaming BTRFS storage volume on storage pool \"%s\" from \"%s\" to \"%s\"", s.pool.Name, s.volume.Name, newName)
- var fullSnapshotName string
-
- if shared.IsSnapshot(newName) {
- // When renaming volume snapshots, newName will contain the full snapshot name
- fullSnapshotName = newName
- } else {
- sourceName, _, ok := containerGetParentAndSnapshotName(s.volume.Name)
- if !ok {
- return fmt.Errorf("Not a snapshot name")
- }
-
- fullSnapshotName = fmt.Sprintf("%s%s%s", sourceName, shared.SnapshotDelimiter, newName)
- }
-
- oldPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- newPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, fullSnapshotName)
-
- if !shared.PathExists(newPath) {
- err := os.MkdirAll(newPath, customDirMode)
- if err != nil {
- return err
- }
- }
-
- err := os.Rename(oldPath, newPath)
- if err != nil {
- return err
- }
-
- logger.Infof("Renamed BTRFS storage volume on storage pool \"%s\" from \"%s\" to \"%s\"", s.pool.Name, s.volume.Name, newName)
-
- return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, fullSnapshotName, storagePoolVolumeTypeCustom, s.poolID)
-}
From c368aa9aa81d120176d1a051911594d458e1ae19 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 16:52:12 +0200
Subject: [PATCH 14/15] lxd: Remove old dir storage code
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage_dir.go | 1587 --------------------------------------------
1 file changed, 1587 deletions(-)
delete mode 100644 lxd/storage_dir.go
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
deleted file mode 100644
index ca1b0a6949..0000000000
--- a/lxd/storage_dir.go
+++ /dev/null
@@ -1,1587 +0,0 @@
-package main
-
-import (
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "path/filepath"
- "strings"
- "syscall"
-
- "github.com/gorilla/websocket"
- "github.com/pkg/errors"
-
- "github.com/lxc/lxd/lxd/migration"
- "github.com/lxc/lxd/lxd/storage/quota"
- "github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/api"
- "github.com/lxc/lxd/shared/ioprogress"
- "github.com/lxc/lxd/shared/logger"
-)
-
-type storageDir struct {
- storageShared
-
- volumeID int64
-}
-
-// Only initialize the minimal information we need about a given storage type.
-func (s *storageDir) StorageCoreInit() error {
- s.sType = storageTypeDir
- typeName, err := storageTypeToString(s.sType)
- if err != nil {
- return err
- }
- s.sTypeName = typeName
- s.sTypeVersion = "1"
-
- return nil
-}
-
-// Initialize a full storage interface.
-func (s *storageDir) StoragePoolInit() error {
- err := s.StorageCoreInit()
- if err != nil {
- return err
- }
-
- return nil
-}
-
-// Initialize a full storage interface.
-func (s *storageDir) StoragePoolCheck() error {
- logger.Debugf("Checking DIR storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolCreate() error {
- logger.Infof("Creating DIR storage pool \"%s\"", s.pool.Name)
-
- s.pool.Config["volatile.initial_source"] = s.pool.Config["source"]
-
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
-
- source := shared.HostPath(s.pool.Config["source"])
- if source == "" {
- source = filepath.Join(shared.VarPath("storage-pools"), s.pool.Name)
- s.pool.Config["source"] = source
- } else {
- cleanSource := filepath.Clean(source)
- lxdDir := shared.VarPath()
- if strings.HasPrefix(cleanSource, lxdDir) &&
- cleanSource != poolMntPoint {
- return fmt.Errorf(`DIR storage pool requests in LXD `+
- `directory "%s" are only valid under `+
- `"%s"\n(e.g. source=%s)`, shared.VarPath(),
- shared.VarPath("storage-pools"), poolMntPoint)
- }
- source = filepath.Clean(source)
- }
-
- revert := true
- if !shared.PathExists(source) {
- err := os.MkdirAll(source, 0711)
- if err != nil {
- return err
- }
- defer func() {
- if !revert {
- return
- }
- os.Remove(source)
- }()
- } else {
- empty, err := shared.PathIsEmpty(source)
- if err != nil {
- return err
- }
-
- if !empty {
- return fmt.Errorf("The provided directory is not empty")
- }
- }
-
- if !shared.PathExists(poolMntPoint) {
- err := os.MkdirAll(poolMntPoint, 0711)
- if err != nil {
- return err
- }
- defer func() {
- if !revert {
- return
- }
- os.Remove(poolMntPoint)
- }()
- }
-
- err := s.StoragePoolCheck()
- if err != nil {
- return err
- }
-
- _, err = s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- revert = false
-
- logger.Infof("Created DIR storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolDelete() error {
- logger.Infof("Deleting DIR storage pool \"%s\"", s.pool.Name)
-
- source := shared.HostPath(s.pool.Config["source"])
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- _, err := s.StoragePoolUmount()
- if err != nil {
- return err
- }
-
- if shared.PathExists(source) {
- err := os.RemoveAll(source)
- if err != nil {
- return err
- }
- }
-
- prefix := shared.VarPath("storage-pools")
- if !strings.HasPrefix(source, prefix) {
- storagePoolSymlink := getStoragePoolMountPoint(s.pool.Name)
- if !shared.PathExists(storagePoolSymlink) {
- return nil
- }
-
- err := os.Remove(storagePoolSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Infof("Deleted DIR storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolMount() (bool, error) {
- source := shared.HostPath(s.pool.Config["source"])
- if source == "" {
- return false, fmt.Errorf("no \"source\" property found for the storage pool")
- }
- cleanSource := filepath.Clean(source)
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- if cleanSource == poolMntPoint {
- return true, nil
- }
-
- logger.Debugf("Mounting DIR storage pool \"%s\"", s.pool.Name)
-
- poolMountLockID := getPoolMountLockID(s.pool.Name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolMountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in mounting the storage pool.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[poolMountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- removeLockFromMap := func() {
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolMountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, poolMountLockID)
- }
- lxdStorageMapLock.Unlock()
- }
- defer removeLockFromMap()
-
- mountSource := cleanSource
- mountFlags := syscall.MS_BIND
-
- if shared.IsMountPoint(poolMntPoint) {
- return false, nil
- }
-
- err := syscall.Mount(mountSource, poolMntPoint, "", uintptr(mountFlags), "")
- if err != nil {
- logger.Errorf(`Failed to mount DIR storage pool "%s" onto "%s": %s`, mountSource, poolMntPoint, err)
- return false, err
- }
-
- logger.Debugf("Mounted DIR storage pool \"%s\"", s.pool.Name)
-
- return true, nil
-}
-
-func (s *storageDir) StoragePoolUmount() (bool, error) {
- source := s.pool.Config["source"]
- if source == "" {
- return false, fmt.Errorf("no \"source\" property found for the storage pool")
- }
- cleanSource := filepath.Clean(source)
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- if cleanSource == poolMntPoint {
- return true, nil
- }
-
- logger.Debugf("Unmounting DIR storage pool \"%s\"", s.pool.Name)
-
- poolUmountLockID := getPoolUmountLockID(s.pool.Name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolUmountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in unmounting the storage pool.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[poolUmountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- removeLockFromMap := func() {
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[poolUmountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, poolUmountLockID)
- }
- lxdStorageMapLock.Unlock()
- }
-
- defer removeLockFromMap()
-
- if !shared.IsMountPoint(poolMntPoint) {
- return false, nil
- }
-
- err := syscall.Unmount(poolMntPoint, 0)
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Unmounted DIR pool \"%s\"", s.pool.Name)
- return true, nil
-}
-
-func (s *storageDir) GetContainerPoolInfo() (int64, string, string) {
- return s.poolID, s.pool.Name, s.pool.Name
-}
-
-func (s *storageDir) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error {
- logger.Infof(`Updating DIR storage pool "%s"`, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- changeable := changeableStoragePoolProperties["dir"]
- unchangeable := []string{}
- for _, change := range changedConfig {
- if !shared.StringInSlice(change, changeable) {
- unchangeable = append(unchangeable, change)
- }
- }
-
- if len(unchangeable) > 0 {
- return updateStoragePoolError(unchangeable, "dir")
- }
-
- // "rsync.bwlimit" requires no on-disk modifications.
-
- logger.Infof(`Updated DIR storage pool "%s"`, s.pool.Name)
- return nil
-}
-
-// Functions dealing with storage pools.
-func (s *storageDir) StoragePoolVolumeCreate() error {
- logger.Infof("Creating DIR storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- isSnapshot := shared.IsSnapshot(s.volume.Name)
-
- var storageVolumePath string
-
- if isSnapshot {
- storageVolumePath = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- } else {
- storageVolumePath = getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- }
-
- err = os.MkdirAll(storageVolumePath, 0711)
- if err != nil {
- return err
- }
-
- err = s.initQuota(storageVolumePath, s.volumeID)
- if err != nil {
- return err
- }
-
- logger.Infof("Created DIR storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolVolumeDelete() error {
- logger.Infof("Deleting DIR storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- storageVolumePath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- if !shared.PathExists(storageVolumePath) {
- return nil
- }
-
- err := s.deleteQuota(storageVolumePath, s.volumeID)
- if err != nil {
- return err
- }
-
- err = os.RemoveAll(storageVolumePath)
- if err != nil {
- return err
- }
-
- err = s.s.Cluster.StoragePoolVolumeDelete(
- "default",
- s.volume.Name,
- storagePoolVolumeTypeCustom,
- s.poolID)
- if err != nil {
- logger.Errorf(`Failed to delete database entry for DIR storage volume "%s" on storage pool "%s"`,
- s.volume.Name, s.pool.Name)
- }
-
- logger.Infof("Deleted DIR storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolVolumeMount() (bool, error) {
- return true, nil
-}
-
-func (s *storageDir) StoragePoolVolumeUmount() (bool, error) {
- return true, nil
-}
-
-func (s *storageDir) StoragePoolVolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error {
- if writable.Restore == "" {
- logger.Infof(`Updating DIR storage volume "%s"`, s.volume.Name)
- }
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- if writable.Restore != "" {
- logger.Infof(`Restoring DIR storage volume "%s" from snapshot "%s"`,
- s.volume.Name, writable.Restore)
-
- sourcePath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name,
- fmt.Sprintf("%s/%s", s.volume.Name, writable.Restore))
- targetPath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
-
- // Restore using rsync
- bwlimit := s.pool.Config["rsync.bwlimit"]
- output, err := rsyncLocalCopy(sourcePath, targetPath, bwlimit)
- if err != nil {
- return fmt.Errorf("failed to rsync container: %s: %s", string(output), err)
- }
-
- logger.Infof(`Restored DIR storage volume "%s" from snapshot "%s"`,
- s.volume.Name, writable.Restore)
- return nil
- }
-
- changeable := changeableStoragePoolVolumeProperties["dir"]
- unchangeable := []string{}
- for _, change := range changedConfig {
- if !shared.StringInSlice(change, changeable) {
- unchangeable = append(unchangeable, change)
- }
- }
-
- if len(unchangeable) > 0 {
- return updateStoragePoolVolumeError(unchangeable, "dir")
- }
-
- logger.Infof(`Updated DIR storage volume "%s"`, s.volume.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolVolumeRename(newName string) error {
- logger.Infof(`Renaming DIR storage volume on storage pool "%s" from "%s" to "%s`,
- s.pool.Name, s.volume.Name, newName)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- usedBy, err := storagePoolVolumeUsedByContainersGet(s.s, "default", s.volume.Name, storagePoolVolumeTypeNameCustom)
- if err != nil {
- return err
- }
- if len(usedBy) > 0 {
- return fmt.Errorf(`DIR storage volume "%s" on storage pool "%s" is attached to containers`,
- s.volume.Name, s.pool.Name)
- }
-
- oldPath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- newPath := getStoragePoolVolumeMountPoint(s.pool.Name, newName)
- err = os.Rename(oldPath, newPath)
- if err != nil {
- return err
- }
-
- logger.Infof(`Renamed DIR storage volume on storage pool "%s" from "%s" to "%s`,
- s.pool.Name, s.volume.Name, newName)
-
- return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, newName,
- storagePoolVolumeTypeCustom, s.poolID)
-}
-
-func (s *storageDir) ContainerStorageReady(container container) bool {
- containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- ok, _ := shared.PathIsEmpty(containerMntPoint)
- return !ok
-}
-
-func (s *storageDir) ContainerCreate(container container) error {
- logger.Debugf("Creating empty DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- err = createContainerMountpoint(containerMntPoint, container.Path(), container.IsPrivileged())
- if err != nil {
- return err
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- deleteContainerMountpoint(containerMntPoint, container.Path(), s.GetStorageTypeName())
- }()
-
- err = s.initQuota(containerMntPoint, s.volumeID)
- if err != nil {
- return err
- }
-
- err = container.TemplateApply("create")
- if err != nil {
- return errors.Wrap(err, "Apply template")
- }
-
- revert = false
-
- logger.Debugf("Created empty DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) ContainerCreateFromImage(container container, imageFingerprint string, tracker *ioprogress.ProgressTracker) error {
- logger.Debugf("Creating DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- privileged := container.IsPrivileged()
- containerName := container.Name()
- containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, containerName)
- err = createContainerMountpoint(containerMntPoint, container.Path(), privileged)
- if err != nil {
- return errors.Wrap(err, "Create container mount point")
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.ContainerDelete(container)
- }()
-
- err = s.initQuota(containerMntPoint, s.volumeID)
- if err != nil {
- return err
- }
-
- imagePath := shared.VarPath("images", imageFingerprint)
- err = unpackImage(imagePath, containerMntPoint, storageTypeDir, s.s.OS.RunningInUserNS, nil)
- if err != nil {
- return errors.Wrap(err, "Unpack image")
- }
-
- err = container.TemplateApply("create")
- if err != nil {
- return errors.Wrap(err, "Apply template")
- }
-
- revert = false
-
- logger.Debugf("Created DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) ContainerDelete(container container) error {
- logger.Debugf("Deleting DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Delete the container on its storage pool:
- // ${POOL}/containers/<container_name>
- containerName := container.Name()
- containerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, containerName)
-
- err = s.deleteQuota(containerMntPoint, s.volumeID)
- if err != nil {
- return err
- }
-
- if shared.PathExists(containerMntPoint) {
- err := os.RemoveAll(containerMntPoint)
- if err != nil {
- // RemovaAll fails on very long paths, so attempt an rm -Rf
- output, err := shared.RunCommand("rm", "-Rf", containerMntPoint)
- if err != nil {
- return fmt.Errorf("error removing %s: %s", containerMntPoint, output)
- }
- }
- }
-
- err = deleteContainerMountpoint(containerMntPoint, container.Path(), s.GetStorageTypeName())
- if err != nil {
- return err
- }
-
- // Delete potential leftover snapshot mountpoints.
- snapshotMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- if shared.PathExists(snapshotMntPoint) {
- err := os.RemoveAll(snapshotMntPoint)
- if err != nil {
- return err
- }
- }
-
- // Delete potential leftover snapshot symlinks:
- // ${LXD_DIR}/snapshots/<container_name> to ${POOL}/snapshots/<container_name>
- snapshotSymlink := shared.VarPath("snapshots", projectPrefix(container.Project(), container.Name()))
- if shared.PathExists(snapshotSymlink) {
- err := os.Remove(snapshotSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Deleted DIR storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) copyContainer(target container, source container) error {
- _, sourcePool, _ := source.Storage().GetContainerPoolInfo()
- _, targetPool, _ := target.Storage().GetContainerPoolInfo()
- sourceContainerMntPoint := getContainerMountPoint(source.Project(), sourcePool, source.Name())
- if source.IsSnapshot() {
- sourceContainerMntPoint = getSnapshotMountPoint(source.Project(), sourcePool, source.Name())
- }
- targetContainerMntPoint := getContainerMountPoint(target.Project(), targetPool, target.Name())
-
- err := createContainerMountpoint(targetContainerMntPoint, target.Path(), target.IsPrivileged())
- if err != nil {
- return err
- }
-
- err = s.initQuota(targetContainerMntPoint, s.volumeID)
- if err != nil {
- return err
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
- output, err := rsyncLocalCopy(sourceContainerMntPoint, targetContainerMntPoint, bwlimit)
- if err != nil {
- return fmt.Errorf("failed to rsync container: %s: %s", string(output), err)
- }
-
- err = target.TemplateApply("copy")
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) copySnapshot(target container, targetPool string, source container, sourcePool string) error {
- sourceName := source.Name()
- targetName := target.Name()
- sourceContainerMntPoint := getSnapshotMountPoint(source.Project(), sourcePool, sourceName)
- targetContainerMntPoint := getSnapshotMountPoint(target.Project(), targetPool, targetName)
-
- targetParentName, _, _ := containerGetParentAndSnapshotName(target.Name())
- containersPath := getSnapshotMountPoint(target.Project(), targetPool, targetParentName)
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", targetPool, "containers-snapshots", projectPrefix(target.Project(), targetParentName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(target.Project(), targetParentName))
- err := createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
- output, err := rsyncLocalCopy(sourceContainerMntPoint, targetContainerMntPoint, bwlimit)
- if err != nil {
- return fmt.Errorf("failed to rsync container: %s: %s", string(output), err)
- }
-
- return nil
-}
-
-func (s *storageDir) ContainerCopy(target container, source container, containerOnly bool) error {
- logger.Debugf("Copying DIR container storage %s to %s", source.Name(), target.Name())
-
- err := s.doContainerCopy(target, source, containerOnly, false, nil)
- if err != nil {
- return err
- }
-
- logger.Debugf("Copied DIR container storage %s to %s", source.Name(), target.Name())
- return nil
-}
-
-func (s *storageDir) doContainerCopy(target container, source container, containerOnly bool, refresh bool, refreshSnapshots []container) error {
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- sourcePool, err := source.StoragePool()
- if err != nil {
- return err
- }
- targetPool, err := target.StoragePool()
- if err != nil {
- return err
- }
-
- srcState := s.s
- if sourcePool != targetPool {
- // setup storage for the source volume
- srcStorage, err := storagePoolVolumeInit(s.s, "default", sourcePool, source.Name(), storagePoolVolumeTypeContainer)
- if err != nil {
- return err
- }
-
- ourMount, err := srcStorage.StoragePoolMount()
- if err != nil {
- return err
- }
- if ourMount {
- defer srcStorage.StoragePoolUmount()
- }
- srcState = srcStorage.GetState()
- }
-
- err = s.copyContainer(target, source)
- if err != nil {
- return err
- }
-
- if containerOnly {
- return nil
- }
-
- var snapshots []container
-
- if refresh {
- snapshots = refreshSnapshots
- } else {
- snapshots, err = source.Snapshots()
- if err != nil {
- return err
- }
- }
-
- if len(snapshots) == 0 {
- return nil
- }
-
- for _, snap := range snapshots {
- sourceSnapshot, err := containerLoadByProjectAndName(srcState, source.Project(), snap.Name())
- if err != nil {
- return err
- }
-
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
- newSnapName := fmt.Sprintf("%s/%s", target.Name(), snapOnlyName)
- targetSnapshot, err := containerLoadByProjectAndName(s.s, source.Project(), newSnapName)
- if err != nil {
- return err
- }
-
- err = s.copySnapshot(targetSnapshot, targetPool, sourceSnapshot, sourcePool)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func (s *storageDir) ContainerRefresh(target container, source container, snapshots []container) error {
- logger.Debugf("Refreshing DIR container storage for %s from %s", target.Name(), source.Name())
-
- err := s.doContainerCopy(target, source, len(snapshots) == 0, true, snapshots)
- if err != nil {
- return err
- }
-
- logger.Debugf("Refreshed DIR container storage for %s from %s", target.Name(), source.Name())
- return nil
-}
-
-func (s *storageDir) ContainerMount(c container) (bool, error) {
- return s.StoragePoolMount()
-}
-
-func (s *storageDir) ContainerUmount(c container, path string) (bool, error) {
- return true, nil
-}
-
-func (s *storageDir) ContainerRename(container container, newName string) error {
- logger.Debugf("Renaming DIR storage volume for container \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- oldContainerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- oldContainerSymlink := containerPath(container.Project(), container.Name(), false)
- newContainerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, newName)
- newContainerSymlink := containerPath(container.Project(), newName, false)
-
- err = renameContainerMountpoint(oldContainerMntPoint, oldContainerSymlink, newContainerMntPoint, newContainerSymlink)
- if err != nil {
- return err
- }
-
- // Rename the snapshot mountpoint for the container if existing:
- // ${POOL}/snapshots/<old_container_name> to ${POOL}/snapshots/<new_container_name>
- oldSnapshotsMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- newSnapshotsMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, newName)
- if shared.PathExists(oldSnapshotsMntPoint) {
- err = os.Rename(oldSnapshotsMntPoint, newSnapshotsMntPoint)
- if err != nil {
- return err
- }
- }
-
- // Remove the old snapshot symlink:
- // ${LXD_DIR}/snapshots/<old_container_name>
- oldSnapshotSymlink := shared.VarPath("snapshots", projectPrefix(container.Project(), container.Name()))
- newSnapshotSymlink := shared.VarPath("snapshots", projectPrefix(container.Project(), newName))
- if shared.PathExists(oldSnapshotSymlink) {
- err := os.Remove(oldSnapshotSymlink)
- if err != nil {
- return err
- }
-
- // Create the new snapshot symlink:
- // ${LXD_DIR}/snapshots/<new_container_name> to ${POOL}/snapshots/<new_container_name>
- err = os.Symlink(newSnapshotsMntPoint, newSnapshotSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Renamed DIR storage volume for container \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
- return nil
-}
-
-func (s *storageDir) ContainerRestore(container container, sourceContainer container) error {
- logger.Debugf("Restoring DIR storage volume for container \"%s\" from %s to %s", s.volume.Name, sourceContainer.Name(), container.Name())
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- targetPath := container.Path()
- sourcePath := sourceContainer.Path()
-
- // Restore using rsync
- bwlimit := s.pool.Config["rsync.bwlimit"]
- output, err := rsyncLocalCopy(sourcePath, targetPath, bwlimit)
- if err != nil {
- return fmt.Errorf("failed to rsync container: %s: %s", string(output), err)
- }
-
- logger.Debugf("Restored DIR storage volume for container \"%s\" from %s to %s", s.volume.Name, sourceContainer.Name(), container.Name())
- return nil
-}
-
-func (s *storageDir) ContainerGetUsage(c container) (int64, error) {
- path := getContainerMountPoint(c.Project(), s.pool.Name, c.Name())
-
- ok, err := quota.Supported(path)
- if err != nil || !ok {
- return -1, fmt.Errorf("The backing filesystem doesn't support quotas")
- }
-
- projectID := uint32(s.volumeID + 10000)
- size, err := quota.GetProjectUsage(path, projectID)
- if err != nil {
- return -1, err
- }
-
- return size, nil
-}
-
-func (s *storageDir) ContainerSnapshotCreate(snapshotContainer container, sourceContainer container) error {
- logger.Debugf("Creating DIR storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Create the path for the snapshot.
- targetContainerName := snapshotContainer.Name()
- targetContainerMntPoint := getSnapshotMountPoint(sourceContainer.Project(), s.pool.Name, targetContainerName)
- err = os.MkdirAll(targetContainerMntPoint, 0711)
- if err != nil {
- return err
- }
-
- rsync := func(snapshotContainer container, oldPath string, newPath string, bwlimit string) error {
- output, err := rsyncLocalCopy(oldPath, newPath, bwlimit)
- if err != nil {
- s.ContainerDelete(snapshotContainer)
- return fmt.Errorf("failed to rsync: %s: %s", string(output), err)
- }
- return nil
- }
-
- ourStart, err := sourceContainer.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer sourceContainer.StorageStop()
- }
-
- _, sourcePool, _ := sourceContainer.Storage().GetContainerPoolInfo()
- sourceContainerName := sourceContainer.Name()
- sourceContainerMntPoint := getContainerMountPoint(sourceContainer.Project(), sourcePool, sourceContainerName)
- bwlimit := s.pool.Config["rsync.bwlimit"]
- err = rsync(snapshotContainer, sourceContainerMntPoint, targetContainerMntPoint, bwlimit)
- if err != nil {
- return err
- }
-
- if sourceContainer.IsRunning() {
- // This is done to ensure consistency when snapshotting. But we
- // probably shouldn't fail just because of that.
- logger.Debugf("Trying to freeze and rsync again to ensure consistency")
-
- err := sourceContainer.Freeze()
- if err != nil {
- logger.Errorf("Trying to freeze and rsync again failed")
- goto onSuccess
- }
- defer sourceContainer.Unfreeze()
-
- err = rsync(snapshotContainer, sourceContainerMntPoint, targetContainerMntPoint, bwlimit)
- if err != nil {
- return err
- }
- }
-
-onSuccess:
- // Check if the symlink
- // ${LXD_DIR}/snapshots/<source_container_name> to ${POOL_PATH}/snapshots/<source_container_name>
- // exists and if not create it.
- sourceContainerSymlink := shared.VarPath("snapshots", projectPrefix(sourceContainer.Project(), sourceContainerName))
- sourceContainerSymlinkTarget := getSnapshotMountPoint(sourceContainer.Project(), sourcePool, sourceContainerName)
- if !shared.PathExists(sourceContainerSymlink) {
- err = os.Symlink(sourceContainerSymlinkTarget, sourceContainerSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Created DIR storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) ContainerSnapshotCreateEmpty(snapshotContainer container) error {
- logger.Debugf("Creating empty DIR storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Create the path for the snapshot.
- targetContainerName := snapshotContainer.Name()
- targetContainerMntPoint := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, targetContainerName)
- err = os.MkdirAll(targetContainerMntPoint, 0711)
- if err != nil {
- return err
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.ContainerSnapshotDelete(snapshotContainer)
- }()
-
- // Check if the symlink
- // ${LXD_DIR}/snapshots/<source_container_name> to ${POOL_PATH}/snapshots/<source_container_name>
- // exists and if not create it.
- targetContainerMntPoint = getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name,
- targetContainerName)
- sourceName, _, _ := containerGetParentAndSnapshotName(targetContainerName)
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools",
- s.pool.Name, "containers-snapshots", projectPrefix(snapshotContainer.Project(), sourceName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(snapshotContainer.Project(), sourceName))
- err = createSnapshotMountpoint(targetContainerMntPoint,
- snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
-
- revert = false
-
- logger.Debugf("Created empty DIR storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func dirSnapshotDeleteInternal(project, poolName string, snapshotName string) error {
- snapshotContainerMntPoint := getSnapshotMountPoint(project, poolName, snapshotName)
- if shared.PathExists(snapshotContainerMntPoint) {
- err := os.RemoveAll(snapshotContainerMntPoint)
- if err != nil {
- return err
- }
- }
-
- sourceContainerName, _, _ := containerGetParentAndSnapshotName(snapshotName)
- snapshotContainerPath := getSnapshotMountPoint(project, poolName, sourceContainerName)
- empty, _ := shared.PathIsEmpty(snapshotContainerPath)
- if empty == true {
- err := os.Remove(snapshotContainerPath)
- if err != nil {
- return err
- }
-
- snapshotSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceContainerName))
- if shared.PathExists(snapshotSymlink) {
- err := os.Remove(snapshotSymlink)
- if err != nil {
- return err
- }
- }
- }
-
- return nil
-}
-
-func (s *storageDir) ContainerSnapshotDelete(snapshotContainer container) error {
- logger.Debugf("Deleting DIR storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- snapshotContainerName := snapshotContainer.Name()
- err = dirSnapshotDeleteInternal(snapshotContainer.Project(), s.pool.Name, snapshotContainerName)
- if err != nil {
- return err
- }
-
- logger.Debugf("Deleted DIR storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) ContainerSnapshotRename(snapshotContainer container, newName string) error {
- logger.Debugf("Renaming DIR storage volume for snapshot \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- // Rename the mountpoint for the snapshot:
- // ${POOL}/snapshots/<old_snapshot_name> to ${POOL}/snapshots/<new_snapshot_name>
- oldSnapshotMntPoint := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, snapshotContainer.Name())
- newSnapshotMntPoint := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, newName)
- err = os.Rename(oldSnapshotMntPoint, newSnapshotMntPoint)
- if err != nil {
- return err
- }
-
- logger.Debugf("Renamed DIR storage volume for snapshot \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
- return nil
-}
-
-func (s *storageDir) ContainerSnapshotStart(container container) (bool, error) {
- return s.StoragePoolMount()
-}
-
-func (s *storageDir) ContainerSnapshotStop(container container) (bool, error) {
- return true, nil
-}
-
-func (s *storageDir) ContainerBackupCreate(backup backup, source container) error {
- // Start storage
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- // Create a temporary path for the backup
- tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_")
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpPath)
-
- // Prepare for rsync
- rsync := func(oldPath string, newPath string, bwlimit string) error {
- output, err := rsyncLocalCopy(oldPath, newPath, bwlimit)
- if err != nil {
- return fmt.Errorf("Failed to rsync: %s: %s", string(output), err)
- }
-
- return nil
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- // Handle snapshots
- if !backup.containerOnly {
- snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath)
-
- // Retrieve the snapshots
- snapshots, err := source.Snapshots()
- if err != nil {
- return err
- }
-
- // Create the snapshot path
- if len(snapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return err
- }
- }
-
- for _, snap := range snapshots {
- _, snapName, _ := containerGetParentAndSnapshotName(snap.Name())
- snapshotMntPoint := getSnapshotMountPoint(snap.Project(), s.pool.Name, snap.Name())
- target := fmt.Sprintf("%s/%s", snapshotsPath, snapName)
-
- // Copy the snapshot
- err = rsync(snapshotMntPoint, target, bwlimit)
- if err != nil {
- return err
- }
- }
- }
-
- if source.IsRunning() {
- // This is done to ensure consistency when snapshotting. But we
- // probably shouldn't fail just because of that.
- logger.Debugf("Freezing container '%s' for backup", source.Name())
-
- err := source.Freeze()
- if err != nil {
- logger.Errorf("Failed to freeze container '%s' for backup: %v", source.Name(), err)
- }
- defer source.Unfreeze()
- }
-
- // Copy the container
- containerPath := fmt.Sprintf("%s/container", tmpPath)
- err = rsync(source.Path(), containerPath, bwlimit)
- if err != nil {
- return err
- }
-
- // Pack the backup
- err = backupCreateTarball(s.s, tmpPath, backup)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- // Create mountpoints
- containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, info.Name)
- err = createContainerMountpoint(containerMntPoint, containerPath(info.Project, info.Name, false), info.Privileged)
- if err != nil {
- return errors.Wrap(err, "Create container mount point")
- }
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--strip-components=2",
- "--xattrs-include=*",
- "-C", containerMntPoint, "backup/container",
- }...)
-
- // Extract container
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- return err
- }
-
- if len(info.Snapshots) > 0 {
- // Create mountpoints
- snapshotMntPoint := getSnapshotMountPoint(info.Project, s.pool.Name, info.Name)
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name,
- "containers-snapshots", projectPrefix(info.Project, info.Name))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(info.Project, info.Name))
- err := createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget,
- snapshotMntPointSymlink)
- if err != nil {
- return err
- }
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--strip-components=2",
- "--xattrs-include=*",
- "-C", snapshotMntPoint, "backup/snapshots",
- }...)
-
- // Extract snapshots
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func (s *storageDir) ImageCreate(fingerprint string, tracker *ioprogress.ProgressTracker) error {
- return nil
-}
-
-func (s *storageDir) ImageDelete(fingerprint string) error {
- err := s.deleteImageDbPoolVolume(fingerprint)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) ImageMount(fingerprint string) (bool, error) {
- return true, nil
-}
-
-func (s *storageDir) ImageUmount(fingerprint string) (bool, error) {
- return true, nil
-}
-
-func (s *storageDir) MigrationType() migration.MigrationFSType {
- return migration.MigrationFSType_RSYNC
-}
-
-func (s *storageDir) PreservesInodes() bool {
- return false
-}
-
-func (s *storageDir) MigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
- return rsyncMigrationSource(args)
-}
-
-func (s *storageDir) MigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
- return rsyncMigrationSink(conn, op, args)
-}
-
-func (s *storageDir) StorageEntitySetQuota(volumeType int, size int64, data interface{}) error {
- var path string
- switch volumeType {
- case storagePoolVolumeTypeContainer:
- c := data.(container)
- path = getContainerMountPoint(c.Project(), s.pool.Name, c.Name())
- case storagePoolVolumeTypeCustom:
- path = getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- }
-
- ok, err := quota.Supported(path)
- if err != nil || !ok {
- logger.Warnf("Skipping setting disk quota for '%s' as the underlying filesystem doesn't support them", s.volume.Name)
- return nil
- }
-
- projectID := uint32(s.volumeID + 10000)
- err = quota.SetProjectQuota(path, projectID, size)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) initQuota(path string, id int64) error {
- if s.volumeID == 0 {
- return fmt.Errorf("Missing volume ID")
- }
-
- ok, err := quota.Supported(path)
- if err != nil || !ok {
- return nil
- }
-
- projectID := uint32(s.volumeID + 10000)
- err = quota.SetProject(path, projectID)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) deleteQuota(path string, id int64) error {
- if s.volumeID == 0 {
- return fmt.Errorf("Missing volume ID")
- }
-
- ok, err := quota.Supported(path)
- if err != nil || !ok {
- return nil
- }
-
- err = quota.SetProject(path, 0)
- if err != nil {
- return err
- }
-
- projectID := uint32(s.volumeID + 10000)
- err = quota.SetProjectQuota(path, projectID, 0)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) StoragePoolResources() (*api.ResourcesStoragePool, error) {
- _, err := s.StoragePoolMount()
- if err != nil {
- return nil, err
- }
-
- poolMntPoint := getStoragePoolMountPoint(s.pool.Name)
-
- return storageResource(poolMntPoint)
-}
-
-func (s *storageDir) StoragePoolVolumeCopy(source *api.StorageVolumeSource) error {
- logger.Infof("Copying DIR storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
- successMsg := fmt.Sprintf("Copied DIR storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
-
- if s.pool.Name != source.Pool {
- // setup storage for the source volume
- srcStorage, err := storagePoolVolumeInit(s.s, "default", source.Pool, source.Name, storagePoolVolumeTypeCustom)
- if err != nil {
- logger.Errorf("Failed to initialize DIR storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- ourMount, err := srcStorage.StoragePoolMount()
- if err != nil {
- logger.Errorf("Failed to mount DIR storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
- if ourMount {
- defer srcStorage.StoragePoolUmount()
- }
- }
-
- err := s.copyVolume(source.Pool, source.Name, s.volume.Name)
- if err != nil {
- return err
- }
-
- if source.VolumeOnly {
- logger.Infof(successMsg)
- return nil
- }
-
- snapshots, err := storagePoolVolumeSnapshotsGet(s.s, source.Pool, source.Name, storagePoolVolumeTypeCustom)
- if err != nil {
- return err
- }
-
- for _, snap := range snapshots {
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap)
- err = s.copyVolumeSnapshot(source.Pool, snap, fmt.Sprintf("%s/%s", s.volume.Name, snapOnlyName))
- if err != nil {
- return err
- }
- }
-
- logger.Infof(successMsg)
- return nil
-}
-
-func (s *storageDir) StorageMigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
- return rsyncStorageMigrationSource(args)
-}
-
-func (s *storageDir) StorageMigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
- return rsyncStorageMigrationSink(conn, op, args)
-}
-
-func (s *storageDir) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
- logger.Infof("Creating DIR storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- _, err := s.StoragePoolMount()
- if err != nil {
- return err
- }
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- sourceName, _, ok := containerGetParentAndSnapshotName(target.Name)
- if !ok {
- return fmt.Errorf("Not a snapshot name")
- }
-
- targetPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target.Name)
- err = os.MkdirAll(targetPath, 0711)
- if err != nil {
- return err
- }
-
- sourcePath := getStoragePoolVolumeMountPoint(s.pool.Name, sourceName)
- bwlimit := s.pool.Config["rsync.bwlimit"]
- msg, err := rsyncLocalCopy(sourcePath, targetPath, bwlimit)
- if err != nil {
- return fmt.Errorf("Failed to rsync: %s: %s", string(msg), err)
- }
-
- logger.Infof("Created DIR storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolVolumeSnapshotDelete() error {
- logger.Infof("Deleting DIR storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- storageVolumePath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- err := os.RemoveAll(storageVolumePath)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- sourceName, _, _ := containerGetParentAndSnapshotName(s.volume.Name)
- storageVolumeSnapshotPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceName)
- empty, err := shared.PathIsEmpty(storageVolumeSnapshotPath)
- if err == nil && empty {
- os.RemoveAll(storageVolumeSnapshotPath)
- }
-
- err = s.s.Cluster.StoragePoolVolumeDelete(
- "default",
- s.volume.Name,
- storagePoolVolumeTypeCustom,
- s.poolID)
- if err != nil {
- logger.Errorf(`Failed to delete database entry for DIR storage volume "%s" on storage pool "%s"`,
- s.volume.Name, s.pool.Name)
- }
-
- logger.Infof("Deleted DIR storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageDir) StoragePoolVolumeSnapshotRename(newName string) error {
- logger.Infof("Renaming DIR storage volume on storage pool \"%s\" from \"%s\" to \"%s\"", s.pool.Name, s.volume.Name, newName)
- var fullSnapshotName string
-
- if shared.IsSnapshot(newName) {
- // When renaming volume snapshots, newName will contain the full snapshot name
- fullSnapshotName = newName
- } else {
- sourceName, _, ok := containerGetParentAndSnapshotName(s.volume.Name)
- if !ok {
- return fmt.Errorf("Not a snapshot name")
- }
-
- fullSnapshotName = fmt.Sprintf("%s%s%s", sourceName, shared.SnapshotDelimiter, newName)
- }
-
- oldPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- newPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, fullSnapshotName)
-
- if !shared.PathExists(newPath) {
- err := os.MkdirAll(newPath, customDirMode)
- if err != nil {
- return err
- }
- }
-
- err := os.Rename(oldPath, newPath)
- if err != nil {
- return err
- }
-
- logger.Infof("Renamed DIR storage volume on storage pool \"%s\" from \"%s\" to \"%s\"", s.pool.Name, s.volume.Name, newName)
-
- return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, fullSnapshotName, storagePoolVolumeTypeCustom, s.poolID)
-}
-
-func (s *storageDir) copyVolume(sourcePool string, source string, target string) error {
- var srcMountPoint string
-
- if shared.IsSnapshot(source) {
- srcMountPoint = getStoragePoolVolumeSnapshotMountPoint(sourcePool, source)
- } else {
- srcMountPoint = getStoragePoolVolumeMountPoint(sourcePool, source)
- }
-
- dstMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, target)
-
- err := os.MkdirAll(dstMountPoint, 0711)
- if err != nil {
- return err
- }
-
- err = s.initQuota(dstMountPoint, s.volumeID)
- if err != nil {
- return err
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- _, err = rsyncLocalCopy(srcMountPoint, dstMountPoint, bwlimit)
- if err != nil {
- os.RemoveAll(dstMountPoint)
- logger.Errorf("Failed to rsync into DIR storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- return nil
-}
-
-func (s *storageDir) copyVolumeSnapshot(sourcePool string, source string, target string) error {
- srcMountPoint := getStoragePoolVolumeSnapshotMountPoint(sourcePool, source)
- dstMountPoint := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target)
-
- err := os.MkdirAll(dstMountPoint, 0711)
- if err != nil {
- return err
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- _, err = rsyncLocalCopy(srcMountPoint, dstMountPoint, bwlimit)
- if err != nil {
- os.RemoveAll(dstMountPoint)
- logger.Errorf("Failed to rsync into DIR storage volume \"%s\" on storage pool \"%s\": %s", target, s.pool.Name, err)
- return err
- }
-
- return nil
-}
From c6f2b30886631664c2bd187816ad7bfced89c404 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 May 2019 16:52:30 +0200
Subject: [PATCH 15/15] lxd: Remove old zfs storage code
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage_zfs.go | 3442 --------------------------------------
lxd/storage_zfs_utils.go | 833 ---------
2 files changed, 4275 deletions(-)
delete mode 100644 lxd/storage_zfs.go
delete mode 100644 lxd/storage_zfs_utils.go
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
deleted file mode 100644
index 93c60f13d0..0000000000
--- a/lxd/storage_zfs.go
+++ /dev/null
@@ -1,3442 +0,0 @@
-package main
-
-import (
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "os/exec"
- "path/filepath"
- "strconv"
- "strings"
- "syscall"
-
- "github.com/gorilla/websocket"
- "github.com/pkg/errors"
-
- "github.com/lxc/lxd/lxd/migration"
- "github.com/lxc/lxd/lxd/util"
- "github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/api"
- "github.com/lxc/lxd/shared/ioprogress"
- "github.com/lxc/lxd/shared/logger"
-
- "github.com/pborman/uuid"
-)
-
-// Global defaults
-var zfsUseRefquota = "false"
-var zfsRemoveSnapshots = "false"
-
-// Cache
-var zfsVersion = ""
-
-type storageZfs struct {
- dataset string
- storageShared
-}
-
-func (s *storageZfs) getOnDiskPoolName() string {
- if s.dataset != "" {
- return s.dataset
- }
-
- return s.pool.Name
-}
-
-// Only initialize the minimal information we need about a given storage type.
-func (s *storageZfs) StorageCoreInit() error {
- s.sType = storageTypeZfs
- typeName, err := storageTypeToString(s.sType)
- if err != nil {
- return err
- }
- s.sTypeName = typeName
-
- if zfsVersion != "" {
- s.sTypeVersion = zfsVersion
- return nil
- }
-
- util.LoadModule("zfs")
-
- if !zfsIsEnabled() {
- return fmt.Errorf("The \"zfs\" tool is not enabled")
- }
-
- s.sTypeVersion, err = zfsToolVersionGet()
- if err != nil {
- s.sTypeVersion, err = zfsModuleVersionGet()
- if err != nil {
- return err
- }
- }
-
- zfsVersion = s.sTypeVersion
-
- return nil
-}
-
-// Functions dealing with storage pools.
-func (s *storageZfs) StoragePoolInit() error {
- err := s.StorageCoreInit()
- if err != nil {
- return err
- }
-
- // Detect whether we have been given a zfs dataset as source.
- if s.pool.Config["zfs.pool_name"] != "" {
- s.dataset = s.pool.Config["zfs.pool_name"]
- }
-
- return nil
-}
-
-func (s *storageZfs) StoragePoolCheck() error {
- logger.Debugf("Checking ZFS storage pool \"%s\"", s.pool.Name)
-
- source := s.pool.Config["source"]
- if source == "" {
- return fmt.Errorf("no \"source\" property found for the storage pool")
- }
-
- poolName := s.getOnDiskPoolName()
- purePoolName := strings.Split(poolName, "/")[0]
- exists := zfsFilesystemEntityExists(purePoolName, "")
- if exists {
- return nil
- }
-
- logger.Debugf("ZFS storage pool \"%s\" does not exist, trying to import it", poolName)
-
- var err error
- var msg string
- if filepath.IsAbs(source) {
- disksPath := shared.VarPath("disks")
- msg, err = shared.RunCommand("zpool", "import", "-d", disksPath, poolName)
- } else {
- msg, err = shared.RunCommand("zpool", "import", purePoolName)
- }
-
- if err != nil {
- return fmt.Errorf("ZFS storage pool \"%s\" could not be imported: %s", poolName, msg)
- }
-
- logger.Debugf("ZFS storage pool \"%s\" successfully imported", poolName)
- return nil
-}
-
-func (s *storageZfs) StoragePoolCreate() error {
- logger.Infof("Creating ZFS storage pool \"%s\"", s.pool.Name)
-
- err := s.zfsPoolCreate()
- if err != nil {
- return err
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.StoragePoolDelete()
- }()
-
- storagePoolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- err = os.MkdirAll(storagePoolMntPoint, 0711)
- if err != nil {
- return err
- }
-
- err = s.StoragePoolCheck()
- if err != nil {
- return err
- }
-
- revert = false
-
- logger.Infof("Created ZFS storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) zfsPoolCreate() error {
- s.pool.Config["volatile.initial_source"] = s.pool.Config["source"]
-
- zpoolName := s.getOnDiskPoolName()
- vdev := s.pool.Config["source"]
- defaultVdev := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", s.pool.Name))
- if vdev == "" || vdev == defaultVdev {
- vdev = defaultVdev
- s.pool.Config["source"] = vdev
-
- if s.pool.Config["zfs.pool_name"] == "" {
- s.pool.Config["zfs.pool_name"] = zpoolName
- }
-
- f, err := os.Create(vdev)
- if err != nil {
- return fmt.Errorf("Failed to open %s: %s", vdev, err)
- }
- defer f.Close()
-
- err = f.Chmod(0600)
- if err != nil {
- return fmt.Errorf("Failed to chmod %s: %s", vdev, err)
- }
-
- size, err := shared.ParseByteSizeString(s.pool.Config["size"])
- if err != nil {
- return err
- }
- err = f.Truncate(size)
- if err != nil {
- return fmt.Errorf("Failed to create sparse file %s: %s", vdev, err)
- }
-
- err = zfsPoolCreate(zpoolName, vdev)
- if err != nil {
- return err
- }
- } else {
- // Unset size property since it doesn't make sense.
- s.pool.Config["size"] = ""
-
- if filepath.IsAbs(vdev) {
- if !shared.IsBlockdevPath(vdev) {
- return fmt.Errorf("Custom loop file locations are not supported")
- }
-
- if s.pool.Config["zfs.pool_name"] == "" {
- s.pool.Config["zfs.pool_name"] = zpoolName
- }
-
- // This is a block device. Note, that we do not store the
- // block device path or UUID or PARTUUID or similar in
- // the database. All of those might change or might be
- // used in a special way (For example, zfs uses a single
- // UUID in a multi-device pool for all devices.). The
- // safest way is to just store the name of the zfs pool
- // we create.
- s.pool.Config["source"] = zpoolName
- err := zfsPoolCreate(zpoolName, vdev)
- if err != nil {
- return err
- }
- } else {
- if s.pool.Config["zfs.pool_name"] != "" && s.pool.Config["zfs.pool_name"] != vdev {
- return fmt.Errorf("Invalid combination of \"source\" and \"zfs.pool_name\" property")
- }
-
- s.pool.Config["zfs.pool_name"] = vdev
- s.dataset = vdev
-
- if strings.Contains(vdev, "/") {
- if !zfsFilesystemEntityExists(vdev, "") {
- err := zfsPoolCreate("", vdev)
- if err != nil {
- return err
- }
- }
- } else {
- err := zfsPoolCheck(vdev)
- if err != nil {
- return err
- }
- }
-
- subvols, err := zfsPoolListSubvolumes(zpoolName, vdev)
- if err != nil {
- return err
- }
-
- if len(subvols) > 0 {
- return fmt.Errorf("Provided ZFS pool (or dataset) isn't empty")
- }
-
- err = zfsPoolApplyDefaults(vdev)
- if err != nil {
- return err
- }
- }
- }
-
- // Create default dummy datasets to avoid zfs races during container
- // creation.
- poolName := s.getOnDiskPoolName()
- dataset := fmt.Sprintf("%s/containers", poolName)
- msg, err := zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create containers dataset: %s", msg)
- return err
- }
-
- fixperms := shared.VarPath("storage-pools", s.pool.Name, "containers")
- err = os.MkdirAll(fixperms, containersDirMode)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- err = os.Chmod(fixperms, containersDirMode)
- if err != nil {
- logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(containersDirMode), 8), err)
- }
-
- dataset = fmt.Sprintf("%s/images", poolName)
- msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create images dataset: %s", msg)
- return err
- }
-
- fixperms = shared.VarPath("storage-pools", s.pool.Name, "images")
- err = os.MkdirAll(fixperms, imagesDirMode)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- err = os.Chmod(fixperms, imagesDirMode)
- if err != nil {
- logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(imagesDirMode), 8), err)
- }
-
- dataset = fmt.Sprintf("%s/custom", poolName)
- msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create custom dataset: %s", msg)
- return err
- }
-
- fixperms = shared.VarPath("storage-pools", s.pool.Name, "custom")
- err = os.MkdirAll(fixperms, customDirMode)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- err = os.Chmod(fixperms, customDirMode)
- if err != nil {
- logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(customDirMode), 8), err)
- }
-
- dataset = fmt.Sprintf("%s/deleted", poolName)
- msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create deleted dataset: %s", msg)
- return err
- }
-
- dataset = fmt.Sprintf("%s/snapshots", poolName)
- msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create snapshots dataset: %s", msg)
- return err
- }
-
- fixperms = shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots")
- err = os.MkdirAll(fixperms, snapshotsDirMode)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- err = os.Chmod(fixperms, snapshotsDirMode)
- if err != nil {
- logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(snapshotsDirMode), 8), err)
- }
-
- dataset = fmt.Sprintf("%s/custom-snapshots", poolName)
- msg, err = zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create snapshots dataset: %s", msg)
- return err
- }
-
- fixperms = shared.VarPath("storage-pools", s.pool.Name, "custom-snapshots")
- err = os.MkdirAll(fixperms, snapshotsDirMode)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
- err = os.Chmod(fixperms, snapshotsDirMode)
- if err != nil {
- logger.Warnf("Failed to chmod \"%s\" to \"0%s\": %s", fixperms, strconv.FormatInt(int64(snapshotsDirMode), 8), err)
- }
-
- return nil
-}
-
-func (s *storageZfs) StoragePoolDelete() error {
- logger.Infof("Deleting ZFS storage pool \"%s\"", s.pool.Name)
-
- poolName := s.getOnDiskPoolName()
- if zfsFilesystemEntityExists(poolName, "") {
- err := zfsFilesystemEntityDelete(s.pool.Config["source"], poolName)
- if err != nil {
- return err
- }
- }
-
- storagePoolMntPoint := getStoragePoolMountPoint(s.pool.Name)
- if shared.PathExists(storagePoolMntPoint) {
- err := os.RemoveAll(storagePoolMntPoint)
- if err != nil {
- return err
- }
- }
-
- logger.Infof("Deleted ZFS storage pool \"%s\"", s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolMount() (bool, error) {
- return true, nil
-}
-
-func (s *storageZfs) StoragePoolUmount() (bool, error) {
- return true, nil
-}
-
-func (s *storageZfs) StoragePoolVolumeCreate() error {
- logger.Infof("Creating ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- isSnapshot := shared.IsSnapshot(s.volume.Name)
-
- var fs string
-
- if isSnapshot {
- fs = fmt.Sprintf("custom-snapshots/%s", s.volume.Name)
- } else {
- fs = fmt.Sprintf("custom/%s", s.volume.Name)
- }
- poolName := s.getOnDiskPoolName()
- dataset := fmt.Sprintf("%s/%s", poolName, fs)
-
- var customPoolVolumeMntPoint string
-
- if isSnapshot {
- customPoolVolumeMntPoint = getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- } else {
- customPoolVolumeMntPoint = getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- }
-
- msg, err := zfsPoolVolumeCreate(dataset, "mountpoint=none", "canmount=noauto")
- if err != nil {
- logger.Errorf("Failed to create ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, msg)
- return err
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.StoragePoolVolumeDelete()
- }()
-
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", customPoolVolumeMntPoint)
- if err != nil {
- return err
- }
-
- if !shared.IsMountPoint(customPoolVolumeMntPoint) {
- err := zfsMount(poolName, fs)
- if err != nil {
- return err
- }
- defer zfsUmount(poolName, fs, customPoolVolumeMntPoint)
- }
-
- // apply quota
- if s.volume.Config["size"] != "" {
- size, err := shared.ParseByteSizeString(s.volume.Config["size"])
- if err != nil {
- return err
- }
-
- err = s.StorageEntitySetQuota(storagePoolVolumeTypeCustom, size, nil)
- if err != nil {
- return err
- }
- }
-
- revert = false
-
- logger.Infof("Created ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeDelete() error {
- logger.Infof("Deleting ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- fs := fmt.Sprintf("custom/%s", s.volume.Name)
- customPoolVolumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
-
- poolName := s.getOnDiskPoolName()
- if zfsFilesystemEntityExists(poolName, fs) {
- removable := true
- snaps, err := zfsPoolListSnapshots(poolName, fs)
- if err != nil {
- return err
- }
-
- for _, snap := range snaps {
- var err error
- removable, err = zfsPoolVolumeSnapshotRemovable(poolName, fs, snap)
- if err != nil {
- return err
- }
-
- if !removable {
- break
- }
- }
-
- if removable {
- origin, err := zfsFilesystemEntityPropertyGet(poolName, fs, "origin")
- if err != nil {
- return err
- }
- poolName := s.getOnDiskPoolName()
- origin = strings.TrimPrefix(origin, fmt.Sprintf("%s/", poolName))
-
- err = zfsPoolVolumeDestroy(poolName, fs)
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeCleanup(poolName, origin)
- if err != nil {
- return err
- }
- } else {
- err := zfsPoolVolumeSet(poolName, fs, "mountpoint", "none")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeRename(poolName, fs, fmt.Sprintf("deleted/custom/%s", uuid.NewRandom().String()), true)
- if err != nil {
- return err
- }
- }
- }
-
- if shared.PathExists(customPoolVolumeMntPoint) {
- err := os.RemoveAll(customPoolVolumeMntPoint)
- if err != nil {
- return err
- }
- }
-
- err := s.s.Cluster.StoragePoolVolumeDelete(
- "default",
- s.volume.Name,
- storagePoolVolumeTypeCustom,
- s.poolID)
- if err != nil {
- logger.Errorf(`Failed to delete database entry for ZFS storage volume "%s" on storage pool "%s"`, s.volume.Name, s.pool.Name)
- }
-
- logger.Infof("Deleted ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeMount() (bool, error) {
- logger.Debugf("Mounting ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- fs := fmt.Sprintf("custom/%s", s.volume.Name)
- customPoolVolumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
-
- customMountLockID := getCustomMountLockID(s.pool.Name, s.volume.Name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[customMountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in mounting the storage volume.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[customMountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- var customerr error
- ourMount := false
- if !shared.IsMountPoint(customPoolVolumeMntPoint) {
- customerr = zfsMount(s.getOnDiskPoolName(), fs)
- ourMount = true
- }
-
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[customMountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, customMountLockID)
- }
- lxdStorageMapLock.Unlock()
-
- if customerr != nil {
- return false, customerr
- }
-
- logger.Debugf("Mounted ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return ourMount, nil
-}
-
-func (s *storageZfs) StoragePoolVolumeUmount() (bool, error) {
- logger.Debugf("Unmounting ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- fs := fmt.Sprintf("custom/%s", s.volume.Name)
- customPoolVolumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
-
- customUmountLockID := getCustomUmountLockID(s.pool.Name, s.volume.Name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[customUmountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in unmounting the storage volume.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[customUmountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- var customerr error
- ourUmount := false
- if shared.IsMountPoint(customPoolVolumeMntPoint) {
- customerr = zfsUmount(s.getOnDiskPoolName(), fs, customPoolVolumeMntPoint)
- ourUmount = true
- }
-
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[customUmountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, customUmountLockID)
- }
- lxdStorageMapLock.Unlock()
-
- if customerr != nil {
- return false, customerr
- }
-
- logger.Debugf("Unmounted ZFS storage volume \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return ourUmount, nil
-}
-
-func (s *storageZfs) GetContainerPoolInfo() (int64, string, string) {
- return s.poolID, s.pool.Name, s.getOnDiskPoolName()
-}
-
-func (s *storageZfs) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error {
- logger.Infof(`Updating ZFS storage pool "%s"`, s.pool.Name)
-
- changeable := changeableStoragePoolProperties["zfs"]
- unchangeable := []string{}
- for _, change := range changedConfig {
- if !shared.StringInSlice(change, changeable) {
- unchangeable = append(unchangeable, change)
- }
- }
-
- if len(unchangeable) > 0 {
- return updateStoragePoolError(unchangeable, "zfs")
- }
-
- // "rsync.bwlimit" requires no on-disk modifications.
- // "volume.zfs.remove_snapshots" requires no on-disk modifications.
- // "volume.zfs.use_refquota" requires no on-disk modifications.
-
- logger.Infof(`Updated ZFS storage pool "%s"`, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeUpdate(writable *api.StorageVolumePut, changedConfig []string) error {
- if writable.Restore != "" {
- logger.Infof(`Restoring ZFS storage volume "%s" from snapshot "%s"`,
- s.volume.Name, writable.Restore)
-
- // Check that we can remove the snapshot
- poolID, err := s.s.Cluster.StoragePoolGetID(s.pool.Name)
- if err != nil {
- return err
- }
-
- // Get the names of all storage volume snapshots of a given volume
- volumes, err := s.s.Cluster.StoragePoolVolumeSnapshotsGetType(s.volume.Name, storagePoolVolumeTypeCustom, poolID)
- if err != nil {
- return err
- }
-
- if volumes[len(volumes)-1] != fmt.Sprintf("%s/%s", s.volume.Name, writable.Restore) {
- return fmt.Errorf("ZFS can only restore from the latest snapshot. Delete newer snapshots or copy the snapshot into a new volume instead")
- }
-
- s.volume.Description = writable.Description
- s.volume.Config = writable.Config
-
- targetSnapshotDataset := fmt.Sprintf("%s/custom/%s at snapshot-%s", s.getOnDiskPoolName(), s.volume.Name, writable.Restore)
- msg, err := shared.RunCommand("zfs", "rollback", "-r", "-R", targetSnapshotDataset)
- if err != nil {
- logger.Errorf("Failed to rollback ZFS dataset: %s", msg)
- return err
- }
-
- logger.Infof(`Restored ZFS storage volume "%s" from snapshot "%s"`,
- s.volume.Name, writable.Restore)
- return nil
- }
-
- logger.Infof(`Updating ZFS storage volume "%s"`, s.volume.Name)
-
- changeable := changeableStoragePoolVolumeProperties["zfs"]
- unchangeable := []string{}
- for _, change := range changedConfig {
- if !shared.StringInSlice(change, changeable) {
- unchangeable = append(unchangeable, change)
- }
- }
-
- if len(unchangeable) > 0 {
- return updateStoragePoolVolumeError(unchangeable, "zfs")
- }
-
- if shared.StringInSlice("size", changedConfig) {
- if s.volume.Type != storagePoolVolumeTypeNameCustom {
- return updateStoragePoolVolumeError([]string{"size"}, "zfs")
- }
-
- if s.volume.Config["size"] != writable.Config["size"] {
- size, err := shared.ParseByteSizeString(writable.Config["size"])
- if err != nil {
- return err
- }
-
- err = s.StorageEntitySetQuota(storagePoolVolumeTypeCustom, size, nil)
- if err != nil {
- return err
- }
- }
- }
-
- logger.Infof(`Updated ZFS storage volume "%s"`, s.volume.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeRename(newName string) error {
- logger.Infof(`Renaming ZFS storage volume on storage pool "%s" from "%s" to "%s`,
- s.pool.Name, s.volume.Name, newName)
-
- usedBy, err := storagePoolVolumeUsedByContainersGet(s.s, "default", s.volume.Name, storagePoolVolumeTypeNameCustom)
- if err != nil {
- return err
- }
- if len(usedBy) > 0 {
- return fmt.Errorf(`ZFS storage volume "%s" on storage pool "%s" is attached to containers`,
- s.volume.Name, s.pool.Name)
- }
-
- isSnapshot := shared.IsSnapshot(s.volume.Name)
-
- var oldPath string
- var newPath string
-
- if isSnapshot {
- oldPath = fmt.Sprintf("custom-snapshots/%s", s.volume.Name)
- newPath = fmt.Sprintf("custom-snapshots/%s", newName)
- } else {
- oldPath = fmt.Sprintf("custom/%s", s.volume.Name)
- newPath = fmt.Sprintf("custom/%s", newName)
- }
- poolName := s.getOnDiskPoolName()
- err = zfsPoolVolumeRename(poolName, oldPath, newPath, false)
- if err != nil {
- return err
- }
-
- logger.Infof(`Renamed ZFS storage volume on storage pool "%s" from "%s" to "%s`,
- s.pool.Name, s.volume.Name, newName)
-
- return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, newName,
- storagePoolVolumeTypeCustom, s.poolID)
-}
-
-// Things we don't need to care about
-func (s *storageZfs) ContainerMount(c container) (bool, error) {
- return s.doContainerMount(c.Project(), c.Name(), c.IsPrivileged())
-}
-
-func (s *storageZfs) ContainerUmount(c container, path string) (bool, error) {
- logger.Debugf("Unmounting ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- name := c.Name()
-
- fs := fmt.Sprintf("containers/%s", projectPrefix(c.Project(), name))
- containerPoolVolumeMntPoint := getContainerMountPoint(c.Project(), s.pool.Name, name)
-
- containerUmountLockID := getContainerUmountLockID(s.pool.Name, name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[containerUmountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in unmounting the storage volume.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[containerUmountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- var imgerr error
- ourUmount := false
- if shared.IsMountPoint(containerPoolVolumeMntPoint) {
- imgerr = zfsUmount(s.getOnDiskPoolName(), fs, containerPoolVolumeMntPoint)
- ourUmount = true
- }
-
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[containerUmountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, containerUmountLockID)
- }
- lxdStorageMapLock.Unlock()
-
- if imgerr != nil {
- return false, imgerr
- }
-
- logger.Debugf("Unmounted ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return ourUmount, nil
-}
-
-// Things we do have to care about
-func (s *storageZfs) ContainerStorageReady(container container) bool {
- volumeName := projectPrefix(container.Project(), container.Name())
- fs := fmt.Sprintf("containers/%s", volumeName)
- return zfsFilesystemEntityExists(s.getOnDiskPoolName(), fs)
-}
-
-func (s *storageZfs) ContainerCreate(container container) error {
- err := s.doContainerCreate(container.Project(), container.Name(), container.IsPrivileged())
- if err != nil {
- s.doContainerDelete(container.Project(), container.Name())
- return err
- }
-
- ourMount, err := s.ContainerMount(container)
- if err != nil {
- return err
- }
- if ourMount {
- defer s.ContainerUmount(container, container.Path())
- }
-
- err = container.TemplateApply("create")
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) ContainerCreateFromImage(container container, fingerprint string, tracker *ioprogress.ProgressTracker) error {
- logger.Debugf("Creating ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- containerPath := container.Path()
- containerName := container.Name()
- volumeName := projectPrefix(container.Project(), containerName)
- fs := fmt.Sprintf("containers/%s", volumeName)
- containerPoolVolumeMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, containerName)
-
- poolName := s.getOnDiskPoolName()
- fsImage := fmt.Sprintf("images/%s", fingerprint)
-
- imageStoragePoolLockID := getImageCreateLockID(s.pool.Name, fingerprint)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[imageStoragePoolLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- } else {
- lxdStorageOngoingOperationMap[imageStoragePoolLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- var imgerr error
- if !zfsFilesystemEntityExists(poolName, fsImage) {
- imgerr = s.ImageCreate(fingerprint, tracker)
- }
-
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[imageStoragePoolLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, imageStoragePoolLockID)
- }
- lxdStorageMapLock.Unlock()
-
- if imgerr != nil {
- return imgerr
- }
- }
-
- err := zfsPoolVolumeClone(container.Project(), poolName, fsImage, "readonly", fs, containerPoolVolumeMntPoint)
- if err != nil {
- return err
- }
-
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.ContainerDelete(container)
- }()
-
- ourMount, err := s.ContainerMount(container)
- if err != nil {
- return err
- }
- if ourMount {
- defer s.ContainerUmount(container, containerPath)
- }
-
- privileged := container.IsPrivileged()
- err = createContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged)
- if err != nil {
- return err
- }
-
- err = container.TemplateApply("create")
- if err != nil {
- return err
- }
-
- revert = false
-
- logger.Debugf("Created ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) ContainerDelete(container container) error {
- err := s.doContainerDelete(container.Project(), container.Name())
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) copyWithoutSnapshotsSparse(target container, source container) error {
- poolName := s.getOnDiskPoolName()
-
- sourceContainerName := source.Name()
- sourceContainerPath := source.Path()
-
- targetContainerName := target.Name()
- targetContainerPath := target.Path()
- targetContainerMountPoint := getContainerMountPoint(target.Project(), s.pool.Name, targetContainerName)
-
- sourceZfsDataset := ""
- sourceZfsDatasetSnapshot := ""
- sourceName, sourceSnapOnlyName, isSnapshotName := containerGetParentAndSnapshotName(sourceContainerName)
-
- targetZfsDataset := fmt.Sprintf("containers/%s", projectPrefix(target.Project(), targetContainerName))
-
- if isSnapshotName {
- sourceZfsDatasetSnapshot = sourceSnapOnlyName
- }
-
- revert := true
- if sourceZfsDatasetSnapshot == "" {
- if zfsFilesystemEntityExists(poolName, fmt.Sprintf("containers/%s", projectPrefix(source.Project(), sourceName))) {
- sourceZfsDatasetSnapshot = fmt.Sprintf("copy-%s", uuid.NewRandom().String())
- sourceZfsDataset = fmt.Sprintf("containers/%s", projectPrefix(source.Project(), sourceName))
- err := zfsPoolVolumeSnapshotCreate(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
- if err != nil {
- return err
- }
- defer func() {
- if !revert {
- return
- }
- zfsPoolVolumeSnapshotDestroy(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
- }()
- }
- } else {
- if zfsFilesystemEntityExists(poolName, fmt.Sprintf("containers/%s at snapshot-%s", projectPrefix(source.Project(), sourceName), sourceZfsDatasetSnapshot)) {
- sourceZfsDataset = fmt.Sprintf("containers/%s", projectPrefix(source.Project(), sourceName))
- sourceZfsDatasetSnapshot = fmt.Sprintf("snapshot-%s", sourceZfsDatasetSnapshot)
- }
- }
-
- if sourceZfsDataset != "" {
- err := zfsPoolVolumeClone(target.Project(), poolName, sourceZfsDataset, sourceZfsDatasetSnapshot, targetZfsDataset, targetContainerMountPoint)
- if err != nil {
- return err
- }
- defer func() {
- if !revert {
- return
- }
- zfsPoolVolumeDestroy(poolName, targetZfsDataset)
- }()
-
- ourMount, err := s.ContainerMount(target)
- if err != nil {
- return err
- }
- if ourMount {
- defer s.ContainerUmount(target, targetContainerPath)
- }
-
- err = createContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged())
- if err != nil {
- return err
- }
- defer func() {
- if !revert {
- return
- }
- deleteContainerMountpoint(targetContainerMountPoint, targetContainerPath, s.GetStorageTypeName())
- }()
- } else {
- err := s.ContainerCreate(target)
- if err != nil {
- return err
- }
- defer func() {
- if !revert {
- return
- }
- s.ContainerDelete(target)
- }()
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
- output, err := rsyncLocalCopy(sourceContainerPath, targetContainerPath, bwlimit)
- if err != nil {
- return fmt.Errorf("rsync failed: %s", string(output))
- }
- }
-
- err := target.TemplateApply("copy")
- if err != nil {
- return err
- }
-
- revert = false
-
- return nil
-}
-
-func (s *storageZfs) copyWithoutSnapshotFull(target container, source container) error {
- logger.Debugf("Creating full ZFS copy \"%s\" to \"%s\"", source.Name(), target.Name())
-
- sourceIsSnapshot := source.IsSnapshot()
- poolName := s.getOnDiskPoolName()
-
- sourceName := source.Name()
- sourceDataset := ""
- snapshotSuffix := ""
-
- targetName := target.Name()
- targetDataset := fmt.Sprintf("%s/containers/%s", poolName, projectPrefix(target.Project(), targetName))
- targetSnapshotDataset := ""
-
- if sourceIsSnapshot {
- sourceParentName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(source.Name())
- snapshotSuffix = fmt.Sprintf("snapshot-%s", sourceSnapOnlyName)
- sourceDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, sourceParentName, snapshotSuffix)
- targetSnapshotDataset = fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(target.Project(), targetName), sourceSnapOnlyName)
- } else {
- snapshotSuffix = uuid.NewRandom().String()
- sourceDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(source.Project(), sourceName), snapshotSuffix)
- targetSnapshotDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(target.Project(), targetName), snapshotSuffix)
-
- fs := fmt.Sprintf("containers/%s", projectPrefix(source.Project(), sourceName))
- err := zfsPoolVolumeSnapshotCreate(poolName, fs, snapshotSuffix)
- if err != nil {
- return err
- }
- defer func() {
- err := zfsPoolVolumeSnapshotDestroy(poolName, fs, snapshotSuffix)
- if err != nil {
- logger.Warnf("Failed to delete temporary ZFS snapshot \"%s\", manual cleanup needed", sourceDataset)
- }
- }()
- }
-
- zfsSendCmd := exec.Command("zfs", "send", sourceDataset)
-
- zfsRecvCmd := exec.Command("zfs", "receive", targetDataset)
-
- zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
- zfsRecvCmd.Stdout = os.Stdout
- zfsRecvCmd.Stderr = os.Stderr
-
- err := zfsRecvCmd.Start()
- if err != nil {
- return err
- }
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- err = zfsRecvCmd.Wait()
- if err != nil {
- return err
- }
-
- msg, err := shared.RunCommand("zfs", "rollback", "-r", "-R", targetSnapshotDataset)
- if err != nil {
- logger.Errorf("Failed to rollback ZFS dataset: %s", msg)
- return err
- }
-
- targetContainerMountPoint := getContainerMountPoint(target.Project(), s.pool.Name, targetName)
- targetfs := fmt.Sprintf("containers/%s", targetName)
-
- err = zfsPoolVolumeSet(poolName, targetfs, "canmount", "noauto")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(poolName, targetfs, "mountpoint", targetContainerMountPoint)
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSnapshotDestroy(poolName, targetfs, snapshotSuffix)
- if err != nil {
- return err
- }
-
- ourMount, err := s.ContainerMount(target)
- if err != nil {
- return err
- }
- if ourMount {
- defer s.ContainerUmount(target, targetContainerMountPoint)
- }
-
- err = createContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged())
- if err != nil {
- return err
- }
-
- logger.Debugf("Created full ZFS copy \"%s\" to \"%s\"", source.Name(), target.Name())
- return nil
-}
-
-func (s *storageZfs) copyWithSnapshots(target container, source container, parentSnapshot string) error {
- sourceName := source.Name()
- targetParentName, targetSnapOnlyName, _ := containerGetParentAndSnapshotName(target.Name())
- containersPath := getSnapshotMountPoint(target.Project(), s.pool.Name, targetParentName)
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(target.Project(), targetParentName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(target.Project(), targetParentName))
- err := createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
-
- poolName := s.getOnDiskPoolName()
- sourceParentName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(sourceName)
- currentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(source.Project(), sourceParentName), sourceSnapOnlyName)
- args := []string{"send", currentSnapshotDataset}
- if parentSnapshot != "" {
- parentName, parentSnaponlyName, _ := containerGetParentAndSnapshotName(parentSnapshot)
- parentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(source.Project(), parentName), parentSnaponlyName)
- args = append(args, "-i", parentSnapshotDataset)
- }
-
- zfsSendCmd := exec.Command("zfs", args...)
- targetSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(target.Project(), targetParentName), targetSnapOnlyName)
- zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetSnapshotDataset)
-
- zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
- zfsRecvCmd.Stdout = os.Stdout
- zfsRecvCmd.Stderr = os.Stderr
-
- err = zfsRecvCmd.Start()
- if err != nil {
- return err
- }
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- err = zfsRecvCmd.Wait()
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) doCrossPoolContainerCopy(target container, source container, containerOnly bool, refresh bool, refreshSnapshots []container) error {
- sourcePool, err := source.StoragePool()
- if err != nil {
- return err
- }
-
- // setup storage for the source volume
- srcStorage, err := storagePoolVolumeInit(s.s, "default", sourcePool, source.Name(), storagePoolVolumeTypeContainer)
- if err != nil {
- return err
- }
-
- ourMount, err := srcStorage.StoragePoolMount()
- if err != nil {
- return err
- }
- if ourMount {
- defer srcStorage.StoragePoolUmount()
- }
-
- targetPool, err := target.StoragePool()
- if err != nil {
- return err
- }
-
- var snapshots []container
-
- if refresh {
- snapshots = refreshSnapshots
- } else {
- snapshots, err = source.Snapshots()
- if err != nil {
- return err
- }
-
- // create the main container
- err = s.doContainerCreate(target.Project(), target.Name(), target.IsPrivileged())
- if err != nil {
- return err
- }
- }
-
- _, err = s.doContainerMount(target.Project(), target.Name(), target.IsPrivileged())
- if err != nil {
- return err
- }
- defer s.ContainerUmount(target, shared.VarPath("containers", projectPrefix(target.Project(), target.Name())))
-
- destContainerMntPoint := getContainerMountPoint(target.Project(), targetPool, target.Name())
- bwlimit := s.pool.Config["rsync.bwlimit"]
- if !containerOnly {
- for _, snap := range snapshots {
- srcSnapshotMntPoint := getSnapshotMountPoint(target.Project(), sourcePool, snap.Name())
- _, err = rsyncLocalCopy(srcSnapshotMntPoint, destContainerMntPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- // create snapshot
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
- err = s.doContainerSnapshotCreate(snap.Project(), fmt.Sprintf("%s/%s", target.Name(), snapOnlyName), target.Name())
- if err != nil {
- return err
- }
- }
- }
-
- srcContainerMntPoint := getContainerMountPoint(source.Project(), sourcePool, source.Name())
- _, err = rsyncLocalCopy(srcContainerMntPoint, destContainerMntPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) ContainerCopy(target container, source container, containerOnly bool) error {
- logger.Debugf("Copying ZFS container storage %s to %s", source.Name(), target.Name())
-
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- _, sourcePool, _ := source.Storage().GetContainerPoolInfo()
- _, targetPool, _ := target.Storage().GetContainerPoolInfo()
- if sourcePool != targetPool {
- return s.doCrossPoolContainerCopy(target, source, containerOnly, false, nil)
- }
-
- snapshots, err := source.Snapshots()
- if err != nil {
- return err
- }
-
- if containerOnly || len(snapshots) == 0 {
- if s.pool.Config["zfs.clone_copy"] != "" && !shared.IsTrue(s.pool.Config["zfs.clone_copy"]) {
- err = s.copyWithoutSnapshotFull(target, source)
- if err != nil {
- return err
- }
- } else {
- err = s.copyWithoutSnapshotsSparse(target, source)
- if err != nil {
- return err
- }
- }
- } else {
- targetContainerName := target.Name()
- targetContainerPath := target.Path()
- targetContainerMountPoint := getContainerMountPoint(target.Project(), s.pool.Name, targetContainerName)
- err = createContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged())
- if err != nil {
- return err
- }
-
- prev := ""
- prevSnapOnlyName := ""
- for i, snap := range snapshots {
- if i > 0 {
- prev = snapshots[i-1].Name()
- }
-
- sourceSnapshot, err := containerLoadByProjectAndName(s.s, source.Project(), snap.Name())
- if err != nil {
- return err
- }
-
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
- prevSnapOnlyName = snapOnlyName
- newSnapName := fmt.Sprintf("%s/%s", target.Name(), snapOnlyName)
- targetSnapshot, err := containerLoadByProjectAndName(s.s, target.Project(), newSnapName)
- if err != nil {
- return err
- }
-
- err = s.copyWithSnapshots(targetSnapshot, sourceSnapshot, prev)
- if err != nil {
- return err
- }
- }
-
- poolName := s.getOnDiskPoolName()
-
- // send actual container
- tmpSnapshotName := fmt.Sprintf("copy-send-%s", uuid.NewRandom().String())
- err = zfsPoolVolumeSnapshotCreate(poolName, fmt.Sprintf("containers/%s", projectPrefix(source.Project(), source.Name())), tmpSnapshotName)
- if err != nil {
- return err
- }
-
- currentSnapshotDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(source.Project(), source.Name()), tmpSnapshotName)
- args := []string{"send", currentSnapshotDataset}
- if prevSnapOnlyName != "" {
- parentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(source.Project(), source.Name()), prevSnapOnlyName)
- args = append(args, "-i", parentSnapshotDataset)
- }
-
- zfsSendCmd := exec.Command("zfs", args...)
- targetSnapshotDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(target.Project(), target.Name()), tmpSnapshotName)
- zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetSnapshotDataset)
-
- zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
- zfsRecvCmd.Stdout = os.Stdout
- zfsRecvCmd.Stderr = os.Stderr
-
- err = zfsRecvCmd.Start()
- if err != nil {
- return err
- }
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- err = zfsRecvCmd.Wait()
- if err != nil {
- return err
- }
-
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(source.Project(), source.Name())), tmpSnapshotName)
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(target.Project(), target.Name())), tmpSnapshotName)
-
- fs := fmt.Sprintf("containers/%s", projectPrefix(target.Project(), target.Name()))
- err = zfsPoolVolumeSet(poolName, fs, "canmount", "noauto")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", targetContainerMountPoint)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Copied ZFS container storage %s to %s", source.Name(), target.Name())
- return nil
-}
-
-func (s *storageZfs) ContainerRefresh(target container, source container, snapshots []container) error {
- logger.Debugf("Refreshing ZFS container storage for %s from %s", target.Name(), source.Name())
-
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- return s.doCrossPoolContainerCopy(target, source, len(snapshots) == 0, true, snapshots)
-}
-
-func (s *storageZfs) ContainerRename(container container, newName string) error {
- logger.Debugf("Renaming ZFS storage volume for container \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
-
- poolName := s.getOnDiskPoolName()
- oldName := container.Name()
-
- // Unmount the dataset.
- _, err := s.ContainerUmount(container, "")
- if err != nil {
- return err
- }
-
- // Rename the dataset.
- oldZfsDataset := fmt.Sprintf("containers/%s", oldName)
- newZfsDataset := fmt.Sprintf("containers/%s", newName)
- err = zfsPoolVolumeRename(poolName, oldZfsDataset, newZfsDataset, false)
- if err != nil {
- return err
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.ContainerRename(container, oldName)
- }()
-
- // Set the new mountpoint for the dataset.
- newContainerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, newName)
- err = zfsPoolVolumeSet(poolName, newZfsDataset, "mountpoint", newContainerMntPoint)
- if err != nil {
- return err
- }
-
- // Unmount the dataset.
- container.(*containerLXC).name = newName
- _, err = s.ContainerUmount(container, "")
- if err != nil {
- return err
- }
-
- // Create new mountpoint on the storage pool.
- oldContainerMntPoint := getContainerMountPoint(container.Project(), s.pool.Name, oldName)
- oldContainerMntPointSymlink := container.Path()
- newContainerMntPointSymlink := shared.VarPath("containers", projectPrefix(container.Project(), newName))
- err = renameContainerMountpoint(oldContainerMntPoint, oldContainerMntPointSymlink, newContainerMntPoint, newContainerMntPointSymlink)
- if err != nil {
- return err
- }
-
- // Rename the snapshot mountpoint on the storage pool.
- oldSnapshotMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, oldName)
- newSnapshotMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, newName)
- if shared.PathExists(oldSnapshotMntPoint) {
- err := os.Rename(oldSnapshotMntPoint, newSnapshotMntPoint)
- if err != nil {
- return err
- }
- }
-
- // Remove old symlink.
- oldSnapshotPath := shared.VarPath("snapshots", projectPrefix(container.Project(), oldName))
- if shared.PathExists(oldSnapshotPath) {
- err := os.Remove(oldSnapshotPath)
- if err != nil {
- return err
- }
- }
-
- // Create new symlink.
- newSnapshotPath := shared.VarPath("snapshots", projectPrefix(container.Project(), newName))
- if shared.PathExists(newSnapshotPath) {
- err := os.Symlink(newSnapshotMntPoint, newSnapshotPath)
- if err != nil {
- return err
- }
- }
-
- revert = false
-
- logger.Debugf("Renamed ZFS storage volume for container \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
- return nil
-}
-
-func (s *storageZfs) ContainerRestore(target container, source container) error {
- logger.Debugf("Restoring ZFS storage volume for container \"%s\" from %s to %s", s.volume.Name, source.Name(), target.Name())
-
- snaps, err := target.Snapshots()
- if err != nil {
- return err
- }
-
- if snaps[len(snaps)-1].Name() != source.Name() {
- if s.pool.Config["volume.zfs.remove_snapshots"] != "" {
- zfsRemoveSnapshots = s.pool.Config["volume.zfs.remove_snapshots"]
- }
-
- if s.volume.Config["zfs.remove_snapshots"] != "" {
- zfsRemoveSnapshots = s.volume.Config["zfs.remove_snapshots"]
- }
-
- if !shared.IsTrue(zfsRemoveSnapshots) {
- return fmt.Errorf("ZFS can only restore from the latest snapshot. Delete newer snapshots or copy the snapshot into a new container instead")
- }
- }
-
- // Start storage for source container
- ourSourceStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourSourceStart {
- defer source.StorageStop()
- }
-
- // Start storage for target container
- ourTargetStart, err := target.StorageStart()
- if err != nil {
- return err
- }
- if ourTargetStart {
- defer target.StorageStop()
- }
-
- for i := len(snaps) - 1; i != 0; i-- {
- if snaps[i].Name() == source.Name() {
- break
- }
-
- err := snaps[i].Delete()
- if err != nil {
- return err
- }
- }
-
- // Restore the snapshot
- cName, snapOnlyName, _ := containerGetParentAndSnapshotName(source.Name())
- snapName := fmt.Sprintf("snapshot-%s", snapOnlyName)
-
- err = zfsPoolVolumeSnapshotRestore(s.getOnDiskPoolName(), fmt.Sprintf("containers/%s", cName), snapName)
- if err != nil {
- return err
- }
-
- logger.Debugf("Restored ZFS storage volume for container \"%s\" from %s to %s", s.volume.Name, source.Name(), target.Name())
- return nil
-}
-
-func (s *storageZfs) ContainerGetUsage(container container) (int64, error) {
- var err error
-
- fs := fmt.Sprintf("containers/%s", container.Name())
-
- property := "used"
-
- if s.pool.Config["volume.zfs.use_refquota"] != "" {
- zfsUseRefquota = s.pool.Config["volume.zfs.use_refquota"]
- }
- if s.volume.Config["zfs.use_refquota"] != "" {
- zfsUseRefquota = s.volume.Config["zfs.use_refquota"]
- }
-
- if shared.IsTrue(zfsUseRefquota) {
- property = "referenced"
- }
-
- // Shortcut for refquota
- mountpoint := getContainerMountPoint(container.Project(), s.pool.Name, container.Name())
- if property == "referenced" && shared.IsMountPoint(mountpoint) {
- var stat syscall.Statfs_t
- err := syscall.Statfs(mountpoint, &stat)
- if err != nil {
- return -1, err
- }
-
- return int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize), nil
- }
-
- value, err := zfsFilesystemEntityPropertyGet(s.getOnDiskPoolName(), fs, property)
- if err != nil {
- return -1, err
- }
-
- valueInt, err := strconv.ParseInt(value, 10, 64)
- if err != nil {
- return -1, err
- }
-
- return valueInt, nil
-}
-
-func (s *storageZfs) doContainerSnapshotCreate(project, targetName string, sourceName string) error {
- snapshotContainerName := targetName
- logger.Debugf("Creating ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", snapshotContainerName, s.pool.Name)
-
- sourceContainerName := sourceName
-
- cName, snapshotSnapOnlyName, _ := containerGetParentAndSnapshotName(snapshotContainerName)
- snapName := fmt.Sprintf("snapshot-%s", snapshotSnapOnlyName)
-
- sourceZfsDataset := fmt.Sprintf("containers/%s", projectPrefix(project, cName))
- err := zfsPoolVolumeSnapshotCreate(s.getOnDiskPoolName(), sourceZfsDataset, snapName)
- if err != nil {
- return err
- }
-
- snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, snapshotContainerName)
- if !shared.PathExists(snapshotMntPoint) {
- err := os.MkdirAll(snapshotMntPoint, 0700)
- if err != nil {
- return err
- }
- }
-
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(project, sourceName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceContainerName))
- if !shared.PathExists(snapshotMntPointSymlink) {
- err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Created ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", snapshotContainerName, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) ContainerSnapshotCreate(snapshotContainer container, sourceContainer container) error {
- err := s.doContainerSnapshotCreate(sourceContainer.Project(), snapshotContainer.Name(), sourceContainer.Name())
- if err != nil {
- s.ContainerSnapshotDelete(snapshotContainer)
- return err
- }
- return nil
-}
-
-func zfsSnapshotDeleteInternal(project, poolName string, ctName string, onDiskPoolName string) error {
- sourceContainerName, sourceContainerSnapOnlyName, _ := containerGetParentAndSnapshotName(ctName)
- snapName := fmt.Sprintf("snapshot-%s", sourceContainerSnapOnlyName)
-
- if zfsFilesystemEntityExists(onDiskPoolName,
- fmt.Sprintf("containers/%s@%s",
- projectPrefix(project, sourceContainerName), snapName)) {
- removable, err := zfsPoolVolumeSnapshotRemovable(onDiskPoolName,
- fmt.Sprintf("containers/%s",
- projectPrefix(project, sourceContainerName)),
- snapName)
- if err != nil {
- return err
- }
-
- if removable {
- err = zfsPoolVolumeSnapshotDestroy(onDiskPoolName,
- fmt.Sprintf("containers/%s",
- projectPrefix(project, sourceContainerName)),
- snapName)
- } else {
- err = zfsPoolVolumeSnapshotRename(onDiskPoolName,
- fmt.Sprintf("containers/%s",
- projectPrefix(project, sourceContainerName)),
- snapName,
- fmt.Sprintf("copy-%s", uuid.NewRandom().String()))
- }
- if err != nil {
- return err
- }
- }
-
- // Delete the snapshot on its storage pool:
- // ${POOL}/snapshots/<snapshot_name>
- snapshotContainerMntPoint := getSnapshotMountPoint(project, poolName, ctName)
- if shared.PathExists(snapshotContainerMntPoint) {
- err := os.RemoveAll(snapshotContainerMntPoint)
- if err != nil {
- return err
- }
- }
-
- // Check if we can remove the snapshot symlink:
- // ${LXD_DIR}/snapshots/<container_name> to ${POOL}/snapshots/<container_name>
- // by checking if the directory is empty.
- snapshotContainerPath := getSnapshotMountPoint(project, poolName, sourceContainerName)
- empty, _ := shared.PathIsEmpty(snapshotContainerPath)
- if empty == true {
- // Remove the snapshot directory for the container:
- // ${POOL}/snapshots/<source_container_name>
- err := os.Remove(snapshotContainerPath)
- if err != nil {
- return err
- }
-
- snapshotSymlink := shared.VarPath("snapshots", projectPrefix(project, sourceContainerName))
- if shared.PathExists(snapshotSymlink) {
- err := os.Remove(snapshotSymlink)
- if err != nil {
- return err
- }
- }
- }
-
- // Legacy
- snapPath := shared.VarPath(fmt.Sprintf("snapshots/%s/%s.zfs", projectPrefix(project, sourceContainerName), sourceContainerSnapOnlyName))
- if shared.PathExists(snapPath) {
- err := os.Remove(snapPath)
- if err != nil {
- return err
- }
- }
-
- // Legacy
- parent := shared.VarPath(fmt.Sprintf("snapshots/%s", projectPrefix(project, sourceContainerName)))
- if ok, _ := shared.PathIsEmpty(parent); ok {
- err := os.Remove(parent)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func (s *storageZfs) ContainerSnapshotDelete(snapshotContainer container) error {
- logger.Debugf("Deleting ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- poolName := s.getOnDiskPoolName()
- err := zfsSnapshotDeleteInternal(snapshotContainer.Project(), s.pool.Name, snapshotContainer.Name(),
- poolName)
- if err != nil {
- return err
- }
-
- logger.Debugf("Deleted ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) ContainerSnapshotRename(snapshotContainer container, newName string) error {
- logger.Debugf("Renaming ZFS storage volume for snapshot \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
-
- oldName := snapshotContainer.Name()
-
- oldcName, oldSnapOnlyName, _ := containerGetParentAndSnapshotName(snapshotContainer.Name())
- oldZfsDatasetName := fmt.Sprintf("snapshot-%s", oldSnapOnlyName)
-
- _, newSnapOnlyName, _ := containerGetParentAndSnapshotName(newName)
- newZfsDatasetName := fmt.Sprintf("snapshot-%s", newSnapOnlyName)
-
- if oldZfsDatasetName != newZfsDatasetName {
- err := zfsPoolVolumeSnapshotRename(
- s.getOnDiskPoolName(), fmt.Sprintf("containers/%s", projectPrefix(snapshotContainer.Project(), oldcName)), oldZfsDatasetName, newZfsDatasetName)
- if err != nil {
- return err
- }
- }
- revert := true
- defer func() {
- if !revert {
- return
- }
- //s.ContainerSnapshotRename(snapshotContainer, oldName)
- }()
-
- oldStyleSnapshotMntPoint := shared.VarPath(fmt.Sprintf("snapshots/%s/%s.zfs", projectPrefix(snapshotContainer.Project(), oldcName), oldSnapOnlyName))
- if shared.PathExists(oldStyleSnapshotMntPoint) {
- err := os.Remove(oldStyleSnapshotMntPoint)
- if err != nil {
- return err
- }
- }
-
- oldSnapshotMntPoint := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, oldName)
- if shared.PathExists(oldSnapshotMntPoint) {
- err := os.Remove(oldSnapshotMntPoint)
- if err != nil {
- return err
- }
- }
-
- newSnapshotMntPoint := getSnapshotMountPoint(snapshotContainer.Project(), s.pool.Name, newName)
- if !shared.PathExists(newSnapshotMntPoint) {
- err := os.MkdirAll(newSnapshotMntPoint, 0700)
- if err != nil {
- return err
- }
- }
-
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(snapshotContainer.Project(), oldcName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(snapshotContainer.Project(), oldcName))
- if !shared.PathExists(snapshotMntPointSymlink) {
- err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
- }
-
- revert = false
-
- logger.Debugf("Renamed ZFS storage volume for snapshot \"%s\" from %s to %s", s.volume.Name, s.volume.Name, newName)
- return nil
-}
-
-func (s *storageZfs) ContainerSnapshotStart(container container) (bool, error) {
- logger.Debugf("Initializing ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- cName, sName, _ := containerGetParentAndSnapshotName(container.Name())
- sourceFs := fmt.Sprintf("containers/%s", projectPrefix(container.Project(), cName))
- sourceSnap := fmt.Sprintf("snapshot-%s", sName)
- destFs := fmt.Sprintf("snapshots/%s/%s", projectPrefix(container.Project(), cName), sName)
-
- poolName := s.getOnDiskPoolName()
- snapshotMntPoint := getSnapshotMountPoint(container.Project(), s.pool.Name, container.Name())
- err := zfsPoolVolumeClone(container.Project(), poolName, sourceFs, sourceSnap, destFs, snapshotMntPoint)
- if err != nil {
- return false, err
- }
-
- err = zfsMount(poolName, destFs)
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Initialized ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return true, nil
-}
-
-func (s *storageZfs) ContainerSnapshotStop(container container) (bool, error) {
- logger.Debugf("Stopping ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- cName, sName, _ := containerGetParentAndSnapshotName(container.Name())
- destFs := fmt.Sprintf("snapshots/%s/%s", projectPrefix(container.Project(), cName), sName)
-
- err := zfsPoolVolumeDestroy(s.getOnDiskPoolName(), destFs)
- if err != nil {
- return false, err
- }
-
- logger.Debugf("Stopped ZFS storage volume for snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return true, nil
-}
-
-func (s *storageZfs) ContainerSnapshotCreateEmpty(snapshotContainer container) error {
- /* don't touch the fs yet, as migration will do that for us */
- return nil
-}
-
-func (s *storageZfs) doContainerOnlyBackup(tmpPath string, backup backup, source container) error {
- sourceIsSnapshot := source.IsSnapshot()
- poolName := s.getOnDiskPoolName()
-
- sourceName := source.Name()
- sourceDataset := ""
- snapshotSuffix := ""
-
- if sourceIsSnapshot {
- sourceParentName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(source.Name())
- snapshotSuffix = fmt.Sprintf("backup-%s", sourceSnapOnlyName)
- sourceDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, sourceParentName, snapshotSuffix)
- } else {
- snapshotSuffix = uuid.NewRandom().String()
- sourceDataset = fmt.Sprintf("%s/containers/%s@%s", poolName, sourceName, snapshotSuffix)
-
- fs := fmt.Sprintf("containers/%s", projectPrefix(source.Project(), sourceName))
- err := zfsPoolVolumeSnapshotCreate(poolName, fs, snapshotSuffix)
- if err != nil {
- return err
- }
-
- defer func() {
- err := zfsPoolVolumeSnapshotDestroy(poolName, fs, snapshotSuffix)
- if err != nil {
- logger.Warnf("Failed to delete temporary ZFS snapshot \"%s\", manual cleanup needed", sourceDataset)
- }
- }()
- }
-
- // Dump the container to a file
- backupFile := fmt.Sprintf("%s/%s", tmpPath, "container.bin")
- f, err := os.OpenFile(backupFile, os.O_RDWR|os.O_CREATE, 0644)
- if err != nil {
- return err
- }
- defer f.Close()
-
- zfsSendCmd := exec.Command("zfs", "send", sourceDataset)
- zfsSendCmd.Stdout = f
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) doSnapshotBackup(tmpPath string, backup backup, source container, parentSnapshot string) error {
- sourceName := source.Name()
- snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath)
-
- // Create backup path for snapshots
- err := os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return err
- }
-
- poolName := s.getOnDiskPoolName()
- sourceParentName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(sourceName)
- currentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, sourceParentName, sourceSnapOnlyName)
- args := []string{"send", currentSnapshotDataset}
- if parentSnapshot != "" {
- parentName, parentSnaponlyName, _ := containerGetParentAndSnapshotName(parentSnapshot)
- parentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, parentName, parentSnaponlyName)
- args = append(args, "-i", parentSnapshotDataset)
- }
-
- backupFile := fmt.Sprintf("%s/%s.bin", snapshotsPath, sourceSnapOnlyName)
- f, err := os.OpenFile(backupFile, os.O_RDWR|os.O_CREATE, 0644)
- if err != nil {
- return err
- }
- defer f.Close()
-
- zfsSendCmd := exec.Command("zfs", args...)
- zfsSendCmd.Stdout = f
- return zfsSendCmd.Run()
-}
-
-func (s *storageZfs) doContainerBackupCreateOptimized(tmpPath string, backup backup, source container) error {
- // Handle snapshots
- snapshots, err := source.Snapshots()
- if err != nil {
- return err
- }
-
- if backup.containerOnly || len(snapshots) == 0 {
- err = s.doContainerOnlyBackup(tmpPath, backup, source)
- } else {
- prev := ""
- prevSnapOnlyName := ""
- for i, snap := range snapshots {
- if i > 0 {
- prev = snapshots[i-1].Name()
- }
-
- sourceSnapshot, err := containerLoadByProjectAndName(s.s, source.Project(), snap.Name())
- if err != nil {
- return err
- }
-
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
- prevSnapOnlyName = snapOnlyName
- err = s.doSnapshotBackup(tmpPath, backup, sourceSnapshot, prev)
- if err != nil {
- return err
- }
- }
-
- // Dump the container to a file
- poolName := s.getOnDiskPoolName()
- tmpSnapshotName := fmt.Sprintf("backup-%s", uuid.NewRandom().String())
- err = zfsPoolVolumeSnapshotCreate(poolName, fmt.Sprintf("containers/%s", projectPrefix(source.Project(), source.Name())), tmpSnapshotName)
- if err != nil {
- return err
- }
-
- currentSnapshotDataset := fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(source.Project(), source.Name()), tmpSnapshotName)
- args := []string{"send", currentSnapshotDataset}
- if prevSnapOnlyName != "" {
- parentSnapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, projectPrefix(source.Project(), source.Name()), prevSnapOnlyName)
- args = append(args, "-i", parentSnapshotDataset)
- }
-
- backupFile := fmt.Sprintf("%s/container.bin", tmpPath)
- f, err := os.OpenFile(backupFile, os.O_RDWR|os.O_CREATE, 0644)
- if err != nil {
- return err
- }
- defer f.Close()
-
- zfsSendCmd := exec.Command("zfs", args...)
- zfsSendCmd.Stdout = f
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", source.Name()), tmpSnapshotName)
- }
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) doContainerBackupCreateVanilla(tmpPath string, backup backup, source container) error {
- // Prepare for rsync
- rsync := func(oldPath string, newPath string, bwlimit string) error {
- output, err := rsyncLocalCopy(oldPath, newPath, bwlimit)
- if err != nil {
- return fmt.Errorf("Failed to rsync: %s: %s", string(output), err)
- }
-
- return nil
- }
-
- bwlimit := s.pool.Config["rsync.bwlimit"]
- project := backup.container.Project()
-
- // Handle snapshots
- if !backup.containerOnly {
- snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath)
-
- // Retrieve the snapshots
- snapshots, err := source.Snapshots()
- if err != nil {
- return errors.Wrap(err, "Retrieve snaphots")
- }
-
- // Create the snapshot path
- if len(snapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return errors.Wrap(err, "Create snapshot path")
- }
- }
-
- for _, snap := range snapshots {
- _, snapName, _ := containerGetParentAndSnapshotName(snap.Name())
-
- // Mount the snapshot to a usable path
- _, err := s.ContainerSnapshotStart(snap)
- if err != nil {
- return errors.Wrap(err, "Mount snapshot")
- }
-
- snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, snap.Name())
- target := fmt.Sprintf("%s/%s", snapshotsPath, snapName)
-
- // Copy the snapshot
- err = rsync(snapshotMntPoint, target, bwlimit)
- s.ContainerSnapshotStop(snap)
- if err != nil {
- return errors.Wrap(err, "Copy snapshot")
- }
- }
- }
-
- // Make a temporary copy of the container
- containersPath := getContainerMountPoint("default", s.pool.Name, "")
- tmpContainerMntPoint, err := ioutil.TempDir(containersPath, source.Name())
- if err != nil {
- return errors.Wrap(err, "Create temporary copy dir")
- }
- defer os.RemoveAll(tmpContainerMntPoint)
-
- err = os.Chmod(tmpContainerMntPoint, 0700)
- if err != nil {
- return errors.Wrap(err, "Change temporary mount point permissions")
- }
-
- snapshotSuffix := uuid.NewRandom().String()
- sourceName := source.Name()
- fs := fmt.Sprintf("containers/%s", projectPrefix(project, sourceName))
- sourceZfsDatasetSnapshot := fmt.Sprintf("snapshot-%s", snapshotSuffix)
- poolName := s.getOnDiskPoolName()
- err = zfsPoolVolumeSnapshotCreate(poolName, fs, sourceZfsDatasetSnapshot)
- if err != nil {
- return err
- }
- defer zfsPoolVolumeSnapshotDestroy(poolName, fs, sourceZfsDatasetSnapshot)
-
- targetZfsDataset := fmt.Sprintf("containers/%s", snapshotSuffix)
- err = zfsPoolVolumeClone(source.Project(), poolName, fs, sourceZfsDatasetSnapshot, targetZfsDataset, tmpContainerMntPoint)
- if err != nil {
- return errors.Wrap(err, "Clone volume")
- }
- defer zfsPoolVolumeDestroy(poolName, targetZfsDataset)
-
- // Mount the temporary copy
- if !shared.IsMountPoint(tmpContainerMntPoint) {
- err = zfsMount(poolName, targetZfsDataset)
- if err != nil {
- return errors.Wrap(err, "Mount temporary copy")
- }
- defer zfsUmount(poolName, targetZfsDataset, tmpContainerMntPoint)
- }
-
- // Copy the container
- containerPath := fmt.Sprintf("%s/container", tmpPath)
- err = rsync(tmpContainerMntPoint, containerPath, bwlimit)
- if err != nil {
- return errors.Wrap(err, "Copy container")
- }
-
- return nil
-}
-
-func (s *storageZfs) ContainerBackupCreate(backup backup, source container) error {
- // Start storage
- ourStart, err := source.StorageStart()
- if err != nil {
- return err
- }
- if ourStart {
- defer source.StorageStop()
- }
-
- // Create a temporary path for the backup
- tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_")
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpPath)
-
- // Generate the actual backup
- if backup.optimizedStorage {
- err = s.doContainerBackupCreateOptimized(tmpPath, backup, source)
- if err != nil {
- return errors.Wrap(err, "Optimized backup")
- }
- } else {
- err = s.doContainerBackupCreateVanilla(tmpPath, backup, source)
- if err != nil {
- return errors.Wrap(err, "Vanilla backup")
- }
- }
-
- // Pack the backup
- err = backupCreateTarball(s.s, tmpPath, backup)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- containerName, _, _ := containerGetParentAndSnapshotName(info.Name)
- containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, containerName)
- err := createContainerMountpoint(containerMntPoint, containerPath(info.Project, info.Name, false), info.Privileged)
- if err != nil {
- return err
- }
-
- unpackPath := fmt.Sprintf("%s/.backup", containerMntPoint)
- err = os.MkdirAll(unpackPath, 0711)
- if err != nil {
- return err
- }
-
- err = os.Chmod(unpackPath, 0700)
- if err != nil {
- // can't use defer because it needs to run before the mount
- os.RemoveAll(unpackPath)
- return err
- }
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--strip-components=1",
- "-C", unpackPath, "backup",
- }...)
-
- // Extract container
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- // can't use defer because it needs to run before the mount
- os.RemoveAll(unpackPath)
- logger.Errorf("Failed to untar \"%s\" into \"%s\": %s", info.Name, unpackPath, err)
- return err
- }
-
- poolName := s.getOnDiskPoolName()
- for _, snapshotOnlyName := range info.Snapshots {
- snapshotBackup := fmt.Sprintf("%s/snapshots/%s.bin", unpackPath, snapshotOnlyName)
- feeder, err := os.Open(snapshotBackup)
- if err != nil {
- // can't use defer because it needs to run before the mount
- os.RemoveAll(unpackPath)
- return err
- }
-
- snapshotDataset := fmt.Sprintf("%s/containers/%s at snapshot-%s", poolName, containerName, snapshotOnlyName)
- zfsRecvCmd := exec.Command("zfs", "receive", "-F", snapshotDataset)
- zfsRecvCmd.Stdin = feeder
- err = zfsRecvCmd.Run()
- feeder.Close()
- if err != nil {
- // can't use defer because it needs to run before the mount
- os.RemoveAll(unpackPath)
- return err
- }
-
- // create mountpoint
- snapshotMntPoint := getSnapshotMountPoint(info.Project, s.pool.Name, fmt.Sprintf("%s/%s", containerName, snapshotOnlyName))
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(info.Project, containerName))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(info.Project, containerName))
- err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- // can't use defer because it needs to run before the mount
- os.RemoveAll(unpackPath)
- return err
- }
- }
-
- containerBackup := fmt.Sprintf("%s/container.bin", unpackPath)
- feeder, err := os.Open(containerBackup)
- if err != nil {
- // can't use defer because it needs to run before the mount
- os.RemoveAll(unpackPath)
- return err
- }
- defer feeder.Close()
-
- containerSnapshotDataset := fmt.Sprintf("%s/containers/%s at backup", poolName, containerName)
- zfsRecvCmd := exec.Command("zfs", "receive", "-F", containerSnapshotDataset)
- zfsRecvCmd.Stdin = feeder
-
- err = zfsRecvCmd.Run()
- os.RemoveAll(unpackPath)
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", containerName), "backup")
- if err != nil {
- return err
- }
-
- fs := fmt.Sprintf("containers/%s", containerName)
- err = zfsPoolVolumeSet(poolName, fs, "canmount", "noauto")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", containerMntPoint)
- if err != nil {
- return err
- }
-
- _, err = s.doContainerMount("default", containerName, info.Privileged)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) doContainerBackupLoadVanilla(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- // create the main container
- err := s.doContainerCreate(info.Project, info.Name, info.Privileged)
- if err != nil {
- s.doContainerDelete(info.Project, info.Name)
- return errors.Wrap(err, "Create container")
- }
-
- _, err = s.doContainerMount(info.Project, info.Name, info.Privileged)
- if err != nil {
- return errors.Wrap(err, "Mount container")
- }
-
- containerMntPoint := getContainerMountPoint(info.Project, s.pool.Name, info.Name)
- // Extract container
- for _, snap := range info.Snapshots {
- // Extract snapshots
- cur := fmt.Sprintf("backup/snapshots/%s", snap)
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--recursive-unlink",
- "--strip-components=3",
- "--xattrs-include=*",
- "-C", containerMntPoint, cur,
- }...)
-
- // Unpack
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- logger.Errorf("Failed to untar \"%s\" into \"%s\": %s", cur, containerMntPoint, err)
- return errors.Wrap(err, "Unpack")
- }
-
- // create snapshot
- err = s.doContainerSnapshotCreate(info.Project, fmt.Sprintf("%s/%s", info.Name, snap), info.Name)
- if err != nil {
- return errors.Wrap(err, "Create snapshot")
- }
- }
-
- // Prepare tar arguments
- args := append(tarArgs, []string{
- "-",
- "--strip-components=2",
- "--xattrs-include=*",
- "-C", containerMntPoint, "backup/container",
- }...)
-
- // Extract container
- data.Seek(0, 0)
- err = shared.RunCommandWithFds(data, nil, "tar", args...)
- if err != nil {
- logger.Errorf("Failed to untar \"backup/container\" into \"%s\": %s", containerMntPoint, err)
- return errors.Wrap(err, "Extract")
- }
-
- return nil
-}
-
-func (s *storageZfs) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error {
- logger.Debugf("Loading ZFS storage volume for backup \"%s\" on storage pool \"%s\"", info.Name, s.pool.Name)
-
- if info.HasBinaryFormat {
- return s.doContainerBackupLoadOptimized(info, data, tarArgs)
- }
-
- return s.doContainerBackupLoadVanilla(info, data, tarArgs)
-}
-
-// - create temporary directory ${LXD_DIR}/images/lxd_images_
-// - create new zfs volume images/<fingerprint>
-// - mount the zfs volume on ${LXD_DIR}/images/lxd_images_
-// - unpack the downloaded image in ${LXD_DIR}/images/lxd_images_
-// - mark new zfs volume images/<fingerprint> readonly
-// - remove mountpoint property from zfs volume images/<fingerprint>
-// - create read-write snapshot from zfs volume images/<fingerprint>
-func (s *storageZfs) ImageCreate(fingerprint string, tracker *ioprogress.ProgressTracker) error {
- logger.Debugf("Creating ZFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
-
- poolName := s.getOnDiskPoolName()
- imageMntPoint := getImageMountPoint(s.pool.Name, fingerprint)
- fs := fmt.Sprintf("images/%s", fingerprint)
- revert := true
- subrevert := true
-
- err := s.createImageDbPoolVolume(fingerprint)
- if err != nil {
- return err
- }
- defer func() {
- if !subrevert {
- return
- }
- s.deleteImageDbPoolVolume(fingerprint)
- }()
-
- if zfsFilesystemEntityExists(poolName, fmt.Sprintf("deleted/%s", fs)) {
- if err := zfsPoolVolumeRename(poolName, fmt.Sprintf("deleted/%s", fs), fs, true); err != nil {
- return err
- }
-
- defer func() {
- if !revert {
- return
- }
- s.ImageDelete(fingerprint)
- }()
-
- // In case this is an image from an older lxd instance, wipe the
- // mountpoint.
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", "none")
- if err != nil {
- return err
- }
-
- revert = false
- subrevert = false
-
- return nil
- }
-
- if !shared.PathExists(imageMntPoint) {
- err := os.MkdirAll(imageMntPoint, 0700)
- if err != nil {
- return err
- }
- defer func() {
- if !subrevert {
- return
- }
- os.RemoveAll(imageMntPoint)
- }()
- }
-
- // Create temporary mountpoint directory.
- tmp := getImageMountPoint(s.pool.Name, "")
- tmpImageDir, err := ioutil.TempDir(tmp, "")
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpImageDir)
-
- imagePath := shared.VarPath("images", fingerprint)
-
- // Create a new storage volume on the storage pool for the image.
- dataset := fmt.Sprintf("%s/%s", poolName, fs)
- msg, err := zfsPoolVolumeCreate(dataset, "mountpoint=none")
- if err != nil {
- logger.Errorf("Failed to create ZFS dataset \"%s\" on storage pool \"%s\": %s", dataset, s.pool.Name, msg)
- return err
- }
- subrevert = false
- defer func() {
- if !revert {
- return
- }
- s.ImageDelete(fingerprint)
- }()
-
- // Set a temporary mountpoint for the image.
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", tmpImageDir)
- if err != nil {
- return err
- }
-
- // Make sure that the image actually got mounted.
- if !shared.IsMountPoint(tmpImageDir) {
- zfsMount(poolName, fs)
- }
-
- // Unpack the image into the temporary mountpoint.
- err = unpackImage(imagePath, tmpImageDir, storageTypeZfs, s.s.OS.RunningInUserNS, nil)
- if err != nil {
- return err
- }
-
- // Mark the new storage volume for the image as readonly.
- if err = zfsPoolVolumeSet(poolName, fs, "readonly", "on"); err != nil {
- return err
- }
-
- // Remove the temporary mountpoint from the image storage volume.
- if err = zfsPoolVolumeSet(poolName, fs, "mountpoint", "none"); err != nil {
- return err
- }
-
- // Make sure that the image actually got unmounted.
- if shared.IsMountPoint(tmpImageDir) {
- zfsUmount(poolName, fs, tmpImageDir)
- }
-
- // Create a snapshot of that image on the storage pool which we clone for
- // container creation.
- err = zfsPoolVolumeSnapshotCreate(poolName, fs, "readonly")
- if err != nil {
- return err
- }
-
- revert = false
-
- logger.Debugf("Created ZFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) ImageDelete(fingerprint string) error {
- logger.Debugf("Deleting ZFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
-
- poolName := s.getOnDiskPoolName()
- fs := fmt.Sprintf("images/%s", fingerprint)
-
- if zfsFilesystemEntityExists(poolName, fs) {
- removable, err := zfsPoolVolumeSnapshotRemovable(poolName, fs, "readonly")
- if err != nil && zfsFilesystemEntityExists(poolName, fmt.Sprintf("%s at readonly", fs)) {
- return err
- }
-
- if removable {
- err := zfsPoolVolumeDestroy(poolName, fs)
- if err != nil {
- return err
- }
- } else {
- if err := zfsPoolVolumeSet(poolName, fs, "mountpoint", "none"); err != nil {
- return err
- }
-
- if err := zfsPoolVolumeRename(poolName, fs, fmt.Sprintf("deleted/%s", fs), true); err != nil {
- return err
- }
- }
- }
-
- err := s.deleteImageDbPoolVolume(fingerprint)
- if err != nil {
- return err
- }
-
- imageMntPoint := getImageMountPoint(s.pool.Name, fingerprint)
- if shared.PathExists(imageMntPoint) {
- err := os.RemoveAll(imageMntPoint)
- if err != nil {
- return err
- }
- }
-
- if shared.PathExists(shared.VarPath(fs + ".zfs")) {
- err := os.RemoveAll(shared.VarPath(fs + ".zfs"))
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Deleted ZFS storage volume for image \"%s\" on storage pool \"%s\"", fingerprint, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) ImageMount(fingerprint string) (bool, error) {
- return true, nil
-}
-
-func (s *storageZfs) ImageUmount(fingerprint string) (bool, error) {
- return true, nil
-}
-
-type zfsMigrationSourceDriver struct {
- container container
- snapshots []container
- zfsSnapshotNames []string
- zfs *storageZfs
- runningSnapName string
- stoppedSnapName string
- zfsFeatures []string
-}
-
-func (s *zfsMigrationSourceDriver) send(conn *websocket.Conn, zfsName string, zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
- sourceParentName, _, _ := containerGetParentAndSnapshotName(s.container.Name())
- poolName := s.zfs.getOnDiskPoolName()
- args := []string{"send"}
-
- // Negotiated options
- if s.zfsFeatures != nil && len(s.zfsFeatures) > 0 {
- if shared.StringInSlice("compress", s.zfsFeatures) {
- args = append(args, "-c")
- args = append(args, "-L")
- }
- }
-
- args = append(args, []string{fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(s.container.Project(), sourceParentName), zfsName)}...)
- if zfsParent != "" {
- args = append(args, "-i", fmt.Sprintf("%s/containers/%s@%s", poolName, projectPrefix(s.container.Project(), s.container.Name()), zfsParent))
- }
-
- cmd := exec.Command("zfs", args...)
-
- stdout, err := cmd.StdoutPipe()
- if err != nil {
- return err
- }
-
- readPipe := io.ReadCloser(stdout)
- if readWrapper != nil {
- readPipe = readWrapper(stdout)
- }
-
- stderr, err := cmd.StderrPipe()
- if err != nil {
- return err
- }
-
- if err := cmd.Start(); err != nil {
- return err
- }
-
- <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
-
- output, err := ioutil.ReadAll(stderr)
- if err != nil {
- logger.Errorf("Problem reading zfs send stderr: %s", err)
- }
-
- err = cmd.Wait()
- if err != nil {
- logger.Errorf("Problem with zfs send: %s", string(output))
- }
-
- return err
-}
-
-func (s *zfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error {
- if s.container.IsSnapshot() {
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(s.container.Name())
- snapshotName := fmt.Sprintf("snapshot-%s", snapOnlyName)
- wrapper := StorageProgressReader(op, "fs_progress", s.container.Name())
- return s.send(conn, snapshotName, "", wrapper)
- }
-
- lastSnap := ""
- if !containerOnly {
- for i, snap := range s.zfsSnapshotNames {
- prev := ""
- if i > 0 {
- prev = s.zfsSnapshotNames[i-1]
- }
-
- lastSnap = snap
-
- wrapper := StorageProgressReader(op, "fs_progress", snap)
- if err := s.send(conn, snap, prev, wrapper); err != nil {
- return err
- }
- }
- }
-
- s.runningSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String())
- if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.runningSnapName); err != nil {
- return err
- }
-
- wrapper := StorageProgressReader(op, "fs_progress", s.container.Name())
- if err := s.send(conn, s.runningSnapName, lastSnap, wrapper); err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *zfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error {
- s.stoppedSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String())
- if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.stoppedSnapName); err != nil {
- return err
- }
-
- if err := s.send(conn, s.stoppedSnapName, s.runningSnapName, nil); err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *zfsMigrationSourceDriver) Cleanup() {
- poolName := s.zfs.getOnDiskPoolName()
- if s.stoppedSnapName != "" {
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.stoppedSnapName)
- }
- if s.runningSnapName != "" {
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(s.container.Project(), s.container.Name())), s.runningSnapName)
- }
-}
-
-func (s *storageZfs) MigrationType() migration.MigrationFSType {
- return migration.MigrationFSType_ZFS
-}
-
-func (s *storageZfs) PreservesInodes() bool {
- return true
-}
-
-func (s *storageZfs) MigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
- /* If the container is a snapshot, let's just send that; we don't need
- * to send anything else, because that's all the user asked for.
- */
- if args.Container.IsSnapshot() {
- return &zfsMigrationSourceDriver{container: args.Container, zfs: s, zfsFeatures: args.ZfsFeatures}, nil
- }
-
- driver := zfsMigrationSourceDriver{
- container: args.Container,
- snapshots: []container{},
- zfsSnapshotNames: []string{},
- zfs: s,
- zfsFeatures: args.ZfsFeatures,
- }
-
- if args.ContainerOnly {
- return &driver, nil
- }
-
- /* List all the snapshots in order of reverse creation. The idea here
- * is that we send the oldest to newest snapshot, hopefully saving on
- * xfer costs. Then, after all that, we send the container itself.
- */
- snapshots, err := zfsPoolListSnapshots(s.getOnDiskPoolName(), fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name())))
- if err != nil {
- return nil, err
- }
-
- for _, snap := range snapshots {
- /* In the case of e.g. multiple copies running at the same
- * time, we will have potentially multiple migration-send
- * snapshots. (Or in the case of the test suite, sometimes one
- * will take too long to delete.)
- */
- if !strings.HasPrefix(snap, "snapshot-") {
- continue
- }
-
- lxdName := fmt.Sprintf("%s%s%s", args.Container.Name(), shared.SnapshotDelimiter, snap[len("snapshot-"):])
- snapshot, err := containerLoadByProjectAndName(s.s, args.Container.Project(), lxdName)
- if err != nil {
- return nil, err
- }
-
- driver.snapshots = append(driver.snapshots, snapshot)
- driver.zfsSnapshotNames = append(driver.zfsSnapshotNames, snap)
- }
-
- return &driver, nil
-}
-
-func (s *storageZfs) MigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
- poolName := s.getOnDiskPoolName()
- zfsRecv := func(zfsName string, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
- zfsFsName := fmt.Sprintf("%s/%s", poolName, zfsName)
- args := []string{"receive", "-F", "-u", zfsFsName}
- cmd := exec.Command("zfs", args...)
-
- stdin, err := cmd.StdinPipe()
- if err != nil {
- return err
- }
-
- stderr, err := cmd.StderrPipe()
- if err != nil {
- return err
- }
-
- if err := cmd.Start(); err != nil {
- return err
- }
-
- writePipe := io.WriteCloser(stdin)
- if writeWrapper != nil {
- writePipe = writeWrapper(stdin)
- }
-
- <-shared.WebsocketRecvStream(writePipe, conn)
-
- output, err := ioutil.ReadAll(stderr)
- if err != nil {
- logger.Debugf("Problem reading zfs recv stderr %s", err)
- }
-
- err = cmd.Wait()
- if err != nil {
- logger.Errorf("Problem with zfs recv: %s", string(output))
- }
- return err
- }
-
- /* In some versions of zfs we can write `zfs recv -F` to mounted
- * filesystems, and in some versions we can't. So, let's always unmount
- * this fs (it's empty anyway) before we zfs recv. N.B. that `zfs recv`
- * of a snapshot also needs tha actual fs that it has snapshotted
- * unmounted, so we do this before receiving anything.
- */
- zfsName := fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name()))
- containerMntPoint := getContainerMountPoint(args.Container.Project(), s.pool.Name, args.Container.Name())
- if shared.IsMountPoint(containerMntPoint) {
- err := zfsUmount(poolName, zfsName, containerMntPoint)
- if err != nil {
- return err
- }
- }
-
- if len(args.Snapshots) > 0 {
- snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", projectPrefix(args.Container.Project(), s.volume.Name))
- snapshotMntPointSymlink := shared.VarPath("snapshots", projectPrefix(args.Container.Project(), args.Container.Name()))
- if !shared.PathExists(snapshotMntPointSymlink) {
- err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
- if err != nil {
- return err
- }
- }
- }
-
- // At this point we have already figured out the parent
- // container's root disk device so we can simply
- // retrieve it from the expanded devices.
- parentStoragePool := ""
- parentExpandedDevices := args.Container.ExpandedDevices()
- parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := shared.GetRootDiskDevice(parentExpandedDevices)
- if parentLocalRootDiskDeviceKey != "" {
- parentStoragePool = parentLocalRootDiskDevice["pool"]
- }
-
- // A little neuroticism.
- if parentStoragePool == "" {
- return fmt.Errorf("detected that the container's root device is missing the pool property during BTRFS migration")
- }
-
- for _, snap := range args.Snapshots {
- ctArgs := snapshotProtobufToContainerArgs(args.Container.Project(), args.Container.Name(), snap)
-
- // Ensure that snapshot and parent container have the
- // same storage pool in their local root disk device.
- // If the root disk device for the snapshot comes from a
- // profile on the new instance as well we don't need to
- // do anything.
- if ctArgs.Devices != nil {
- snapLocalRootDiskDeviceKey, _, _ := shared.GetRootDiskDevice(ctArgs.Devices)
- if snapLocalRootDiskDeviceKey != "" {
- ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
- }
- }
- _, err := containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs)
- if err != nil {
- return err
- }
-
- wrapper := StorageProgressWriter(op, "fs_progress", snap.GetName())
- name := fmt.Sprintf("containers/%s at snapshot-%s", projectPrefix(args.Container.Project(), args.Container.Name()), snap.GetName())
- if err := zfsRecv(name, wrapper); err != nil {
- return err
- }
-
- snapshotMntPoint := getSnapshotMountPoint(args.Container.Project(), poolName, fmt.Sprintf("%s/%s", args.Container.Name(), *snap.Name))
- if !shared.PathExists(snapshotMntPoint) {
- err := os.MkdirAll(snapshotMntPoint, 0700)
- if err != nil {
- return err
- }
- }
- }
-
- defer func() {
- /* clean up our migration-send snapshots that we got from recv. */
- zfsSnapshots, err := zfsPoolListSnapshots(poolName, fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name())))
- if err != nil {
- logger.Errorf("Failed listing snapshots post migration: %s", err)
- return
- }
-
- for _, snap := range zfsSnapshots {
- // If we received a bunch of snapshots, remove the migration-send-* ones, if not, wipe any snapshot we got
- if args.Snapshots != nil && len(args.Snapshots) > 0 && !strings.HasPrefix(snap, "migration-send") {
- continue
- }
-
- zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", projectPrefix(args.Container.Project(), args.Container.Name())), snap)
- }
- }()
-
- /* finally, do the real container */
- wrapper := StorageProgressWriter(op, "fs_progress", args.Container.Name())
- if err := zfsRecv(zfsName, wrapper); err != nil {
- return err
- }
-
- if args.Live {
- /* and again for the post-running snapshot if this was a live migration */
- wrapper := StorageProgressWriter(op, "fs_progress", args.Container.Name())
- if err := zfsRecv(zfsName, wrapper); err != nil {
- return err
- }
- }
-
- /* Sometimes, zfs recv mounts this anyway, even if we pass -u
- * (https://forums.freebsd.org/threads/zfs-receive-u-shouldnt-mount-received-filesystem-right.36844/)
- * but sometimes it doesn't. Let's try to mount, but not complain about
- * failure.
- */
- zfsMount(poolName, zfsName)
- return nil
-}
-
-func (s *storageZfs) StorageEntitySetQuota(volumeType int, size int64, data interface{}) error {
- logger.Debugf(`Setting ZFS quota for "%s"`, s.volume.Name)
-
- if !shared.IntInSlice(volumeType, supportedVolumeTypes) {
- return fmt.Errorf("Invalid storage type")
- }
-
- var c container
- var fs string
- switch volumeType {
- case storagePoolVolumeTypeContainer:
- c = data.(container)
- fs = fmt.Sprintf("containers/%s", projectPrefix(c.Project(), c.Name()))
- case storagePoolVolumeTypeCustom:
- fs = fmt.Sprintf("custom/%s", s.volume.Name)
- }
-
- property := "quota"
-
- if s.pool.Config["volume.zfs.use_refquota"] != "" {
- zfsUseRefquota = s.pool.Config["volume.zfs.use_refquota"]
- }
- if s.volume.Config["zfs.use_refquota"] != "" {
- zfsUseRefquota = s.volume.Config["zfs.use_refquota"]
- }
-
- if shared.IsTrue(zfsUseRefquota) {
- property = "refquota"
- }
-
- poolName := s.getOnDiskPoolName()
- var err error
- if size > 0 {
- err = zfsPoolVolumeSet(poolName, fs, property, fmt.Sprintf("%d", size))
- } else {
- err = zfsPoolVolumeSet(poolName, fs, property, "none")
- }
-
- if err != nil {
- return err
- }
-
- logger.Debugf(`Set ZFS quota for "%s"`, s.volume.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolResources() (*api.ResourcesStoragePool, error) {
- poolName := s.getOnDiskPoolName()
-
- totalBuf, err := zfsFilesystemEntityPropertyGet(poolName, "", "available")
- if err != nil {
- return nil, err
- }
-
- totalStr := string(totalBuf)
- totalStr = strings.TrimSpace(totalStr)
- total, err := strconv.ParseUint(totalStr, 10, 64)
- if err != nil {
- return nil, err
- }
-
- usedBuf, err := zfsFilesystemEntityPropertyGet(poolName, "", "used")
- if err != nil {
- return nil, err
- }
-
- usedStr := string(usedBuf)
- usedStr = strings.TrimSpace(usedStr)
- used, err := strconv.ParseUint(usedStr, 10, 64)
- if err != nil {
- return nil, err
- }
-
- res := api.ResourcesStoragePool{}
- res.Space.Total = total
- res.Space.Used = used
-
- // Inode allocation is dynamic so no use in reporting them.
-
- return &res, nil
-}
-
-func (s *storageZfs) doCrossPoolStorageVolumeCopy(source *api.StorageVolumeSource) error {
- successMsg := fmt.Sprintf("Copied ZFS storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
- // setup storage for the source volume
- srcStorage, err := storagePoolVolumeInit(s.s, "default", source.Pool, source.Name, storagePoolVolumeTypeCustom)
- if err != nil {
- logger.Errorf("Failed to initialize ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- ourMount, err := srcStorage.StoragePoolMount()
- if err != nil {
- return err
- }
- if ourMount {
- defer srcStorage.StoragePoolUmount()
- }
-
- // Create the main volume
- err = s.StoragePoolVolumeCreate()
- if err != nil {
- logger.Errorf("Failed to create ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- ourMount, err = s.StoragePoolVolumeMount()
- if err != nil {
- logger.Errorf("Failed to mount ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
- if ourMount {
- defer s.StoragePoolVolumeUmount()
- }
-
- dstMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- snapshots, err := storagePoolVolumeSnapshotsGet(s.s, source.Pool, source.Name, storagePoolVolumeTypeCustom)
- if err != nil {
- return err
- }
-
- if !source.VolumeOnly {
- for _, snap := range snapshots {
- srcMountPoint := getStoragePoolVolumeSnapshotMountPoint(source.Pool, snap)
-
- _, err = rsyncLocalCopy(srcMountPoint, dstMountPoint, bwlimit)
- if err != nil {
- logger.Errorf("Failed to rsync into ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- _, snapOnlyName, _ := containerGetParentAndSnapshotName(source.Name)
-
- s.StoragePoolVolumeSnapshotCreate(&api.StorageVolumeSnapshotsPost{Name: fmt.Sprintf("%s/%s", s.volume.Name, snapOnlyName)})
- }
- }
-
- var srcMountPoint string
-
- if strings.Contains(source.Name, "/") {
- srcMountPoint = getStoragePoolVolumeSnapshotMountPoint(source.Pool, source.Name)
- } else {
- srcMountPoint = getStoragePoolVolumeMountPoint(source.Pool, source.Name)
- }
-
- _, err = rsyncLocalCopy(srcMountPoint, dstMountPoint, bwlimit)
- if err != nil {
- os.RemoveAll(dstMountPoint)
- logger.Errorf("Failed to rsync into ZFS storage volume \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, err)
- return err
- }
-
- logger.Infof(successMsg)
- return nil
-}
-
-func (s *storageZfs) copyVolumeWithoutSnapshotsFull(source *api.StorageVolumeSource) error {
- sourceIsSnapshot := shared.IsSnapshot(source.Name)
-
- var snapshotSuffix string
- var sourceDataset string
- var targetDataset string
- var targetSnapshotDataset string
-
- poolName := s.getOnDiskPoolName()
-
- if sourceIsSnapshot {
- sourceVolumeName, sourceSnapOnlyName, _ := containerGetParentAndSnapshotName(source.Name)
- snapshotSuffix = fmt.Sprintf("snapshot-%s", sourceSnapOnlyName)
- sourceDataset = fmt.Sprintf("%s/custom/%s@%s", source.Pool, sourceVolumeName, snapshotSuffix)
- targetSnapshotDataset = fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, s.volume.Name, sourceSnapOnlyName)
- } else {
- snapshotSuffix = uuid.NewRandom().String()
- sourceDataset = fmt.Sprintf("%s/custom/%s@%s", poolName, source.Name, snapshotSuffix)
- targetSnapshotDataset = fmt.Sprintf("%s/custom/%s@%s", poolName, s.volume.Name, snapshotSuffix)
-
- fs := fmt.Sprintf("custom/%s", source.Name)
- err := zfsPoolVolumeSnapshotCreate(poolName, fs, snapshotSuffix)
- if err != nil {
- return err
- }
- defer func() {
- err := zfsPoolVolumeSnapshotDestroy(poolName, fs, snapshotSuffix)
- if err != nil {
- logger.Warnf("Failed to delete temporary ZFS snapshot \"%s\", manual cleanup needed", sourceDataset)
- }
- }()
- }
-
- zfsSendCmd := exec.Command("zfs", "send", sourceDataset)
-
- zfsRecvCmd := exec.Command("zfs", "receive", targetDataset)
-
- zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
- zfsRecvCmd.Stdout = os.Stdout
- zfsRecvCmd.Stderr = os.Stderr
-
- err := zfsRecvCmd.Start()
- if err != nil {
- return err
- }
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- err = zfsRecvCmd.Wait()
- if err != nil {
- return err
- }
-
- msg, err := shared.RunCommand("zfs", "rollback", "-r", "-R", targetSnapshotDataset)
- if err != nil {
- logger.Errorf("Failed to rollback ZFS dataset: %s", msg)
- return err
- }
-
- targetContainerMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- targetfs := fmt.Sprintf("custom/%s", s.volume.Name)
-
- err = zfsPoolVolumeSet(poolName, targetfs, "canmount", "noauto")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(poolName, targetfs, "mountpoint", targetContainerMountPoint)
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSnapshotDestroy(poolName, targetfs, snapshotSuffix)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *storageZfs) copyVolumeWithoutSnapshotsSparse(source *api.StorageVolumeSource) error {
- poolName := s.getOnDiskPoolName()
-
- sourceVolumeName := source.Name
- sourceVolumePath := getStoragePoolVolumeMountPoint(source.Pool, source.Name)
-
- targetVolumeName := s.volume.Name
- targetVolumePath := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
-
- sourceZfsDataset := ""
- sourceZfsDatasetSnapshot := ""
- sourceName, sourceSnapOnlyName, isSnapshotName := containerGetParentAndSnapshotName(sourceVolumeName)
-
- targetZfsDataset := fmt.Sprintf("custom/%s", targetVolumeName)
-
- if isSnapshotName {
- sourceZfsDatasetSnapshot = sourceSnapOnlyName
- }
-
- revert := true
- if sourceZfsDatasetSnapshot == "" {
- if zfsFilesystemEntityExists(poolName, fmt.Sprintf("custom/%s", sourceName)) {
- sourceZfsDatasetSnapshot = fmt.Sprintf("copy-%s", uuid.NewRandom().String())
- sourceZfsDataset = fmt.Sprintf("custom/%s", sourceName)
-
- err := zfsPoolVolumeSnapshotCreate(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
- if err != nil {
- return err
- }
-
- defer func() {
- if !revert {
- return
- }
- zfsPoolVolumeSnapshotDestroy(poolName, sourceZfsDataset, sourceZfsDatasetSnapshot)
- }()
- }
- } else {
- if zfsFilesystemEntityExists(poolName, fmt.Sprintf("custom/%s at snapshot-%s", sourceName, sourceZfsDatasetSnapshot)) {
- sourceZfsDataset = fmt.Sprintf("custom/%s", sourceName)
- sourceZfsDatasetSnapshot = fmt.Sprintf("snapshot-%s", sourceZfsDatasetSnapshot)
- }
- }
-
- if sourceZfsDataset != "" {
- err := zfsPoolVolumeClone("default", poolName, sourceZfsDataset, sourceZfsDatasetSnapshot, targetZfsDataset, targetVolumePath)
- if err != nil {
- return err
- }
-
- defer func() {
- if !revert {
- return
- }
- zfsPoolVolumeDestroy(poolName, targetZfsDataset)
- }()
- } else {
- bwlimit := s.pool.Config["rsync.bwlimit"]
-
- output, err := rsyncLocalCopy(sourceVolumePath, targetVolumePath, bwlimit)
- if err != nil {
- return fmt.Errorf("rsync failed: %s", string(output))
- }
- }
-
- revert = false
-
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeCopy(source *api.StorageVolumeSource) error {
- logger.Infof("Copying ZFS storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
- successMsg := fmt.Sprintf("Copied ZFS storage volume \"%s\" on storage pool \"%s\" as \"%s\" to storage pool \"%s\"", source.Name, source.Pool, s.volume.Name, s.pool.Name)
-
- if source.Pool != s.pool.Name {
- return s.doCrossPoolStorageVolumeCopy(source)
- }
-
- var snapshots []string
-
- poolName := s.getOnDiskPoolName()
-
- if !shared.IsSnapshot(source.Name) {
- allSnapshots, err := zfsPoolListSnapshots(poolName, fmt.Sprintf("custom/%s", source.Name))
- if err != nil {
- return err
- }
-
- for _, snap := range allSnapshots {
- if strings.HasPrefix(snap, "snapshot-") {
- snapshots = append(snapshots, strings.TrimPrefix(snap, "snapshot-"))
- }
- }
- }
-
- targetStorageVolumeMountPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
- fs := fmt.Sprintf("custom/%s", s.volume.Name)
-
- if source.VolumeOnly || len(snapshots) == 0 {
- var err error
-
- if s.pool.Config["zfs.clone_copy"] != "" && !shared.IsTrue(s.pool.Config["zfs.clone_copy"]) {
- err = s.copyVolumeWithoutSnapshotsFull(source)
- } else {
- err = s.copyVolumeWithoutSnapshotsSparse(source)
- }
- if err != nil {
- return err
- }
- } else {
- targetVolumeMountPoint := getStoragePoolVolumeMountPoint(poolName, s.volume.Name)
-
- err := os.MkdirAll(targetVolumeMountPoint, 0711)
- if err != nil {
- return err
- }
-
- prev := ""
- prevSnapOnlyName := ""
-
- for i, snap := range snapshots {
- if i > 0 {
- prev = snapshots[i-1]
- }
-
- sourceDataset := fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, source.Name, snap)
- targetDataset := fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, s.volume.Name, snap)
-
- snapshotMntPoint := getStoragePoolVolumeSnapshotMountPoint(poolName, fmt.Sprintf("%s/%s", s.volume.Name, snap))
-
- err := os.MkdirAll(snapshotMntPoint, 0700)
- if err != nil {
- return err
- }
-
- prevSnapOnlyName = snap
-
- args := []string{"send", sourceDataset}
-
- if prev != "" {
- parentDataset := fmt.Sprintf("%s/custom/%s/snapshot-%s", poolName, source.Name, prev)
- args = append(args, "-i", parentDataset)
- }
-
- zfsSendCmd := exec.Command("zfs", args...)
- zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetDataset)
-
- zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
- zfsRecvCmd.Stdout = os.Stdout
- zfsRecvCmd.Stderr = os.Stderr
-
- err = zfsRecvCmd.Start()
- if err != nil {
- return err
- }
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- err = zfsRecvCmd.Wait()
- if err != nil {
- return err
- }
- }
-
- tmpSnapshotName := fmt.Sprintf("copy-send-%s", uuid.NewRandom().String())
-
- err = zfsPoolVolumeSnapshotCreate(poolName, fmt.Sprintf("custom/%s", source.Name), tmpSnapshotName)
- if err != nil {
- return err
- }
-
- defer zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("custom/%s", source.Name), tmpSnapshotName)
-
- currentSnapshotDataset := fmt.Sprintf("%s/custom/%s@%s", poolName, source.Name, tmpSnapshotName)
-
- args := []string{"send", currentSnapshotDataset}
-
- if prevSnapOnlyName != "" {
- args = append(args, "-i", fmt.Sprintf("%s/custom/%s at snapshot-%s", poolName, source.Name, prevSnapOnlyName))
- }
-
- zfsSendCmd := exec.Command("zfs", args...)
- targetDataset := fmt.Sprintf("%s/custom/%s", poolName, s.volume.Name)
- zfsRecvCmd := exec.Command("zfs", "receive", "-F", targetDataset)
-
- zfsRecvCmd.Stdin, _ = zfsSendCmd.StdoutPipe()
- zfsRecvCmd.Stdout = os.Stdout
- zfsRecvCmd.Stderr = os.Stderr
-
- err = zfsRecvCmd.Start()
- if err != nil {
- return err
- }
-
- err = zfsSendCmd.Run()
- if err != nil {
- return err
- }
-
- err = zfsRecvCmd.Wait()
- if err != nil {
- return err
- }
-
- defer zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("custom/%s", s.volume.Name), tmpSnapshotName)
-
- err = zfsPoolVolumeSet(poolName, fs, "canmount", "noauto")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", targetStorageVolumeMountPoint)
- if err != nil {
- return err
- }
- }
-
- if !shared.IsMountPoint(targetStorageVolumeMountPoint) {
- err := zfsMount(poolName, fs)
- if err != nil {
- return err
- }
- defer zfsUmount(poolName, fs, targetStorageVolumeMountPoint)
- }
-
- // apply quota
- if s.volume.Config["size"] != "" {
- size, err := shared.ParseByteSizeString(s.volume.Config["size"])
- if err != nil {
- return err
- }
-
- err = s.StorageEntitySetQuota(storagePoolVolumeTypeCustom, size, nil)
- if err != nil {
- return err
- }
- }
-
- logger.Infof(successMsg)
- return nil
-}
-
-func (s *zfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error {
- msg := fmt.Sprintf("Function not implemented")
- logger.Errorf(msg)
- return fmt.Errorf(msg)
-}
-
-func (s *storageZfs) StorageMigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) {
- return rsyncStorageMigrationSource(args)
-}
-
-func (s *storageZfs) StorageMigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error {
- return rsyncStorageMigrationSink(conn, op, args)
-}
-
-func (s *storageZfs) StoragePoolVolumeSnapshotCreate(target *api.StorageVolumeSnapshotsPost) error {
- logger.Infof("Creating ZFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- sourceOnlyName, snapshotOnlyName, ok := containerGetParentAndSnapshotName(target.Name)
- if !ok {
- return fmt.Errorf("Not a snapshot name")
- }
-
- sourceDataset := fmt.Sprintf("custom/%s", sourceOnlyName)
- poolName := s.getOnDiskPoolName()
- snapName := fmt.Sprintf("snapshot-%s", snapshotOnlyName)
- err := zfsPoolVolumeSnapshotCreate(poolName, sourceDataset, snapName)
- if err != nil {
- return err
- }
-
- snapshotMntPoint := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, target.Name)
- if !shared.PathExists(snapshotMntPoint) {
- err := os.MkdirAll(snapshotMntPoint, 0700)
- if err != nil {
- return err
- }
- }
-
- logger.Infof("Created ZFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeSnapshotDelete() error {
- logger.Infof("Deleting ZFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- sourceName, snapshotOnlyName, _ := containerGetParentAndSnapshotName(s.volume.Name)
- snapshotName := fmt.Sprintf("snapshot-%s", snapshotOnlyName)
-
- onDiskPoolName := s.getOnDiskPoolName()
- if zfsFilesystemEntityExists(onDiskPoolName, fmt.Sprintf("custom/%s@%s", sourceName, snapshotName)) {
- removable, err := zfsPoolVolumeSnapshotRemovable(onDiskPoolName, fmt.Sprintf("custom/%s", sourceName), snapshotName)
- if err != nil {
- return err
- }
-
- if removable {
- err = zfsPoolVolumeSnapshotDestroy(onDiskPoolName, fmt.Sprintf("custom/%s", sourceName), snapshotName)
- } else {
- err = zfsPoolVolumeSnapshotRename(onDiskPoolName, fmt.Sprintf("custom/%s", sourceName), snapshotName, fmt.Sprintf("copy-%s", uuid.NewRandom().String()))
- }
- if err != nil {
- return err
- }
- }
-
- storageVolumePath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, s.volume.Name)
- err := os.RemoveAll(storageVolumePath)
- if err != nil && !os.IsNotExist(err) {
- return err
- }
-
- storageVolumeSnapshotPath := getStoragePoolVolumeSnapshotMountPoint(s.pool.Name, sourceName)
- empty, err := shared.PathIsEmpty(storageVolumeSnapshotPath)
- if err == nil && empty {
- os.RemoveAll(storageVolumeSnapshotPath)
- }
-
- err = s.s.Cluster.StoragePoolVolumeDelete(
- "default",
- s.volume.Name,
- storagePoolVolumeTypeCustom,
- s.poolID)
- if err != nil {
- logger.Errorf(`Failed to delete database entry for DIR storage volume "%s" on storage pool "%s"`,
- s.volume.Name, s.pool.Name)
- }
-
- logger.Infof("Deleted ZFS storage volume snapshot \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) StoragePoolVolumeSnapshotRename(newName string) error {
- logger.Infof("Renaming ZFS storage volume snapshot on storage pool \"%s\" from \"%s\" to \"%s\"", s.pool.Name, s.volume.Name, newName)
-
- sourceName, snapshotOnlyName, ok := containerGetParentAndSnapshotName(s.volume.Name)
- if !ok {
- return fmt.Errorf("Not a snapshot name")
- }
-
- oldZfsDatasetName := fmt.Sprintf("snapshot-%s", snapshotOnlyName)
- newZfsDatasetName := fmt.Sprintf("snapshot-%s", newName)
- err := zfsPoolVolumeSnapshotRename(s.getOnDiskPoolName(), fmt.Sprintf("custom/%s", sourceName), oldZfsDatasetName, newZfsDatasetName)
- if err != nil {
- return err
- }
-
- logger.Infof("Renamed ZFS storage volume snapshot on storage pool \"%s\" from \"%s\" to \"%s\"", s.pool.Name, s.volume.Name, newName)
-
- return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, fmt.Sprintf("%s/%s", sourceName, newName), storagePoolVolumeTypeCustom, s.poolID)
-}
diff --git a/lxd/storage_zfs_utils.go b/lxd/storage_zfs_utils.go
deleted file mode 100644
index a5f0eaaef6..0000000000
--- a/lxd/storage_zfs_utils.go
+++ /dev/null
@@ -1,833 +0,0 @@
-package main
-
-import (
- "fmt"
- "io/ioutil"
- "os"
- "os/exec"
- "path/filepath"
- "strings"
- "syscall"
- "time"
-
- "github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/logger"
-
- "github.com/pborman/uuid"
-)
-
-// zfsIsEnabled returns whether zfs backend is supported.
-func zfsIsEnabled() bool {
- out, err := exec.LookPath("zfs")
- if err != nil || len(out) == 0 {
- return false
- }
-
- return true
-}
-
-// zfsToolVersionGet returns the ZFS tools version
-func zfsToolVersionGet() (string, error) {
- // This function is only really ever relevant on Ubuntu as the only
- // distro that ships out of sync tools and kernel modules
- out, err := shared.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux")
- if err != nil {
- return "", err
- }
-
- return strings.TrimSpace(string(out)), nil
-}
-
-// zfsModuleVersionGet returns the ZFS module version
-func zfsModuleVersionGet() (string, error) {
- var zfsVersion string
-
- if shared.PathExists("/sys/module/zfs/version") {
- out, err := ioutil.ReadFile("/sys/module/zfs/version")
- if err != nil {
- return "", fmt.Errorf("Could not determine ZFS module version")
- }
-
- zfsVersion = string(out)
- } else {
- out, err := shared.RunCommand("modinfo", "-F", "version", "zfs")
- if err != nil {
- return "", fmt.Errorf("Could not determine ZFS module version")
- }
-
- zfsVersion = out
- }
-
- return strings.TrimSpace(zfsVersion), nil
-}
-
-// zfsPoolVolumeCreate creates a ZFS dataset with a set of given properties.
-func zfsPoolVolumeCreate(dataset string, properties ...string) (string, error) {
- cmd := []string{"zfs", "create"}
-
- for _, prop := range properties {
- cmd = append(cmd, []string{"-o", prop}...)
- }
-
- cmd = append(cmd, []string{"-p", dataset}...)
-
- return shared.RunCommand(cmd[0], cmd[1:]...)
-}
-
-func zfsPoolCheck(pool string) error {
- output, err := shared.RunCommand(
- "zfs", "get", "-H", "-o", "value", "type", pool)
- if err != nil {
- return fmt.Errorf(strings.Split(output, "\n")[0])
- }
-
- poolType := strings.Split(output, "\n")[0]
- if poolType != "filesystem" {
- return fmt.Errorf("Unsupported pool type: %s", poolType)
- }
-
- return nil
-}
-
-// zfsPoolVolumeExists verifies if a specific ZFS pool or volume exists.
-func zfsPoolVolumeExists(dataset string) (bool, error) {
- output, err := shared.RunCommand(
- "zfs", "list", "-Ho", "name")
-
- if err != nil {
- return false, err
- }
-
- for _, name := range strings.Split(output, "\n") {
- if name == dataset {
- return true, nil
- }
- }
- return false, nil
-}
-
-func zfsPoolCreate(pool string, vdev string) error {
- var output string
- var err error
-
- dataset := ""
-
- if pool == "" {
- output, err := shared.RunCommand(
- "zfs", "create", "-p", "-o", "mountpoint=none", vdev)
- if err != nil {
- logger.Errorf("zfs create failed: %s", output)
- return fmt.Errorf("Failed to create ZFS filesystem: %s", output)
- }
- dataset = vdev
- } else {
- output, err = shared.RunCommand(
- "zpool", "create", "-f", "-m", "none", "-O", "compression=on", pool, vdev)
- if err != nil {
- logger.Errorf("zfs create failed: %s", output)
- return fmt.Errorf("Failed to create the ZFS pool: %s", output)
- }
-
- dataset = pool
- }
-
- err = zfsPoolApplyDefaults(dataset)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func zfsPoolApplyDefaults(dataset string) error {
- err := zfsPoolVolumeSet(dataset, "", "mountpoint", "none")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(dataset, "", "setuid", "on")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(dataset, "", "exec", "on")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(dataset, "", "devices", "on")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(dataset, "", "acltype", "posixacl")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeSet(dataset, "", "xattr", "sa")
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func zfsPoolVolumeClone(project, pool string, source string, name string, dest string, mountpoint string) error {
- output, err := shared.RunCommand(
- "zfs",
- "clone",
- "-p",
- "-o", fmt.Sprintf("mountpoint=%s", mountpoint),
- "-o", "canmount=noauto",
- fmt.Sprintf("%s/%s@%s", pool, source, name),
- fmt.Sprintf("%s/%s", pool, dest))
- if err != nil {
- logger.Errorf("zfs clone failed: %s", output)
- return fmt.Errorf("Failed to clone the filesystem: %s", output)
- }
-
- subvols, err := zfsPoolListSubvolumes(pool, fmt.Sprintf("%s/%s", pool, source))
- if err != nil {
- return err
- }
-
- for _, sub := range subvols {
- snaps, err := zfsPoolListSnapshots(pool, sub)
- if err != nil {
- return err
- }
-
- if !shared.StringInSlice(name, snaps) {
- continue
- }
-
- destSubvol := dest + strings.TrimPrefix(sub, source)
- snapshotMntPoint := getSnapshotMountPoint(project, pool, destSubvol)
-
- output, err := shared.RunCommand(
- "zfs",
- "clone",
- "-p",
- "-o", fmt.Sprintf("mountpoint=%s", snapshotMntPoint),
- "-o", "canmount=noauto",
- fmt.Sprintf("%s/%s@%s", pool, sub, name),
- fmt.Sprintf("%s/%s", pool, destSubvol))
- if err != nil {
- logger.Errorf("zfs clone failed: %s", output)
- return fmt.Errorf("Failed to clone the sub-volume: %s", output)
- }
- }
-
- return nil
-}
-
-func zfsFilesystemEntityDelete(vdev string, pool string) error {
- var output string
- var err error
- if strings.Contains(pool, "/") {
- // Command to destroy a zfs dataset.
- output, err = shared.RunCommand("zfs", "destroy", "-r", pool)
- } else {
- // Command to destroy a zfs pool.
- output, err = shared.RunCommand("zpool", "destroy", "-f", pool)
- }
- if err != nil {
- return fmt.Errorf("Failed to delete the ZFS pool: %s", output)
- }
-
- // Cleanup storage
- if filepath.IsAbs(vdev) && !shared.IsBlockdevPath(vdev) {
- os.RemoveAll(vdev)
- }
-
- return nil
-}
-
-func zfsPoolVolumeDestroy(pool string, path string) error {
- mountpoint, err := zfsFilesystemEntityPropertyGet(pool, path, "mountpoint")
- if err != nil {
- return err
- }
-
- if mountpoint != "none" && shared.IsMountPoint(mountpoint) {
- err := syscall.Unmount(mountpoint, syscall.MNT_DETACH)
- if err != nil {
- logger.Errorf("umount failed: %s", err)
- return err
- }
- }
-
- // Due to open fds or kernel refs, this may fail for a bit, give it 10s
- output, err := shared.TryRunCommand(
- "zfs",
- "destroy",
- "-r",
- fmt.Sprintf("%s/%s", pool, path))
-
- if err != nil {
- logger.Errorf("zfs destroy failed: %s", output)
- return fmt.Errorf("Failed to destroy ZFS filesystem: %s", output)
- }
-
- return nil
-}
-
-func zfsPoolVolumeCleanup(pool string, path string) error {
- if strings.HasPrefix(path, "deleted/") {
- // Cleanup of filesystems kept for refcount reason
- removablePath, err := zfsPoolVolumeSnapshotRemovable(pool, path, "")
- if err != nil {
- return err
- }
-
- // Confirm that there are no more clones
- if removablePath {
- if strings.Contains(path, "@") {
- // Cleanup snapshots
- err = zfsPoolVolumeDestroy(pool, path)
- if err != nil {
- return err
- }
-
- // Check if the parent can now be deleted
- subPath := strings.SplitN(path, "@", 2)[0]
- snaps, err := zfsPoolListSnapshots(pool, subPath)
- if err != nil {
- return err
- }
-
- if len(snaps) == 0 {
- err := zfsPoolVolumeCleanup(pool, subPath)
- if err != nil {
- return err
- }
- }
- } else {
- // Cleanup filesystems
- origin, err := zfsFilesystemEntityPropertyGet(pool, path, "origin")
- if err != nil {
- return err
- }
- origin = strings.TrimPrefix(origin, fmt.Sprintf("%s/", pool))
-
- err = zfsPoolVolumeDestroy(pool, path)
- if err != nil {
- return err
- }
-
- // Attempt to remove its parent
- if origin != "-" {
- err := zfsPoolVolumeCleanup(pool, origin)
- if err != nil {
- return err
- }
- }
- }
-
- return nil
- }
- } else if strings.HasPrefix(path, "containers") && strings.Contains(path, "@copy-") {
- // Just remove the copy- snapshot for copies of active containers
- err := zfsPoolVolumeDestroy(pool, path)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func zfsFilesystemEntityPropertyGet(pool string, path string, key string) (string, error) {
- entity := pool
- if path != "" {
- entity = fmt.Sprintf("%s/%s", pool, path)
- }
- output, err := shared.RunCommand(
- "zfs",
- "get",
- "-H",
- "-p",
- "-o", "value",
- key,
- entity)
- if err != nil {
- return "", fmt.Errorf("Failed to get ZFS config: %s", output)
- }
-
- return strings.TrimRight(output, "\n"), nil
-}
-
-func zfsPoolVolumeRename(pool string, source string, dest string, ignoreMounts bool) error {
- var err error
- var output string
-
- for i := 0; i < 20; i++ {
- if ignoreMounts {
- output, err = shared.RunCommand(
- "/proc/self/exe",
- "forkzfs",
- "--",
- "rename",
- "-p",
- fmt.Sprintf("%s/%s", pool, source),
- fmt.Sprintf("%s/%s", pool, dest))
- } else {
- output, err = shared.RunCommand(
- "zfs",
- "rename",
- "-p",
- fmt.Sprintf("%s/%s", pool, source),
- fmt.Sprintf("%s/%s", pool, dest))
- }
-
- // Success
- if err == nil {
- return nil
- }
-
- // zfs rename can fail because of descendants, yet still manage the rename
- if !zfsFilesystemEntityExists(pool, source) && zfsFilesystemEntityExists(pool, dest) {
- return nil
- }
-
- time.Sleep(500 * time.Millisecond)
- }
-
- // Timeout
- logger.Errorf("zfs rename failed: %s", output)
- return fmt.Errorf("Failed to rename ZFS filesystem: %s", output)
-}
-
-func zfsPoolVolumeSet(pool string, path string, key string, value string) error {
- vdev := pool
- if path != "" {
- vdev = fmt.Sprintf("%s/%s", pool, path)
- }
- output, err := shared.RunCommand(
- "zfs",
- "set",
- fmt.Sprintf("%s=%s", key, value),
- vdev)
- if err != nil {
- logger.Errorf("zfs set failed: %s", output)
- return fmt.Errorf("Failed to set ZFS config: %s", output)
- }
-
- return nil
-}
-
-func zfsPoolVolumeSnapshotCreate(pool string, path string, name string) error {
- output, err := shared.RunCommand(
- "zfs",
- "snapshot",
- "-r",
- fmt.Sprintf("%s/%s@%s", pool, path, name))
- if err != nil {
- logger.Errorf("zfs snapshot failed: %s", output)
- return fmt.Errorf("Failed to create ZFS snapshot: %s", output)
- }
-
- return nil
-}
-
-func zfsPoolVolumeSnapshotDestroy(pool, path string, name string) error {
- output, err := shared.RunCommand(
- "zfs",
- "destroy",
- "-r",
- fmt.Sprintf("%s/%s@%s", pool, path, name))
- if err != nil {
- logger.Errorf("zfs destroy failed: %s", output)
- return fmt.Errorf("Failed to destroy ZFS snapshot: %s", output)
- }
-
- return nil
-}
-
-func zfsPoolVolumeSnapshotRestore(pool string, path string, name string) error {
- output, err := shared.TryRunCommand(
- "zfs",
- "rollback",
- fmt.Sprintf("%s/%s@%s", pool, path, name))
- if err != nil {
- logger.Errorf("zfs rollback failed: %s", output)
- return fmt.Errorf("Failed to restore ZFS snapshot: %s", output)
- }
-
- subvols, err := zfsPoolListSubvolumes(pool, fmt.Sprintf("%s/%s", pool, path))
- if err != nil {
- return err
- }
-
- for _, sub := range subvols {
- snaps, err := zfsPoolListSnapshots(pool, sub)
- if err != nil {
- return err
- }
-
- if !shared.StringInSlice(name, snaps) {
- continue
- }
-
- output, err := shared.TryRunCommand(
- "zfs",
- "rollback",
- fmt.Sprintf("%s/%s@%s", pool, sub, name))
- if err != nil {
- logger.Errorf("zfs rollback failed: %s", output)
- return fmt.Errorf("Failed to restore ZFS sub-volume snapshot: %s", output)
- }
- }
-
- return nil
-}
-
-func zfsPoolVolumeSnapshotRename(pool string, path string, oldName string, newName string) error {
- output, err := shared.RunCommand(
- "zfs",
- "rename",
- "-r",
- fmt.Sprintf("%s/%s@%s", pool, path, oldName),
- fmt.Sprintf("%s/%s@%s", pool, path, newName))
- if err != nil {
- logger.Errorf("zfs snapshot rename failed: %s", output)
- return fmt.Errorf("Failed to rename ZFS snapshot: %s", output)
- }
-
- return nil
-}
-
-func zfsMount(poolName string, path string) error {
- output, err := shared.TryRunCommand(
- "zfs",
- "mount",
- fmt.Sprintf("%s/%s", poolName, path))
- if err != nil {
- return fmt.Errorf("Failed to mount ZFS filesystem: %s", output)
- }
-
- return nil
-}
-
-func zfsUmount(poolName string, path string, mountpoint string) error {
- output, err := shared.TryRunCommand(
- "zfs",
- "unmount",
- fmt.Sprintf("%s/%s", poolName, path))
- if err != nil {
- logger.Warnf("Failed to unmount ZFS filesystem via zfs unmount: %s. Trying lazy umount (MNT_DETACH)...", output)
- err := tryUnmount(mountpoint, syscall.MNT_DETACH)
- if err != nil {
- logger.Warnf("Failed to unmount ZFS filesystem via lazy umount (MNT_DETACH)...")
- return err
- }
- }
-
- return nil
-}
-
-func zfsPoolListSubvolumes(pool string, path string) ([]string, error) {
- output, err := shared.RunCommand(
- "zfs",
- "list",
- "-t", "filesystem",
- "-o", "name",
- "-H",
- "-r", path)
- if err != nil {
- logger.Errorf("zfs list failed: %s", output)
- return []string{}, fmt.Errorf("Failed to list ZFS filesystems: %s", output)
- }
-
- children := []string{}
- for _, entry := range strings.Split(output, "\n") {
- if entry == "" {
- continue
- }
-
- if entry == path {
- continue
- }
-
- children = append(children, strings.TrimPrefix(entry, fmt.Sprintf("%s/", pool)))
- }
-
- return children, nil
-}
-
-func zfsPoolListSnapshots(pool string, path string) ([]string, error) {
- path = strings.TrimRight(path, "/")
- fullPath := pool
- if path != "" {
- fullPath = fmt.Sprintf("%s/%s", pool, path)
- }
-
- output, err := shared.RunCommand(
- "zfs",
- "list",
- "-t", "snapshot",
- "-o", "name",
- "-H",
- "-d", "1",
- "-s", "creation",
- "-r", fullPath)
- if err != nil {
- logger.Errorf("zfs list failed: %s", output)
- return []string{}, fmt.Errorf("Failed to list ZFS snapshots: %s", output)
- }
-
- children := []string{}
- for _, entry := range strings.Split(output, "\n") {
- if entry == "" {
- continue
- }
-
- if entry == fullPath {
- continue
- }
-
- children = append(children, strings.SplitN(entry, "@", 2)[1])
- }
-
- return children, nil
-}
-
-func zfsPoolVolumeSnapshotRemovable(pool string, path string, name string) (bool, error) {
- var snap string
- if name == "" {
- snap = path
- } else {
- snap = fmt.Sprintf("%s@%s", path, name)
- }
-
- clones, err := zfsFilesystemEntityPropertyGet(pool, snap, "clones")
- if err != nil {
- return false, err
- }
-
- if clones == "-" || clones == "" {
- return true, nil
- }
-
- return false, nil
-}
-
-func zfsFilesystemEntityExists(pool string, path string) bool {
- vdev := pool
- if path != "" {
- vdev = fmt.Sprintf("%s/%s", pool, path)
- }
- output, err := shared.RunCommand(
- "zfs",
- "get",
- "-H",
- "-o",
- "name",
- "type",
- vdev)
- if err != nil {
- return false
- }
-
- detectedName := strings.TrimSpace(output)
- return detectedName == vdev
-}
-
-func (s *storageZfs) doContainerMount(project, name string, privileged bool) (bool, error) {
- logger.Debugf("Mounting ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- volumeName := projectPrefix(project, name)
- fs := fmt.Sprintf("containers/%s", volumeName)
- containerPoolVolumeMntPoint := getContainerMountPoint(project, s.pool.Name, name)
-
- containerMountLockID := getContainerMountLockID(s.pool.Name, name)
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[containerMountLockID]; ok {
- lxdStorageMapLock.Unlock()
- if _, ok := <-waitChannel; ok {
- logger.Warnf("Received value over semaphore, this should not have happened")
- }
- // Give the benefit of the doubt and assume that the other
- // thread actually succeeded in mounting the storage volume.
- return false, nil
- }
-
- lxdStorageOngoingOperationMap[containerMountLockID] = make(chan bool)
- lxdStorageMapLock.Unlock()
-
- removeLockFromMap := func() {
- lxdStorageMapLock.Lock()
- if waitChannel, ok := lxdStorageOngoingOperationMap[containerMountLockID]; ok {
- close(waitChannel)
- delete(lxdStorageOngoingOperationMap, containerMountLockID)
- }
- lxdStorageMapLock.Unlock()
- }
-
- defer removeLockFromMap()
-
- // Since we're using mount() directly zfs will not automatically create
- // the mountpoint for us. So let's check and do it if needed.
- if !shared.PathExists(containerPoolVolumeMntPoint) {
- err := createContainerMountpoint(containerPoolVolumeMntPoint, shared.VarPath(fs), privileged)
- if err != nil {
- return false, err
- }
- }
-
- ourMount := false
- if !shared.IsMountPoint(containerPoolVolumeMntPoint) {
- source := fmt.Sprintf("%s/%s", s.getOnDiskPoolName(), fs)
- zfsMountOptions := fmt.Sprintf("rw,zfsutil,mntpoint=%s", containerPoolVolumeMntPoint)
- mounterr := tryMount(source, containerPoolVolumeMntPoint, "zfs", 0, zfsMountOptions)
- if mounterr != nil {
- if mounterr != syscall.EBUSY {
- logger.Errorf("Failed to mount ZFS dataset \"%s\" onto \"%s\": %v", source, containerPoolVolumeMntPoint, mounterr)
- return false, mounterr
- }
- // EBUSY error in zfs are related to a bug we're
- // tracking. So ignore them for now, report back that
- // the mount isn't ours and proceed.
- logger.Warnf("ZFS returned EBUSY while \"%s\" is actually not a mountpoint", containerPoolVolumeMntPoint)
- return false, mounterr
- }
- ourMount = true
- }
-
- logger.Debugf("Mounted ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return ourMount, nil
-}
-
-func (s *storageZfs) doContainerDelete(project, name string) error {
- logger.Debugf("Deleting ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- poolName := s.getOnDiskPoolName()
- containerName := name
- fs := fmt.Sprintf("containers/%s", projectPrefix(project, containerName))
- containerPoolVolumeMntPoint := getContainerMountPoint(project, s.pool.Name, containerName)
-
- if zfsFilesystemEntityExists(poolName, fs) {
- removable := true
- snaps, err := zfsPoolListSnapshots(poolName, fs)
- if err != nil {
- return err
- }
-
- for _, snap := range snaps {
- var err error
- removable, err = zfsPoolVolumeSnapshotRemovable(poolName, fs, snap)
- if err != nil {
- return err
- }
-
- if !removable {
- break
- }
- }
-
- if removable {
- origin, err := zfsFilesystemEntityPropertyGet(poolName, fs, "origin")
- if err != nil {
- return err
- }
- poolName := s.getOnDiskPoolName()
- origin = strings.TrimPrefix(origin, fmt.Sprintf("%s/", poolName))
-
- err = zfsPoolVolumeDestroy(poolName, fs)
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeCleanup(poolName, origin)
- if err != nil {
- return err
- }
- } else {
- err := zfsPoolVolumeSet(poolName, fs, "mountpoint", "none")
- if err != nil {
- return err
- }
-
- err = zfsPoolVolumeRename(poolName, fs, fmt.Sprintf("deleted/containers/%s", uuid.NewRandom().String()), true)
- if err != nil {
- return err
- }
- }
- }
-
- err := deleteContainerMountpoint(containerPoolVolumeMntPoint, shared.VarPath("containers", projectPrefix(project, name)), s.GetStorageTypeName())
- if err != nil {
- return err
- }
-
- snapshotZfsDataset := fmt.Sprintf("snapshots/%s", containerName)
- zfsPoolVolumeDestroy(poolName, snapshotZfsDataset)
-
- // Delete potential leftover snapshot mountpoints.
- snapshotMntPoint := getSnapshotMountPoint(project, s.pool.Name, containerName)
- if shared.PathExists(snapshotMntPoint) {
- err := os.RemoveAll(snapshotMntPoint)
- if err != nil {
- return err
- }
- }
-
- // Delete potential leftover snapshot symlinks:
- // ${LXD_DIR}/snapshots/<container_name> to ${POOL}/snapshots/<container_name>
- snapshotSymlink := shared.VarPath("snapshots", projectPrefix(project, containerName))
- if shared.PathExists(snapshotSymlink) {
- err := os.Remove(snapshotSymlink)
- if err != nil {
- return err
- }
- }
-
- logger.Debugf("Deleted ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func (s *storageZfs) doContainerCreate(project, name string, privileged bool) error {
- logger.Debugf("Creating empty ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
-
- containerPath := shared.VarPath("containers", projectPrefix(project, name))
- containerName := name
- fs := fmt.Sprintf("containers/%s", projectPrefix(project, containerName))
- poolName := s.getOnDiskPoolName()
- dataset := fmt.Sprintf("%s/%s", poolName, fs)
- containerPoolVolumeMntPoint := getContainerMountPoint(project, s.pool.Name, containerName)
-
- // Create volume.
- msg, err := zfsPoolVolumeCreate(dataset, "mountpoint=none", "canmount=noauto")
- if err != nil {
- logger.Errorf("Failed to create ZFS storage volume for container \"%s\" on storage pool \"%s\": %s", s.volume.Name, s.pool.Name, msg)
- return err
- }
-
- // Set mountpoint.
- err = zfsPoolVolumeSet(poolName, fs, "mountpoint", containerPoolVolumeMntPoint)
- if err != nil {
- return err
- }
-
- err = createContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged)
- if err != nil {
- return err
- }
-
- logger.Debugf("Created empty ZFS storage volume for container \"%s\" on storage pool \"%s\"", s.volume.Name, s.pool.Name)
- return nil
-}
-
-func zfsIdmapSetSkipper(dir string, absPath string, fi os.FileInfo) bool {
- strippedPath := absPath
- if dir != "" {
- strippedPath = absPath[len(dir):]
- }
-
- if fi.IsDir() && strippedPath == "/.zfs/snapshot" {
- return true
- }
-
- return false
-}
More information about the lxc-devel
mailing list