[lxc-devel] [lxd/master] lxd/storage: Add rsync.compression to storagePoolConfigKeys
drs-11 on Github
lxc-bot at linuxcontainers.org
Sat Oct 3 20:01:14 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 796 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201003/8dd753f1/attachment.bin>
-------------- next part --------------
From 858f589003cb244856d500b51571b93cca73e240 Mon Sep 17 00:00:00 2001
From: drs-11 <siddharthadr11 at gmail.com>
Date: Sun, 4 Oct 2020 00:15:40 +0530
Subject: [PATCH] lxd/storage: Adds rsync.compression config key to
storagePoolConfigKeys
Signed-off-by: D R Siddhartha <siddharthadr11 at gmail.com>
---
doc/api-extensions.md | 4 ++++
doc/storage.md | 1 +
lxd/storage/drivers/driver_btrfs.go | 10 +++++++++-
lxd/storage/drivers/driver_ceph.go | 10 +++++++++-
lxd/storage/drivers/driver_cephfs.go | 12 +++++++++++-
lxd/storage/drivers/driver_common.go | 11 ++++++++++-
lxd/storage/drivers/driver_zfs.go | 10 +++++++++-
lxd/storage_pools_config.go | 3 +++
shared/version/api.go | 1 +
test/suites/migration.sh | 7 +++++++
10 files changed, 64 insertions(+), 5 deletions(-)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 52be415f36..159d86d419 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1176,3 +1176,7 @@ This includes the following new endpoints (see [RESTful API](rest-api.md) for de
The following existing endpoint has been modified:
* `POST /1.0/storage-pools/<pool>/<type>/<volume>` accepts the new source type `backup`
+
+## storage\_rsync\_compression
+Adds `rsync.compression` config key to storage pools. This key can be used
+to disable compression in rsync while migrating storage pools.
\ No newline at end of file
diff --git a/doc/storage.md b/doc/storage.md
index 6e0dd9f5fa..67dc5786f1 100644
--- a/doc/storage.md
+++ b/doc/storage.md
@@ -27,6 +27,7 @@ lvm.vg.force\_reuse | bool | lvm driver
volume.lvm.stripes | string | lvm driver | - | storage\_lvm\_stripes | Number of stripes to use for new volumes (or thin pool volume).
volume.lvm.stripes.size | string | lvm driver | - | storage\_lvm\_stripes | Size of stripes to use (at least 4096 bytes and multiple of 512bytes).
rsync.bwlimit | string | - | 0 (no limit) | storage\_rsync\_bwlimit | Specifies the upper limit to be placed on the socket I/O whenever rsync has to be used to transfer storage entities.
+rsync.compression | bool | appropriate driver | true | storage\_rsync\_compression | Whether to use compression while migrating storage pools.
volatile.initial\_source | string | - | - | storage\_volatile\_initial\_source | Records the actual source passed during creating (e.g. /dev/sdb).
volatile.pool.pristine | string | - | true | storage\_driver\_ceph | Whether the pool has been empty on creation time.
volume.block.filesystem | string | block based driver (lvm) | ext4 | storage | Filesystem to use for new volumes
diff --git a/lxd/storage/drivers/driver_btrfs.go b/lxd/storage/drivers/driver_btrfs.go
index 988e741277..d50d80366d 100644
--- a/lxd/storage/drivers/driver_btrfs.go
+++ b/lxd/storage/drivers/driver_btrfs.go
@@ -372,9 +372,17 @@ func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
- rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}
+ // Do not pass compression argument to rsync if the assoicated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
+ }
+
// Only offer rsync for refreshes or if running in an unprivileged container.
if refresh || d.state.OS.RunningInUserNS {
var transportType migration.MigrationFSType
diff --git a/lxd/storage/drivers/driver_ceph.go b/lxd/storage/drivers/driver_ceph.go
index f13f1a3c0e..17b16339c5 100644
--- a/lxd/storage/drivers/driver_ceph.go
+++ b/lxd/storage/drivers/driver_ceph.go
@@ -340,7 +340,15 @@ func (d *ceph) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *ceph) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
- rsyncFeatures := []string{"delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the assoicated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"delete", "compress", "bidirectional"}
+ }
if refresh {
var transportType migration.MigrationFSType
diff --git a/lxd/storage/drivers/driver_cephfs.go b/lxd/storage/drivers/driver_cephfs.go
index d275212c45..fe70d1698b 100644
--- a/lxd/storage/drivers/driver_cephfs.go
+++ b/lxd/storage/drivers/driver_cephfs.go
@@ -312,6 +312,16 @@ func (d *cephfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationTypes returns the supported migration types and options supported by the driver.
func (d *cephfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the assoicated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"delete", "compress", "bidirectional"}
+ }
+
if contentType != ContentTypeFS {
return nil
}
@@ -320,7 +330,7 @@ func (d *cephfs) MigrationTypes(contentType ContentType, refresh bool) []migrati
return []migration.Type{
{
FSType: migration.MigrationFSType_RSYNC,
- Features: []string{"delete", "compress", "bidirectional"},
+ Features: rsyncFeatures,
},
}
}
diff --git a/lxd/storage/drivers/driver_common.go b/lxd/storage/drivers/driver_common.go
index 843f5652f0..10c880f8bd 100644
--- a/lxd/storage/drivers/driver_common.go
+++ b/lxd/storage/drivers/driver_common.go
@@ -132,6 +132,15 @@ func (d *common) validateVolume(vol Volume, driverRules map[string]func(value st
// in preference order.
func (d *common) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
var transportType migration.MigrationFSType
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the assoicated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
+ }
if contentType == ContentTypeBlock {
transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
@@ -142,7 +151,7 @@ func (d *common) MigrationTypes(contentType ContentType, refresh bool) []migrati
return []migration.Type{
{
FSType: transportType,
- Features: []string{"xattrs", "delete", "compress", "bidirectional"},
+ Features: rsyncFeatures,
},
}
}
diff --git a/lxd/storage/drivers/driver_zfs.go b/lxd/storage/drivers/driver_zfs.go
index 387a351f5f..1ddc78facc 100644
--- a/lxd/storage/drivers/driver_zfs.go
+++ b/lxd/storage/drivers/driver_zfs.go
@@ -452,7 +452,15 @@ func (d *zfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *zfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
- rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the assoicated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
+ }
// When performing a refresh, always use rsync. Using zfs send/receive
// here doesn't make sense since it would need to send everything again
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index cfcdf0b234..5c6892bd12 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -81,6 +81,9 @@ var storagePoolConfigKeys = map[string]func(value string) error{
"zfs.clone_copy": validate.Optional(validate.IsBool),
"zfs.pool_name": validate.IsAny,
"rsync.bwlimit": validate.IsAny,
+
+ // valid drivers: btrfs, ceph, cephfs, zfs
+ "rsync.compression": validate.Optional(validate.IsBool),
}
func storagePoolValidateConfig(name string, driver string, config map[string]string, oldConfig map[string]string) error {
diff --git a/shared/version/api.go b/shared/version/api.go
index dfe5156e54..f893568048 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -227,6 +227,7 @@ var APIExtensions = []string{
"projects_networks",
"projects_networks_restricted_uplinks",
"custom_volume_backup",
+ "storage_rsync_compress",
}
// APIExtensionsCount returns the number of available API extensions.
diff --git a/test/suites/migration.sh b/test/suites/migration.sh
index 6b3896f117..f2416eed6b 100644
--- a/test/suites/migration.sh
+++ b/test/suites/migration.sh
@@ -329,6 +329,13 @@ migration() {
lxc_remote storage volume delete l2:"$remote_pool2" vol5
lxc_remote storage volume delete l2:"$remote_pool2" vol6
+ # Test migration when rsync compression is disabled
+ lxc_remote storage set l1:"$remote_pool1" rsync.compression false
+ lxc_remote storage volume create l1:"$remote_pool1" foo
+ lxc_remote storage volume copy l1:"$remote_pool1"/foo l2:"$remote_pool2"/bar
+ lxc_remote storage volume delete l1:"$remote_pool1" foo
+ lxc_remote storage volume delete l2:"$remote_pool2" bar
+
# Test some migration between projects
lxc_remote project create l1:proj -c features.images=false -c features.profiles=false
lxc_remote project switch l1:proj
More information about the lxc-devel
mailing list