[lxc-devel] [lxd/master] Adds config key for disabling rsync compression
drs-11 on Github
lxc-bot at linuxcontainers.org
Sun Oct 4 16:27:20 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 407 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201004/6250e151/attachment.bin>
-------------- next part --------------
From 34097139ae42fc62082adb6ed8e7b2415b3a0b30 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthadr11 at gmail.com>
Date: Sun, 4 Oct 2020 21:40:53 +0530
Subject: [PATCH 1/4] lxd/storage: Adds rsync.compression config key
Signed-off-by: D R Siddhartha <siddharthadr11 at gmail.com>
---
lxd/storage/drivers/driver_btrfs.go | 10 +++++++++-
lxd/storage/drivers/driver_ceph.go | 10 +++++++++-
lxd/storage/drivers/driver_cephfs.go | 12 +++++++++++-
lxd/storage/drivers/driver_common.go | 11 ++++++++++-
lxd/storage/drivers/driver_zfs.go | 10 +++++++++-
lxd/storage/utils.go | 1 +
lxd/storage_pools_config.go | 3 +++
7 files changed, 52 insertions(+), 5 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs.go b/lxd/storage/drivers/driver_btrfs.go
index 988e741277..c98b8409de 100644
--- a/lxd/storage/drivers/driver_btrfs.go
+++ b/lxd/storage/drivers/driver_btrfs.go
@@ -372,9 +372,17 @@ func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
- rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
+ }
+
// Only offer rsync for refreshes or if running in an unprivileged container.
if refresh || d.state.OS.RunningInUserNS {
var transportType migration.MigrationFSType
diff --git a/lxd/storage/drivers/driver_ceph.go b/lxd/storage/drivers/driver_ceph.go
index f13f1a3c0e..2500bb94ac 100644
--- a/lxd/storage/drivers/driver_ceph.go
+++ b/lxd/storage/drivers/driver_ceph.go
@@ -340,7 +340,15 @@ func (d *ceph) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *ceph) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
- rsyncFeatures := []string{"delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"delete", "compress", "bidirectional"}
+ }
if refresh {
var transportType migration.MigrationFSType
diff --git a/lxd/storage/drivers/driver_cephfs.go b/lxd/storage/drivers/driver_cephfs.go
index d275212c45..015da2c949 100644
--- a/lxd/storage/drivers/driver_cephfs.go
+++ b/lxd/storage/drivers/driver_cephfs.go
@@ -312,6 +312,16 @@ func (d *cephfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationTypes returns the supported migration types and options supported by the driver.
func (d *cephfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"delete", "compress", "bidirectional"}
+ }
+
if contentType != ContentTypeFS {
return nil
}
@@ -320,7 +330,7 @@ func (d *cephfs) MigrationTypes(contentType ContentType, refresh bool) []migrati
return []migration.Type{
{
FSType: migration.MigrationFSType_RSYNC,
- Features: []string{"delete", "compress", "bidirectional"},
+ Features: rsyncFeatures,
},
}
}
diff --git a/lxd/storage/drivers/driver_common.go b/lxd/storage/drivers/driver_common.go
index 843f5652f0..e15950acad 100644
--- a/lxd/storage/drivers/driver_common.go
+++ b/lxd/storage/drivers/driver_common.go
@@ -132,6 +132,15 @@ func (d *common) validateVolume(vol Volume, driverRules map[string]func(value st
// in preference order.
func (d *common) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
var transportType migration.MigrationFSType
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
+ }
if contentType == ContentTypeBlock {
transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
@@ -142,7 +151,7 @@ func (d *common) MigrationTypes(contentType ContentType, refresh bool) []migrati
return []migration.Type{
{
FSType: transportType,
- Features: []string{"xattrs", "delete", "compress", "bidirectional"},
+ Features: rsyncFeatures,
},
}
}
diff --git a/lxd/storage/drivers/driver_zfs.go b/lxd/storage/drivers/driver_zfs.go
index 387a351f5f..5e290af54f 100644
--- a/lxd/storage/drivers/driver_zfs.go
+++ b/lxd/storage/drivers/driver_zfs.go
@@ -452,7 +452,15 @@ func (d *zfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *zfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
- rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
+ }
// When performing a refresh, always use rsync. Using zfs send/receive
// here doesn't make sense since it would need to send everything again
diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go
index 93201ad46e..6f4c9e8c0b 100644
--- a/lxd/storage/utils.go
+++ b/lxd/storage/utils.go
@@ -424,6 +424,7 @@ func validatePoolCommonRules() map[string]func(string) error {
"volume.size": validate.Optional(validate.IsSize),
"size": validate.Optional(validate.IsSize),
"rsync.bwlimit": validate.IsAny,
+ "rsync.compression": validate.Optional(validate.IsBool),
}
}
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index cfcdf0b234..5c6892bd12 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -81,6 +81,9 @@ var storagePoolConfigKeys = map[string]func(value string) error{
"zfs.clone_copy": validate.Optional(validate.IsBool),
"zfs.pool_name": validate.IsAny,
"rsync.bwlimit": validate.IsAny,
+
+ // valid drivers: btrfs, ceph, cephfs, zfs
+ "rsync.compression": validate.Optional(validate.IsBool),
}
func storagePoolValidateConfig(name string, driver string, config map[string]string, oldConfig map[string]string) error {
From f371c6c931afca0a033a345c9116efc91fa8e2ae Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthadr11 at gmail.com>
Date: Sun, 4 Oct 2020 21:44:06 +0530
Subject: [PATCH 2/4] doc: Adds rsync.compression
Signed-off-by: D R Siddhartha <siddharthadr11 at gmail.com>
---
doc/storage.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/storage.md b/doc/storage.md
index 6e0dd9f5fa..67dc5786f1 100644
--- a/doc/storage.md
+++ b/doc/storage.md
@@ -27,6 +27,7 @@ lvm.vg.force\_reuse | bool | lvm driver
volume.lvm.stripes | string | lvm driver | - | storage\_lvm\_stripes | Number of stripes to use for new volumes (or thin pool volume).
volume.lvm.stripes.size | string | lvm driver | - | storage\_lvm\_stripes | Size of stripes to use (at least 4096 bytes and multiple of 512bytes).
rsync.bwlimit | string | - | 0 (no limit) | storage\_rsync\_bwlimit | Specifies the upper limit to be placed on the socket I/O whenever rsync has to be used to transfer storage entities.
+rsync.compression | bool | appropriate driver | true | storage\_rsync\_compression | Whether to use compression while migrating storage pools.
volatile.initial\_source | string | - | - | storage\_volatile\_initial\_source | Records the actual source passed during creating (e.g. /dev/sdb).
volatile.pool.pristine | string | - | true | storage\_driver\_ceph | Whether the pool has been empty on creation time.
volume.block.filesystem | string | block based driver (lvm) | ext4 | storage | Filesystem to use for new volumes
From f9b21d257a4e8deb79b36d1540d151b4d14bc242 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthadr11 at gmail.com>
Date: Sun, 4 Oct 2020 21:46:45 +0530
Subject: [PATCH 3/4] api: storage_rsync_compression
Signed-off-by: D R Siddhartha <siddharthadr11 at gmail.com>
---
doc/api-extensions.md | 4 ++++
shared/version/api.go | 1 +
2 files changed, 5 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index a0cd6ec30c..d7c233c3e6 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1182,3 +1182,7 @@ Adds `Name` field to `InstanceBackupArgs` to allow specifying a different instan
Adds `Name` and `PoolName` fields to `StoragePoolVolumeBackupArgs` to allow specifying a different volume name
when restoring a custom volume backup.
+
+## storage\_rsync\_compression
+Adds `rsync.compression` config key to storage pools. This key can be used
+to disable compression in rsync while migrating storage pools.
\ No newline at end of file
diff --git a/shared/version/api.go b/shared/version/api.go
index 41a64fac67..ba9380b676 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -228,6 +228,7 @@ var APIExtensions = []string{
"projects_networks_restricted_uplinks",
"custom_volume_backup",
"backup_override_name",
+ "storage_rsync_compression",
}
// APIExtensionsCount returns the number of available API extensions.
From 1d0da811313c7c0e1ab3bd6449ea9ad591ad34d0 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthadr11 at gmail.com>
Date: Sun, 4 Oct 2020 21:48:35 +0530
Subject: [PATCH 4/4] tests: Valid rsync.compression
Signed-off-by: D R Siddhartha <siddharthadr11 at gmail.com>
---
test/suites/migration.sh | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/test/suites/migration.sh b/test/suites/migration.sh
index 6b3896f117..f2416eed6b 100644
--- a/test/suites/migration.sh
+++ b/test/suites/migration.sh
@@ -329,6 +329,13 @@ migration() {
lxc_remote storage volume delete l2:"$remote_pool2" vol5
lxc_remote storage volume delete l2:"$remote_pool2" vol6
+ # Test migration when rsync compression is disabled
+ lxc_remote storage set l1:"$remote_pool1" rsync.compression false
+ lxc_remote storage volume create l1:"$remote_pool1" foo
+ lxc_remote storage volume copy l1:"$remote_pool1"/foo l2:"$remote_pool2"/bar
+ lxc_remote storage volume delete l1:"$remote_pool1" foo
+ lxc_remote storage volume delete l2:"$remote_pool2" bar
+
# Test some migration between projects
lxc_remote project create l1:proj -c features.images=false -c features.profiles=false
lxc_remote project switch l1:proj
More information about the lxc-devel
mailing list