[lxc-devel] [lxd/master] feature: lvm-striping support
rcash on Github
lxc-bot at linuxcontainers.org
Thu Dec 12 18:48:55 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 366 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191212/bbe2c86f/attachment-0001.bin>
-------------- next part --------------
From 1865be9f41da45ff90f163a168d2562035532bcd Mon Sep 17 00:00:00 2001
From: Layton Seal <laytonseal at gmail.com>
Date: Mon, 9 Dec 2019 16:11:26 -0600
Subject: [PATCH] feature: lvm-striping support
Signed-off-by: Layton Seal <laytonseal at gmail.com>
Signed-off-by: Rowdy Larson <rowdycash at gmail.com>
---
lxd/patches.go | 15 ++++
lxd/storage/utils.go | 39 +++++++++
lxd/storage_lvm.go | 51 +++++++++--
lxd/storage_lvm_utils.go | 156 +++++++++++++++++++++++++++++-----
lxd/storage_pools_config.go | 19 ++++-
lxd/storage_volumes_config.go | 2 +
scripts/bash/lxd-client | 6 +-
7 files changed, 257 insertions(+), 31 deletions(-)
diff --git a/lxd/patches.go b/lxd/patches.go
index 99923e43cf..1b0b9db08c 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -2212,6 +2212,15 @@ func patchStorageApiUpdateStorageConfigs(name string, d *Daemon) error {
if pool.Config["volume.block.filesystem"] == "ext4" {
pool.Config["volume.block.filesystem"] = ""
}
+
+ if pool.Config["volume.lvm.stripes"] != "" {
+ pool.Config["volume.lvm.stripes"] = ""
+ }
+
+ if pool.Config["volume.lvm.stripes.size"] != "" {
+ pool.Config["volume.lvm.stripes.size"] = ""
+ }
+
case "zfs":
// Unset default values.
if !shared.IsTrue(pool.Config["volume.zfs.use_refquota"]) {
@@ -2272,6 +2281,12 @@ func patchStorageApiUpdateStorageConfigs(name string, d *Daemon) error {
if volume.Config["block.mount_options"] == "discard" {
volume.Config["block.mount_options"] = ""
}
+ if volume.Config["lvm.stripes"] != "" {
+ volume.Config["lvm.stripes"] = ""
+ }
+ if volume.Config["lvm.stripes.size"] != "" {
+ volume.Config["lvm.stripes.size"] = ""
+ }
case "zfs":
// Unset default values.
if !shared.IsTrue(volume.Config["zfs.use_refquota"]) {
diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go
index db4eb0f1ff..4645686542 100644
--- a/lxd/storage/utils.go
+++ b/lxd/storage/utils.go
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
+ "strconv"
"strings"
"time"
@@ -486,6 +487,21 @@ var StorageVolumeConfigKeys = map[string]func(value string) ([]string, error){
"block.mount_options": func(value string) ([]string, error) {
return []string{"ceph", "lvm"}, shared.IsAny(value)
},
+ "lvm.stripes": func(value string) ([]string, error) {
+ return SupportedPoolTypes, shared.IsUint32(value)
+ },
+ "lvm.stripes.size": func(value string) ([]string, error){
+ if value == "" {
+ return SupportedPoolTypes, nil
+ }
+
+ _, err := units.ParseByteSizeString(value)
+ if err != nil {
+ return nil, err
+ }
+
+ return SupportedPoolTypes, nil
+ },
"security.shifted": func(value string) ([]string, error) {
return SupportedPoolTypes, shared.IsBool(value)
},
@@ -556,6 +572,16 @@ func VolumeValidateConfig(name string, config map[string]string, parentPool *api
return err
}
+ if parentPool.Driver != "lvm" {
+ if config["lvm.stripes"] != "" {
+ return fmt.Errorf("the key lvm.stripes cannot be used with non lvm storage volumes")
+ }
+
+ if config["lvm.stripes.size"] != "" {
+ return fmt.Errorf("the key lvm.stripes.size cannot be used with non lvm storage volumes")
+ }
+ }
+
if parentPool.Driver != "zfs" || parentPool.Driver == "dir" {
if config["zfs.use_refquota"] != "" {
return fmt.Errorf("the key volume.zfs.use_refquota cannot be used with non zfs storage volumes")
@@ -599,6 +625,19 @@ func VolumeFillDefault(name string, config map[string]string, parentPool *api.St
config["block.mount_options"] = "discard"
}
+ if parentPool.Driver == "lvm" {
+ if parentPool.Config["volume.lvm.stripes"] != "1" {
+ config["lvm.stripes"] = parentPool.Config["volume.lvm.stripes"]
+ } else {
+ config["lvm.stripes"] = strconv.FormatUint(uint64(1), 10)
+ }
+ if parentPool.Config["volume.lvm.stripes.size"] != "" {
+ config["lvm.stripes.size"] = parentPool.Config["volume.lvm.stripes.size"]
+ } else {
+ config["lvm.stripes"] = ""
+ }
+ }
+
// Does the pool request a default size for new storage volumes?
if config["size"] == "0" || config["size"] == "" {
config["size"] = parentPool.Config["volume.size"]
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index 51b46e935b..d21fbd6711 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -31,6 +31,8 @@ type storageLvm struct {
vgName string
thinPoolName string
useThinpool bool
+ stripes uint64
+ stripesSize string
loopInfo *os.File
storageShared
}
@@ -333,6 +335,12 @@ func (s *storageLvm) StoragePoolCreate() error {
return err
}
+ s.stripes, err = s.getNumberOfStripes()
+ if err != nil {
+ return err
+ }
+
+ s.stripesSize = s.getSizeOfStripes()
// Deregister cleanup.
tryUndo = false
@@ -506,14 +514,20 @@ func (s *storageLvm) StoragePoolVolumeCreate() error {
return err
}
+ stripes, err := s.getNumberOfStripes()
+ if err != nil {
+ return err
+ }
+ stripesSize := s.getSizeOfStripes()
+
if s.useThinpool {
- err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType)
+ err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType, stripes, stripesSize)
if err != nil {
return err
}
}
- err = lvmCreateLv("default", poolName, thinPoolName, volumeLvmName, lvFsType, lvSize, volumeType, s.useThinpool)
+ err = lvmCreateLv("default", poolName, thinPoolName, volumeLvmName, lvFsType, lvSize, volumeType, s.useThinpool, stripes, stripesSize)
if err != nil {
return fmt.Errorf("Error Creating LVM LV for new image: %v", err)
}
@@ -948,14 +962,21 @@ func (s *storageLvm) ContainerCreate(container instance.Instance) error {
}
poolName := s.getOnDiskPoolName()
+
+ stripes, err := s.getNumberOfStripes()
+ if err != nil {
+ return err
+ }
+ stripesSize := s.getSizeOfStripes()
+
if s.useThinpool {
- err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType)
+ err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType, stripes, stripesSize)
if err != nil {
return err
}
}
- err = lvmCreateLv(container.Project(), poolName, thinPoolName, containerLvmName, lvFsType, lvSize, storagePoolVolumeAPIEndpointContainers, s.useThinpool)
+ err = lvmCreateLv(container.Project(), poolName, thinPoolName, containerLvmName, lvFsType, lvSize, storagePoolVolumeAPIEndpointContainers, s.useThinpool, stripes, stripesSize)
if err != nil {
return err
}
@@ -1831,8 +1852,15 @@ func (s *storageLvm) doContainerBackupLoad(projectName, containerName string, pr
}
poolName := s.getOnDiskPoolName()
+
+ stripes, err := s.getNumberOfStripes()
+ if err != nil {
+ return "", err
+ }
+ stripesSize := s.getSizeOfStripes()
+
if s.useThinpool {
- err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType)
+ err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType, stripes, stripesSize)
if err != nil {
return "", err
}
@@ -1840,7 +1868,7 @@ func (s *storageLvm) doContainerBackupLoad(projectName, containerName string, pr
if !snapshot {
err = lvmCreateLv(projectName, poolName, thinPoolName, containerLvmName, lvFsType, lvSize,
- storagePoolVolumeAPIEndpointContainers, s.useThinpool)
+ storagePoolVolumeAPIEndpointContainers, s.useThinpool, stripes, stripesSize)
} else {
cname, _, _ := shared.InstanceGetParentAndSnapshotName(containerName)
_, err = s.createSnapshotLV(projectName, poolName, containerNameToLVName(cname), storagePoolVolumeAPIEndpointContainers,
@@ -1920,12 +1948,19 @@ func (s *storageLvm) ImageCreate(fingerprint string, tracker *ioprogress.Progres
}()
if s.useThinpool {
- err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType)
+ stripes, err := s.getNumberOfStripes()
+ if err != nil {
+ return err
+ }
+ stripesSize := s.getSizeOfStripes()
+
+ err = lvmCreateThinpool(s.s, s.sTypeVersion, poolName, thinPoolName, lvFsType, stripes, stripesSize)
if err != nil {
return err
}
- err = lvmCreateLv("default", poolName, thinPoolName, fingerprint, lvFsType, lvSize, storagePoolVolumeAPIEndpointImages, true)
+ err = lvmCreateLv("default", poolName, thinPoolName, fingerprint, lvFsType, lvSize,
+ storagePoolVolumeAPIEndpointImages, true, stripes, stripesSize)
if err != nil {
return fmt.Errorf("Error Creating LVM LV for new image: %v", err)
}
diff --git a/lxd/storage_lvm_utils.go b/lxd/storage_lvm_utils.go
index cec7c223f9..8ad5559c21 100644
--- a/lxd/storage_lvm_utils.go
+++ b/lxd/storage_lvm_utils.go
@@ -145,6 +145,24 @@ func (s *storageLvm) getLvmThinpoolName() string {
return "LXDThinPool"
}
+func (s *storageLvm) getNumberOfStripes() (uint64, error) {
+ stripesString := s.pool.Config["volume.lvm.stripes"]
+
+ if stripesString != "" {
+ stripes, err := strconv.ParseUint(stripesString, 10, 32)
+
+ if err != nil {
+ return 0, err
+ }
+ return stripes, err
+ }
+ return 1, nil
+}
+
+func (s *storageLvm) getSizeOfStripes() (string) {
+ return s.pool.Config["volume.lvm.stripes.size"]
+}
+
func (s *storageLvm) usesThinpool() bool {
// Default is to use a thinpool.
if s.pool.Config["lvm.use_thinpool"] == "" {
@@ -829,7 +847,7 @@ func getPrefixedLvName(projectName, volumeType string, lvmVolume string) string
return fmt.Sprintf("%s_%s", volumeType, lvmVolume)
}
-func lvmCreateLv(projectName, vgName string, thinPoolName string, lvName string, lvFsType string, lvSize string, volumeType string, makeThinLv bool) error {
+func lvmCreateLv(projectName, vgName string, thinPoolName string, lvName string, lvFsType string, lvSize string, volumeType string, makeThinLv bool, numStripes uint64, stripeSize string) error {
var output string
var err error
@@ -842,15 +860,37 @@ func lvmCreateLv(projectName, vgName string, thinPoolName string, lvName string,
lvSizeInt = int64(lvSizeInt/512) * 512
lvSizeString := units.GetByteSizeString(lvSizeInt, 0)
+ numStripesString := fmt.Sprintf("%d", numStripes)
+ stripeSizeUint, err := units.ParseByteSizeString(stripeSize)
+
+ if err != nil {
+ return err
+ }
+
+ stripeSize = units.GetByteSizeString(stripeSizeUint, 0)
+ stripesSizeString, err := getLVCreateSize(stripeSize)
+ if err != nil {
+ return err
+ }
+
lvmPoolVolumeName := getPrefixedLvName(projectName, volumeType, lvName)
+ //numStripes and stripesSize should only be used if makeThinLv is false. If it is true, thin volumes will inherit
+ //striping properites from their parent pool's properites
if makeThinLv {
targetVg := fmt.Sprintf("%s/%s", vgName, thinPoolName)
- _, err = shared.TryRunCommand("lvcreate", "-Wy", "--yes", "--thin", "-n", lvmPoolVolumeName, "--virtualsize", lvSizeString, targetVg)
+ if numStripes > 1 {
+ _, err = shared.TryRunCommand("lvcreate", "-i", numStripesString, "-I", stripesSizeString, "-Wy", "--yes", "--thin", "-n", lvmPoolVolumeName, "--virtualsize", lvSizeString, targetVg)
+ } else {
+ _, err = shared.TryRunCommand("lvcreate", "-Wy", "--yes", "--thin", "-n", lvmPoolVolumeName, "--virtualsize", lvSizeString, targetVg)
+ }
} else {
- _, err = shared.TryRunCommand("lvcreate", "-Wy", "--yes", "-n", lvmPoolVolumeName, "--size", lvSizeString, vgName)
+ if numStripes > 1 {
+ _, err = shared.TryRunCommand("lvcreate", "-i", numStripesString, "-I", stripesSizeString, "-Wy", "--yes", "-n", lvmPoolVolumeName, "--size", lvSizeString, vgName)
+ } else {
+ _, err = shared.TryRunCommand("lvcreate", "-Wy", "--yes", "-n", lvmPoolVolumeName, "--size", lvSizeString, vgName)
+ }
}
if err != nil {
- logger.Errorf("Could not create LV \"%s\": %v", lvmPoolVolumeName, err)
return fmt.Errorf("Could not create thin LV named %s: %v", lvmPoolVolumeName, err)
}
@@ -858,14 +898,13 @@ func lvmCreateLv(projectName, vgName string, thinPoolName string, lvName string,
output, err = driver.MakeFSType(fsPath, lvFsType, nil)
if err != nil {
- logger.Errorf("Filesystem creation failed: %v (%s)", err, output)
return fmt.Errorf("Error making filesystem on image LV: %v (%s)", err, output)
}
return nil
}
-func lvmCreateThinpool(s *state.State, sTypeVersion string, vgName string, thinPoolName string, lvFsType string) error {
+func lvmCreateThinpool(s *state.State, sTypeVersion string, vgName string, thinPoolName string, lvFsType string, numStripes uint64, stripesSize string) error {
exists, err := storageLVMThinpoolExists(vgName, thinPoolName)
if err != nil {
return err
@@ -875,7 +914,19 @@ func lvmCreateThinpool(s *state.State, sTypeVersion string, vgName string, thinP
return nil
}
- err = createDefaultThinPool(sTypeVersion, vgName, thinPoolName, lvFsType)
+ stripesSizeUint, err := units.ParseByteSizeString(stripesSize)
+ if err != nil {
+ return err
+ }
+ stripesSize = units.GetByteSizeString(stripesSizeUint, 0)
+ stripesSizeLVString, err := getLVCreateSize(stripesSize)
+ if err != nil {
+ return nil
+ }
+
+ numStripesString := fmt.Sprintf("%d", numStripes)
+
+ err = createDefaultThinPool(sTypeVersion, vgName, thinPoolName, lvFsType, numStripesString, stripesSizeLVString)
if err != nil {
return err
}
@@ -889,28 +940,53 @@ func lvmCreateThinpool(s *state.State, sTypeVersion string, vgName string, thinP
return nil
}
-func createDefaultThinPool(sTypeVersion string, vgName string, thinPoolName string, lvFsType string) error {
+func createDefaultThinPool(sTypeVersion string, vgName string, thinPoolName string, lvFsType string, numStripes string, stripesSize string) error {
isRecent, err := lvmVersionIsAtLeast(sTypeVersion, "2.02.99")
if err != nil {
return fmt.Errorf("Error checking LVM version: %s", err)
}
+ numStripesUint, err := strconv.ParseUint(numStripes, 10, 0)
+ if err != nil {
+ return fmt.Errorf("Error creating striped thinpool: %s", err)
+ }
+
// Create the thin pool
lvmThinPool := fmt.Sprintf("%s/%s", vgName, thinPoolName)
if isRecent {
- _, err = shared.TryRunCommand(
- "lvcreate",
- "-Wy", "--yes",
- "--poolmetadatasize", "1G",
- "-l", "100%FREE",
- "--thinpool", lvmThinPool)
+ if numStripesUint > 1 {
+ _, err = shared.TryRunCommand(
+ "lvcreate",
+ "-Wy", "--yes",
+ "--poolmetadatasize", "1G",
+ "-l", "100%FREE", "-i",
+ numStripes, "-I", stripesSize,
+ "--thinpool", lvmThinPool)
+ } else {
+ _, err = shared.TryRunCommand(
+ "lvcreate",
+ "-Wy", "--yes",
+ "--poolmetadatasize", "1G",
+ "-l", "100%FREE",
+ "--thinpool", lvmThinPool)
+ }
} else {
- _, err = shared.TryRunCommand(
- "lvcreate",
- "-Wy", "--yes",
- "--poolmetadatasize", "1G",
- "-L", "1G",
- "--thinpool", lvmThinPool)
+ if numStripesUint > 1 {
+ _, err = shared.TryRunCommand(
+ "lvcreate",
+ "-Wy", "--yes",
+ "--poolmetadatasize", "1G",
+ "-L", "1G", "-i",
+ numStripes, "-I", stripesSize,
+ "--thinpool", lvmThinPool)
+ } else {
+ _, err = shared.TryRunCommand(
+ "lvcreate",
+ "-Wy", "--yes",
+ "--poolmetadatasize", "1G",
+ "-L", "1G",
+ "--thinpool", lvmThinPool)
+ }
}
if err != nil {
@@ -1088,3 +1164,43 @@ func (s *storageLvm) copyVolumeThinpool(source string, target string, readOnly b
return nil
}
+
+func getLVCreateSize(input string) (string, error) {
+ suffixLen := 0
+
+ for i, chr := range []byte(input) {
+ _, err := strconv.Atoi(string([]byte{chr}))
+ if err != nil {
+ suffixLen = len(input) - i
+ break
+ }
+ }
+
+ if suffixLen == len(input) {
+ return "", fmt.Errorf("Invalid value: %s", input)
+ }
+
+ suffix := input[len(input)-suffixLen:]
+ switch suffix {
+ case "", "B", "bytes":
+ suffix = "B"
+ case "KiB", "kB":
+ suffix = "K"
+ case "MiB", "MB":
+ suffix = "M"
+ case "GiB", "GB":
+ suffix = "G"
+ case "TiB", "TB":
+ suffix = "T"
+ case "PiB", "PB":
+ suffix = "P"
+ case "EiB", "EB":
+ suffix = "E"
+ default:
+ return "", fmt.Errorf("Invalid value: %s", input)
+ }
+ prefix := input[:len(input)-suffixLen]
+ lvCreateString := fmt.Sprintf("%s%s", prefix, suffix)
+
+ return lvCreateString, nil
+}
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index de12a2c089..a2e8715ac8 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -37,6 +37,8 @@ var changeableStoragePoolProperties = map[string][]string{
"lvm.vg_name",
"volume.block.filesystem",
"volume.block.mount_options",
+ "volume.lvm.stripes",
+ "volume.lvm.stripes.size",
"volume.size"},
"zfs": {
@@ -117,6 +119,17 @@ var storagePoolConfigKeys = map[string]func(value string) error{
return err
},
+ //valid drivers: lvm
+ "volume.lvm.stripes": shared.IsUint32,
+ "volume.lvm.stripes.size": func(value string) error {
+ if value == "" {
+ return nil
+ }
+
+ _, err := units.ParseByteSizeString(value)
+ return err
+ },
+
// valid drivers: zfs
"volume.zfs.remove_snapshots": shared.IsBool,
"volume.zfs.use_refquota": shared.IsBool,
@@ -171,7 +184,7 @@ func storagePoolValidateConfig(name string, driver string, config map[string]str
}
if driver != "lvm" {
- if prfx(key, "lvm.") {
+ if prfx(key, "volume.lvm.") || prfx(key, "lvm.") {
return fmt.Errorf("the key %s cannot be used with %s storage pools", key, strings.ToUpper(driver))
}
}
@@ -245,6 +258,10 @@ func storagePoolFillDefault(name string, driver string, config map[string]string
// Unchangeable pool property: Set unconditionally.
config["lvm.thinpool_name"] = "LXDThinPool"
}
+
+ if config["volume.lvm.stripes"] == "" {
+ config["volume.lvm.stripes"] = strconv.FormatUint(uint64(1), 10)
+ }
}
if driver == "btrfs" || driver == "ceph" || driver == "cephfs" || driver == "lvm" || driver == "zfs" {
diff --git a/lxd/storage_volumes_config.go b/lxd/storage_volumes_config.go
index fff20d0496..a9e52ac61d 100644
--- a/lxd/storage_volumes_config.go
+++ b/lxd/storage_volumes_config.go
@@ -44,6 +44,8 @@ var changeableStoragePoolVolumeProperties = map[string][]string{
"security.shifted",
"security.unmapped",
"size",
+ "lvm.stripes",
+ "lvm.stripes.size",
},
"zfs": {
diff --git a/scripts/bash/lxd-client b/scripts/bash/lxd-client
index b437c10524..e4fe0453ef 100644
--- a/scripts/bash/lxd-client
+++ b/scripts/bash/lxd-client
@@ -131,10 +131,12 @@ _have lxc && {
lvm.vg_name rsync.bwlimit volatile.initial_source \
volatile.pool.pristine volume.block.filesystem \
volume.block.mount_options volume.size volume.zfs.remove_snapshots \
- volume.zfs.use_refquota zfs.clone_copy zfs.pool_name"
+ volume.lvm.stripes volume.lvm.stripes.size volume.zfs.use_refquota \
+ zfs.clone_copy zfs.pool_name"
storage_volume_keys="size block.filesystem block.mount_options \
- security.unmapped security.shifted zfs.remove_snapshots zfs.use_refquota"
+ lvm.stripes lvm.stripes.size security.unmapped security.shifted \
+ zfs.remove_snapshots zfs.use_refquota"
if [ $COMP_CWORD -eq 1 ]; then
COMPREPLY=( $(compgen -W "$lxc_cmds" -- ${COMP_WORDS[COMP_CWORD]}) )
More information about the lxc-devel
mailing list