[lxc-devel] [lxd/master] VM: Backup
tomponline on Github
lxc-bot at linuxcontainers.org
Tue Mar 17 17:40:49 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 468 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200317/3090332b/attachment-0001.bin>
-------------- next part --------------
From fe4c3f2d91dc62cd298f336de2875d3c0e313286 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 13:42:21 +0000
Subject: [PATCH 1/9] lxd/db/containers: Renames ContainerBackupCreate and
ContainerBackupRemove
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/db/containers.go | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/lxd/db/containers.go b/lxd/db/containers.go
index 99a4abadb9..ad025f5990 100644
--- a/lxd/db/containers.go
+++ b/lxd/db/containers.go
@@ -1231,8 +1231,8 @@ WHERE projects.name=? AND instances.name=?`
return result, nil
}
-// ContainerBackupCreate creates a new backup
-func (c *Cluster) ContainerBackupCreate(args InstanceBackupArgs) error {
+// InstanceBackupCreate creates a new backup.
+func (c *Cluster) InstanceBackupCreate(args InstanceBackupArgs) error {
_, err := c.ContainerBackupID(args.Name)
if err == nil {
return ErrAlreadyDefined
@@ -1264,7 +1264,7 @@ func (c *Cluster) ContainerBackupCreate(args InstanceBackupArgs) error {
_, err = result.LastInsertId()
if err != nil {
- return fmt.Errorf("Error inserting %s into database", args.Name)
+ return fmt.Errorf("Error inserting %q into database", args.Name)
}
return nil
@@ -1273,9 +1273,8 @@ func (c *Cluster) ContainerBackupCreate(args InstanceBackupArgs) error {
return err
}
-// ContainerBackupRemove removes the container backup with the given name from
-// the database.
-func (c *Cluster) ContainerBackupRemove(name string) error {
+// InstanceBackupRemove removes the container backup with the given name from the database.
+func (c *Cluster) InstanceBackupRemove(name string) error {
id, err := c.ContainerBackupID(name)
if err != nil {
return err
From d4e7c6c2f0d8cb2e8148d7e8980beda81066e435 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 13:43:08 +0000
Subject: [PATCH 2/9] lxd/backup: backupCreate minor clean up
Use of renamed functions
Error message quoting
Revert package usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup.go | 21 ++++++++++-----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index cccdf585da..f2082a87c7 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -18,6 +18,7 @@ import (
"github.com/lxc/lxd/lxd/instance/instancetype"
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/project"
+ "github.com/lxc/lxd/lxd/revert"
"github.com/lxc/lxd/lxd/state"
storagePools "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/lxd/task"
@@ -29,23 +30,20 @@ import (
// Create a new backup.
func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instance.Instance) error {
+ revert := revert.New()
+ defer revert.Fail()
+
// Create the database entry.
- err := s.Cluster.ContainerBackupCreate(args)
+ err := s.Cluster.InstanceBackupCreate(args)
if err != nil {
if err == db.ErrAlreadyDefined {
- return fmt.Errorf("backup '%s' already exists", args.Name)
+ return fmt.Errorf("Backup %q already exists", args.Name)
}
return errors.Wrap(err, "Insert backup info into database")
}
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.Cluster.ContainerBackupRemove(args.Name)
- }()
+ revert.Add(func() { s.Cluster.InstanceBackupRemove(args.Name) })
// Get the backup struct.
b, err := instance.BackupLoadByName(s, sourceInst.Project(), args.Name)
@@ -60,7 +58,8 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
if err != nil {
return err
}
- defer os.RemoveAll(tmpPath)
+
+ revert.Add(func() { os.RemoveAll(tmpPath) })
pool, err := storagePools.GetPoolByInstance(s, sourceInst)
if err != nil {
@@ -78,7 +77,7 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
return err
}
- revert = false
+ revert.Success()
return nil
}
From a87e73093f7fd3212cb337c961512906b8f1d09a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 13:45:15 +0000
Subject: [PATCH 3/9] lxd/backup: InstanceBackupRemove usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup/backup.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go
index 440e8fdb4d..f0cec28c62 100644
--- a/lxd/backup/backup.go
+++ b/lxd/backup/backup.go
@@ -239,7 +239,7 @@ func DoBackupDelete(s *state.State, projectName, backupName, containerName strin
}
// Remove the database record
- err := s.Cluster.ContainerBackupRemove(backupName)
+ err := s.Cluster.InstanceBackupRemove(backupName)
if err != nil {
return err
}
From 41d7652b1e886b720e92c8ae9879650601d6bb3b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:51:24 +0000
Subject: [PATCH 4/9] lxd/backup: Minor tweaks to backupCreateTarball
Improves logging
Uses revert package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup.go | 33 +++++++++++++++------------------
1 file changed, 15 insertions(+), 18 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index f2082a87c7..3e7beed880 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -15,7 +15,6 @@ import (
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/instance"
- "github.com/lxc/lxd/lxd/instance/instancetype"
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/project"
"github.com/lxc/lxd/lxd/revert"
@@ -82,16 +81,15 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
}
func backupCreateTarball(s *state.State, path string, b backup.Backup, c instance.Instance) error {
- // Create the index
+ revert := revert.New()
+ defer revert.Fail()
+
+ // Create the index.
poolName, err := c.StoragePool()
if err != nil {
return err
}
- if c.Type() != instancetype.Container {
- return fmt.Errorf("Instance type must be container")
- }
-
indexFile := backup.Info{
Name: c.Name(),
Privileged: c.IsPrivileged(),
@@ -124,18 +122,21 @@ func backupCreateTarball(s *state.State, path string, b backup.Backup, c instanc
return err
}
- file, err := os.Create(filepath.Join(path, "index.yaml"))
+ indexFilePath := filepath.Join(path, "index.yaml")
+ file, err := os.Create(indexFilePath)
if err != nil {
return err
}
+ revert.Add(func() { os.Remove(indexFilePath) })
+
_, err = file.Write(data)
file.Close()
if err != nil {
return err
}
- // Create the target path if needed
+ // Create the target path if needed.
backupsPath := shared.VarPath("backups", project.Instance(c.Project(), c.Name()))
if !shared.PathExists(backupsPath) {
err := os.MkdirAll(backupsPath, 0700)
@@ -144,17 +145,12 @@ func backupCreateTarball(s *state.State, path string, b backup.Backup, c instanc
}
}
- // Create the tarball
+ // Create the tarball.
backupPath := shared.VarPath("backups", project.Instance(c.Project(), b.Name()))
- success := false
- defer func() {
- if success {
- return
- }
- os.RemoveAll(backupPath)
- }()
+ revert.Add(func() { os.RemoveAll(backupPath) })
+ logger.Debugf("Creating backup tarball from %q to %q", path, backupPath)
args := []string{"-cf", backupPath, "--numeric-owner", "--xattrs", "-C", path, "--transform", "s,^./,backup/,S", "."}
_, err = shared.RunCommand("tar", args...)
if err != nil {
@@ -193,6 +189,7 @@ func backupCreateTarball(s *state.State, path string, b backup.Backup, c instanc
defer compressed.Close()
defer os.Remove(compressedName)
+ logger.Debugf("Compressing backup tarball %q", backupPath)
err = compressFile(compress, infile, compressed)
if err != nil {
return err
@@ -209,13 +206,13 @@ func backupCreateTarball(s *state.State, path string, b backup.Backup, c instanc
}
}
- // Set permissions
+ // Set permissions.
err = os.Chmod(backupPath, 0600)
if err != nil {
return err
}
- success = true
+ revert.Success()
return nil
}
From 4d0f223fa3901be801b1f3f37e3bf8d05d6dc38f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:51:54 +0000
Subject: [PATCH 5/9] lxd/rsync: Adds support for rsync arguments to LocalCopy
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/rsync/rsync.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/lxd/rsync/rsync.go b/lxd/rsync/rsync.go
index 6b18c743e1..101f4a7d65 100644
--- a/lxd/rsync/rsync.go
+++ b/lxd/rsync/rsync.go
@@ -19,7 +19,7 @@ import (
)
// LocalCopy copies a directory using rsync (with the --devices option).
-func LocalCopy(source string, dest string, bwlimit string, xattrs bool) (string, error) {
+func LocalCopy(source string, dest string, bwlimit string, xattrs bool, rsyncArgs ...string) (string, error) {
err := os.MkdirAll(dest, 0755)
if err != nil {
return "", err
@@ -52,6 +52,10 @@ func LocalCopy(source string, dest string, bwlimit string, xattrs bool) (string,
args = append(args, "--bwlimit", bwlimit)
}
+ if len(rsyncArgs) > 0 {
+ args = append(args, rsyncArgs...)
+ }
+
args = append(args,
rsyncVerbosity,
shared.AddSlash(source),
From 5c3c8dcb007a17253541b1e5e07babcb88da2f2c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:53:12 +0000
Subject: [PATCH 6/9] lxd/storage/drivers/utils: Minor tweak to copyDevice
error message
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/utils.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/utils.go b/lxd/storage/drivers/utils.go
index 430f9673b9..4981e6daf2 100644
--- a/lxd/storage/drivers/utils.go
+++ b/lxd/storage/drivers/utils.go
@@ -568,7 +568,7 @@ func copyDevice(inputPath, outputPath string) error {
to, err := os.OpenFile(outputPath, os.O_WRONLY, 0)
if err != nil {
- return errors.Wrapf(err, "Error opening file writing %q", outputPath)
+ return errors.Wrapf(err, "Error opening file for writing %q", outputPath)
}
defer to.Close()
From 608fe31c50a256917931afee6c0a8021057e6433 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:52:16 +0000
Subject: [PATCH 7/9] lxd/stroage/drivers/generic: Tweak error message of
genericCreateVolumeFromMigration
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/generic.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/generic.go b/lxd/storage/drivers/generic.go
index 02eb795e4d..ca18a04fc1 100644
--- a/lxd/storage/drivers/generic.go
+++ b/lxd/storage/drivers/generic.go
@@ -190,7 +190,7 @@ func genericCreateVolumeFromMigration(d Driver, initVolume func(vol Volume) (fun
to, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0)
if err != nil {
- return errors.Wrapf(err, "Error opening file writing %q", path)
+ return errors.Wrapf(err, "Error opening file for writing %q", path)
}
defer to.Close()
From 4f4c7fd0e67fc791999939a1aa4d65e45f330a1e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:52:54 +0000
Subject: [PATCH 8/9] lxd/storage/drivers/generic/vfs: Adds VM support to
genericVFSBackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/generic_vfs.go | 82 ++++++++++++++++++++++--------
1 file changed, 61 insertions(+), 21 deletions(-)
diff --git a/lxd/storage/drivers/generic_vfs.go b/lxd/storage/drivers/generic_vfs.go
index 93a41e74c3..5d65a898ea 100644
--- a/lxd/storage/drivers/generic_vfs.go
+++ b/lxd/storage/drivers/generic_vfs.go
@@ -277,13 +277,64 @@ func genericVFSGetVolumeDiskPath(vol Volume) (string, error) {
func genericVFSBackupVolume(d Driver, vol Volume, targetPath string, snapshots bool, op *operations.Operation) error {
bwlimit := d.Config()["rsync.bwlimit"]
- // Backups only implemented for containers currently.
- if vol.volType != VolumeTypeContainer {
- return ErrNotImplemented
+ // Define a function that can copy a volume into the backup target location.
+ backupVolume := func(v Volume, target string) error {
+ return v.MountTask(func(mountPath string, op *operations.Operation) error {
+ if v.IsVMBlock() {
+ blockPath, err := d.GetVolumeDiskPath(v)
+ if err != nil {
+ return errors.Wrapf(err, "Error getting VM block volume disk path")
+ }
+
+ var rsyncArgs []string
+
+ // Exclude the VM root disk path, if a file image.
+ if !shared.IsBlockdevPath(blockPath) {
+ rsyncArgs = []string{"--exclude", filepath.Base(blockPath)}
+ }
+
+ d.Logger().Debug("Copying virtual machine config volume", log.Ctx{"sourcePath": mountPath, "targetPath": target})
+ _, err = rsync.LocalCopy(mountPath, target, bwlimit, true, rsyncArgs...)
+ if err != nil {
+ return err
+ }
+
+ targetFile := filepath.Join(target, "root.img")
+ d.Logger().Debug("Copying virtual machine block volume", log.Ctx{"sourcePath": blockPath, "targetPath": targetFile})
+ from, err := os.Open(blockPath)
+ if err != nil {
+ return errors.Wrapf(err, "Error opening file for reading %q", blockPath)
+ }
+ defer from.Close()
+
+ to, err := os.OpenFile(targetFile, os.O_CREATE|os.O_WRONLY, 0600)
+ if err != nil {
+ return errors.Wrapf(err, "Error opening file for writing %q", targetFile)
+ }
+ defer to.Close()
+
+ _, err = io.Copy(to, from)
+ if err != nil {
+ return errors.Wrapf(err, "Error copying file %q to %q", blockPath, targetFile)
+ }
+ } else {
+ d.Logger().Debug("Copying container filesystem volume", log.Ctx{"sourcePath": mountPath, "targetPath": target})
+ _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }, op)
}
+
// Handle snapshots.
if snapshots {
snapshotsPath := filepath.Join(targetPath, "snapshots")
+ if vol.IsVMBlock() {
+ snapshotsPath = filepath.Join(targetPath, "virtual-machine-snapshots")
+ }
// List the snapshots.
snapshots, err := vol.Snapshots(op)
@@ -295,7 +346,7 @@ func genericVFSBackupVolume(d Driver, vol Volume, targetPath string, snapshots b
if len(snapshots) > 0 {
err = os.MkdirAll(snapshotsPath, 0711)
if err != nil {
- return errors.Wrapf(err, "Failed to create directory '%s'", snapshotsPath)
+ return errors.Wrapf(err, "Failed to create directory %q", snapshotsPath)
}
}
@@ -303,31 +354,20 @@ func genericVFSBackupVolume(d Driver, vol Volume, targetPath string, snapshots b
_, snapName, _ := shared.InstanceGetParentAndSnapshotName(snapshot.Name())
target := filepath.Join(snapshotsPath, snapName)
- // Copy the snapshot.
- err = snapshot.MountTask(func(mountPath string, op *operations.Operation) error {
- _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
- if err != nil {
- return err
- }
-
- return nil
- }, op)
+ err := backupVolume(snapshot, target)
if err != nil {
return err
}
}
}
- // Copy the parent volume itself.
+ // Copy the main volume itself.
target := filepath.Join(targetPath, "container")
- err := vol.MountTask(func(mountPath string, op *operations.Operation) error {
- _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
- if err != nil {
- return err
- }
+ if vol.IsVMBlock() {
+ target = filepath.Join(targetPath, "virtual-machine")
+ }
- return nil
- }, op)
+ err := backupVolume(vol, target)
if err != nil {
return err
}
From 3a0f3267418c18a0fa06bb53c7a04bc8102b86fc Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 17:29:45 +0000
Subject: [PATCH 9/9] lxd/storage/drivers/driver/lvm/volumes: Fixes LVM VM
snapshot list
This bug was introduced in d782daf202cda0de0a88886e21aa056775024139 due to the way that VM snapshot LVs are named differently than container snapshot LVs.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_lvm_volumes.go | 23 ++++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/lxd/storage/drivers/driver_lvm_volumes.go b/lxd/storage/drivers/driver_lvm_volumes.go
index 76d7e93b2c..9eadf49623 100644
--- a/lxd/storage/drivers/driver_lvm_volumes.go
+++ b/lxd/storage/drivers/driver_lvm_volumes.go
@@ -734,8 +734,6 @@ func (d *lvm) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (b
// VolumeSnapshots returns a list of snapshots for the volume.
func (d *lvm) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) {
- fullVolName := d.lvmFullVolumeName(vol.volType, vol.contentType, vol.name)
-
// We use the volume list rather than inspecting the logical volumes themselves because the origin
// property of an LVM snapshot can be removed/changed when restoring snapshots, such that they are no
// marked as origin of the parent volume. Instead we use prefix matching on the volume names to find the
@@ -757,10 +755,29 @@ func (d *lvm) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, e
}
snapshots := []string{}
- scanner := bufio.NewScanner(stdout)
+ fullVolName := d.lvmFullVolumeName(vol.volType, vol.contentType, vol.name)
+
+ // If block volume, remove the block suffix ready for comparison with LV list.
+ if vol.IsVMBlock() {
+ fullVolName = strings.TrimSuffix(fullVolName, lvmBlockVolSuffix)
+ }
+
prefix := fmt.Sprintf("%s-", fullVolName)
+
+ scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
snapLine := strings.TrimSpace(scanner.Text())
+
+ // If block volume, skip any LVs that don't end with the block suffix, and then for those that do
+ // remove the block suffix so that they can be compared with the fullVolName prefix.
+ if vol.IsVMBlock() {
+ if !strings.HasSuffix(snapLine, lvmBlockVolSuffix) {
+ continue // Ignore non-block volumes.
+ }
+
+ snapLine = strings.TrimSuffix(snapLine, lvmBlockVolSuffix)
+ }
+
if strings.HasPrefix(snapLine, prefix) {
// Remove volume name prefix (including snapshot delimiter) and unescape snapshot name.
snapshots = append(snapshots, strings.Replace(strings.TrimPrefix(snapLine, prefix), "--", "-", -1))
More information about the lxc-devel
mailing list