[lxc-devel] [lxd/master] Backup: Only read up to original size of file during export
tomponline on Github
lxc-bot at linuxcontainers.org
Mon Mar 30 14:26:35 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 420 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200330/72b05bff/attachment.bin>
-------------- next part --------------
From eac2a4fe9072b6bb81aec3953efb1e99c8842211 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 30 Mar 2020 15:22:57 +0100
Subject: [PATCH 1/3] shared/instancewriter/instance/tar/writer: Adds
ignoreGrowth arg to WriteFile
This is to allow us to restore the old behavior of export that only reads up to the original size of a file that is being grown during backup.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/instancewriter/instance_tar_writer.go | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/shared/instancewriter/instance_tar_writer.go b/shared/instancewriter/instance_tar_writer.go
index 021858fa64..d8f3950243 100644
--- a/shared/instancewriter/instance_tar_writer.go
+++ b/shared/instancewriter/instance_tar_writer.go
@@ -35,7 +35,10 @@ func (ctw *InstanceTarWriter) ResetHardLinkMap() {
}
// WriteFile adds a file to the tarball with the specified name using the srcPath file as the contents of the file.
-func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo) error {
+// The ignoreGrowth argument indicates whether to error if the srcPath file increases in size beyond the size in fi
+// during the write. If false the write will return an error. If true, no error is returned, instead only the size
+// specified in fi is written to the tarball. This can be used when you don't need a consistent copy of the file.
+func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo, ignoreGrowth bool) error {
var err error
var major, minor uint32
var nlink int
@@ -115,7 +118,12 @@ func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileI
}
defer f.Close()
- _, err = io.Copy(ctw.tarWriter, f)
+ r := io.Reader(f)
+ if ignoreGrowth {
+ r = io.LimitReader(r, fi.Size())
+ }
+
+ _, err = io.Copy(ctw.tarWriter, r)
if err != nil {
return errors.Wrapf(err, "Failed to copy file content %q", srcPath)
}
From e488b337c30caf2a80963bc9cf034f41b004d173 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 30 Mar 2020 15:24:15 +0100
Subject: [PATCH 2/3] lxd/storage/drivers/generic/vfs: Sets ignoreGrowth arg
true in WriteFile usage
This restores the old behavior of export that will only read up to original file size during backup, even if file is grown during the copy.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/generic_vfs.go | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/lxd/storage/drivers/generic_vfs.go b/lxd/storage/drivers/generic_vfs.go
index 647321f8c9..56cdd9564b 100644
--- a/lxd/storage/drivers/generic_vfs.go
+++ b/lxd/storage/drivers/generic_vfs.go
@@ -486,7 +486,7 @@ func genericVFSBackupVolume(d Driver, vol Volume, tarWriter *instancewriter.Inst
}
name := filepath.Join(prefix, strings.TrimPrefix(srcPath, mountPath))
- err = tarWriter.WriteFile(name, srcPath, fi)
+ err = tarWriter.WriteFile(name, srcPath, fi, false)
if err != nil {
return errors.Wrapf(err, "Error adding %q as %q to tarball", srcPath, name)
}
@@ -524,7 +524,11 @@ func genericVFSBackupVolume(d Driver, vol Volume, tarWriter *instancewriter.Inst
}
name := filepath.Join(prefix, strings.TrimPrefix(srcPath, mountPath))
- err = tarWriter.WriteFile(name, srcPath, fi)
+
+ // Write the file to the tarball with ignoreGrowth enabled so that if the
+ // source file grows during copy we only copy up to the original size.
+ // This means that the file in the tarball may be inconsistent.
+ err = tarWriter.WriteFile(name, srcPath, fi, true)
if err != nil {
return errors.Wrapf(err, "Error adding %q as %q to tarball", srcPath, name)
}
From 7e3559839db5ed39eecb566a8ed043fce6a0210c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 30 Mar 2020 15:25:17 +0100
Subject: [PATCH 3/3] lxd: Existing WriteFile usage updated to set ignoreGrowth
to false
No functional change.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instance/drivers/driver_lxc.go | 8 ++++----
lxd/instance/drivers/driver_qemu.go | 10 +++++-----
lxd/storage/drivers/driver_btrfs_volumes.go | 2 +-
lxd/storage/drivers/driver_zfs_volumes.go | 2 +-
4 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/lxd/instance/drivers/driver_lxc.go b/lxd/instance/drivers/driver_lxc.go
index 9558311863..e600b70143 100644
--- a/lxd/instance/drivers/driver_lxc.go
+++ b/lxd/instance/drivers/driver_lxc.go
@@ -4592,7 +4592,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = tarWriter.WriteFile(path[offset:], path, fi)
+ err = tarWriter.WriteFile(path[offset:], path, fi, false)
if err != nil {
logger.Debugf("Error tarring up %s: %s", path, err)
return err
@@ -4666,7 +4666,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
}
tmpOffset := len(path.Dir(fnam)) + 1
- if err := tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi); err != nil {
+ if err := tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false); err != nil {
tarWriter.Close()
logger.Debugf("Error writing to tarfile: %s", err)
logger.Error("Failed exporting instance", ctxMap)
@@ -4728,9 +4728,9 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
if properties != nil {
tmpOffset := len(path.Dir(fnam)) + 1
- err = tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi)
+ err = tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false)
} else {
- err = tarWriter.WriteFile(fnam[offset:], fnam, fi)
+ err = tarWriter.WriteFile(fnam[offset:], fnam, fi, false)
}
if err != nil {
tarWriter.Close()
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 642df0545c..506e71a0a0 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -2997,7 +2997,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = tarWriter.WriteFile(path[offset:], path, fi)
+ err = tarWriter.WriteFile(path[offset:], path, fi, false)
if err != nil {
logger.Debugf("Error tarring up %s: %s", path, err)
return err
@@ -3072,7 +3072,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
}
tmpOffset := len(filepath.Dir(fnam)) + 1
- if err := tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi); err != nil {
+ if err := tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false); err != nil {
tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -3133,9 +3133,9 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
if properties != nil {
tmpOffset := len(filepath.Dir(fnam)) + 1
- err = tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi)
+ err = tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false)
} else {
- err = tarWriter.WriteFile(fnam[offset:], fnam, fi)
+ err = tarWriter.WriteFile(fnam[offset:], fnam, fi, false)
}
if err != nil {
tarWriter.Close()
@@ -3175,7 +3175,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
}
imgOffset := len(tmpPath) + 1
- err = tarWriter.WriteFile(fPath[imgOffset:], fPath, fi)
+ err = tarWriter.WriteFile(fPath[imgOffset:], fPath, fi, false)
if err != nil {
return err
}
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index f4b4fc44d7..6a592f1af9 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -684,7 +684,7 @@ func (d *btrfs) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWr
return err
}
- err = tarWriter.WriteFile(fileName, tmpFile.Name(), tmpFileInfo)
+ err = tarWriter.WriteFile(fileName, tmpFile.Name(), tmpFileInfo, false)
if err != nil {
return err
}
diff --git a/lxd/storage/drivers/driver_zfs_volumes.go b/lxd/storage/drivers/driver_zfs_volumes.go
index e61a46f5ce..143c660e20 100644
--- a/lxd/storage/drivers/driver_zfs_volumes.go
+++ b/lxd/storage/drivers/driver_zfs_volumes.go
@@ -1246,7 +1246,7 @@ func (d *zfs) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWrit
return err
}
- err = tarWriter.WriteFile(fileName, tmpFile.Name(), tmpFileInfo)
+ err = tarWriter.WriteFile(fileName, tmpFile.Name(), tmpFileInfo, false)
if err != nil {
return err
}
More information about the lxc-devel
mailing list