[lxc-devel] [lxd/master] Container: tar writer backup
tomponline on Github
lxc-bot at linuxcontainers.org
Thu Mar 19 14:24:31 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 400 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200319/fcecac81/attachment-0001.bin>
-------------- next part --------------
From cbe0c96f28ff79f6ce1514ddd6a6e0b7682ff767 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 18 Mar 2020 11:14:09 +0000
Subject: [PATCH 01/23] shared/containerwriter: Renames to instancewriter
And uses errors package for error wrapping
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
.../instance_tar_writer.go} | 62 +++++++++----------
1 file changed, 30 insertions(+), 32 deletions(-)
rename shared/{containerwriter/container_tar_writer.go => instancewriter/instance_tar_writer.go} (51%)
diff --git a/shared/containerwriter/container_tar_writer.go b/shared/instancewriter/instance_tar_writer.go
similarity index 51%
rename from shared/containerwriter/container_tar_writer.go
rename to shared/instancewriter/instance_tar_writer.go
index 7ebbf43546..2199cbadfe 100644
--- a/shared/containerwriter/container_tar_writer.go
+++ b/shared/instancewriter/instance_tar_writer.go
@@ -1,35 +1,35 @@
-package containerwriter
+package instancewriter
import (
"archive/tar"
- "fmt"
"io"
"os"
"strings"
+ "github.com/pkg/errors"
+
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/idmap"
)
-// ContainerTarWriter provides a TarWriter implementation that handles
-// ID shifting and hardlink tracking for containers
-type ContainerTarWriter struct {
+// InstanceTarWriter provides a TarWriter implementation that handles ID shifting and hardlink tracking.
+type InstanceTarWriter struct {
tarWriter *tar.Writer
idmapSet *idmap.IdmapSet
linkMap map[uint64]string
}
-// NewContainerTarWriter returns a ContainerTarWriter for the provided target Writer and id map
-func NewContainerTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *ContainerTarWriter {
- ctw := new(ContainerTarWriter)
+// NewInstanceTarWriter returns a ContainerTarWriter for the provided target Writer and id map.
+func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *InstanceTarWriter {
+ ctw := new(InstanceTarWriter)
ctw.tarWriter = tar.NewWriter(writer)
ctw.idmapSet = idmapSet
ctw.linkMap = map[uint64]string{}
return ctw
}
-// WriteFile adds a file to the tarball
-func (ctw *ContainerTarWriter) WriteFile(offset int, path string, fi os.FileInfo) error {
+// WriteFile adds a file to the tarball.
+func (ctw *InstanceTarWriter) WriteFile(offset int, path string, fi os.FileInfo) error {
var err error
var major, minor uint32
var nlink int
@@ -39,18 +39,18 @@ func (ctw *ContainerTarWriter) WriteFile(offset int, path string, fi os.FileInfo
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err = os.Readlink(path)
if err != nil {
- return fmt.Errorf("failed to resolve symlink: %s", err)
+ return errors.Wrap(err, "Failed to resolve symlink")
}
}
- // Sockets cannot be stored in tarballs, just skip them (consistent with tar)
+ // Sockets cannot be stored in tarballs, just skip them (consistent with tar).
if fi.Mode()&os.ModeSocket == os.ModeSocket {
return nil
}
hdr, err := tar.FileInfoHeader(fi, link)
if err != nil {
- return fmt.Errorf("failed to create tar info header: %s", err)
+ return errors.Wrap(err, "Failed to create tar info header")
}
hdr.Name = path[offset:]
@@ -62,25 +62,23 @@ func (ctw *ContainerTarWriter) WriteFile(offset int, path string, fi os.FileInfo
hdr.Uid, hdr.Gid, major, minor, ino, nlink, err = shared.GetFileStat(path)
if err != nil {
- return fmt.Errorf("failed to get file stat: %s", err)
+ return errors.Wrap(err, "Failed to get file stat")
}
- // Unshift the id under rootfs/ for unpriv containers
- if strings.HasPrefix(hdr.Name, "rootfs") {
- if ctw.idmapSet != nil {
- hUID, hGID := ctw.idmapSet.ShiftFromNs(int64(hdr.Uid), int64(hdr.Gid))
- hdr.Uid = int(hUID)
- hdr.Gid = int(hGID)
- if hdr.Uid == -1 || hdr.Gid == -1 {
- return nil
- }
+ // Unshift the id under rootfs/ for unpriv containers.
+ if strings.HasPrefix(hdr.Name, "rootfs") && ctw.idmapSet != nil {
+ hUID, hGID := ctw.idmapSet.ShiftFromNs(int64(hdr.Uid), int64(hdr.Gid))
+ hdr.Uid = int(hUID)
+ hdr.Gid = int(hGID)
+ if hdr.Uid == -1 || hdr.Gid == -1 {
+ return nil
}
}
hdr.Devmajor = int64(major)
hdr.Devminor = int64(minor)
- // If it's a hardlink we've already seen use the old name
+ // If it's a hardlink we've already seen use the old name.
if fi.Mode().IsRegular() && nlink > 1 {
if firstPath, found := ctw.linkMap[ino]; found {
hdr.Typeflag = tar.TypeLink
@@ -91,38 +89,38 @@ func (ctw *ContainerTarWriter) WriteFile(offset int, path string, fi os.FileInfo
}
}
- // Handle xattrs (for real files only)
+ // Handle xattrs (for real files only).
if link == "" {
hdr.Xattrs, err = shared.GetAllXattr(path)
if err != nil {
- return fmt.Errorf("failed to read xattr for '%s': %s", path, err)
+ return errors.Wrapf(err, "Failed to read xattr for %q", path)
}
}
if err := ctw.tarWriter.WriteHeader(hdr); err != nil {
- return fmt.Errorf("failed to write tar header: %s", err)
+ return errors.Wrap(err, "Failed to write tar header")
}
if hdr.Typeflag == tar.TypeReg {
f, err := os.Open(path)
if err != nil {
- return fmt.Errorf("failed to open the file: %s", err)
+ return errors.Wrap(err, "Failed to open the file")
}
defer f.Close()
if _, err := io.Copy(ctw.tarWriter, f); err != nil {
- return fmt.Errorf("failed to copy file content: %s", err)
+ return errors.Wrap(err, "Failed to copy file content")
}
}
return nil
}
-// Close finishes writing the tarball
-func (ctw *ContainerTarWriter) Close() error {
+// Close finishes writing the tarball.
+func (ctw *InstanceTarWriter) Close() error {
err := ctw.tarWriter.Close()
if err != nil {
- return fmt.Errorf("failed to close tar writer: %s", err)
+ return errors.Wrap(err, "Failed to close tar writer")
}
return nil
}
From f15d07c0f588e12ed8123acdbb32e9275afb078f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 18 Mar 2020 11:20:42 +0000
Subject: [PATCH 02/23] lxd/instance/drivers: instancetarwriter usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instance/drivers/driver_lxc.go | 40 +++++++++++++--------------
lxd/instance/drivers/driver_qemu.go | 42 ++++++++++++++---------------
2 files changed, 41 insertions(+), 41 deletions(-)
diff --git a/lxd/instance/drivers/driver_lxc.go b/lxd/instance/drivers/driver_lxc.go
index 0507b5be8a..6b052edf60 100644
--- a/lxd/instance/drivers/driver_lxc.go
+++ b/lxd/instance/drivers/driver_lxc.go
@@ -48,8 +48,8 @@ import (
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
- "github.com/lxc/lxd/shared/containerwriter"
"github.com/lxc/lxd/shared/idmap"
+ "github.com/lxc/lxd/shared/instancewriter"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/lxc/lxd/shared/netutils"
@@ -4561,7 +4561,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
}
// Create the tarball.
- ctw := containerwriter.NewContainerTarWriter(w, idmap)
+ tarWriter := instancewriter.NewInstanceTarWriter(w, idmap)
// Keep track of the first path we saw for each path with nlink>1.
cDir := c.Path()
@@ -4574,7 +4574,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = ctw.WriteFile(offset, path, fi)
+ err = tarWriter.WriteFile(offset, path, fi)
if err != nil {
logger.Debugf("Error tarring up %s: %s", path, err)
return err
@@ -4588,7 +4588,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
// Generate a new metadata.yaml.
tempDir, err := ioutil.TempDir("", "lxd_lxd_metadata_")
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4600,7 +4600,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
parentName, _, _ := shared.InstanceGetParentAndSnapshotName(c.name)
parent, err := instance.LoadByProjectAndName(c.state, c.project, parentName)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4626,7 +4626,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
data, err := yaml.Marshal(&meta)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4635,21 +4635,21 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
fnam = filepath.Join(tempDir, "metadata.yaml")
err = ioutil.WriteFile(fnam, data, 0644)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
fi, err := os.Lstat(fnam)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
tmpOffset := len(path.Dir(fnam)) + 1
- if err := ctw.WriteFile(tmpOffset, fnam, fi); err != nil {
- ctw.Close()
+ if err := tarWriter.WriteFile(tmpOffset, fnam, fi); err != nil {
+ tarWriter.Close()
logger.Debugf("Error writing to tarfile: %s", err)
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -4659,7 +4659,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
// Parse the metadata.
content, err := ioutil.ReadFile(fnam)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4667,7 +4667,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
metadata := new(api.ImageMetadata)
err = yaml.Unmarshal(content, &metadata)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4676,7 +4676,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
// Generate a new metadata.yaml.
tempDir, err := ioutil.TempDir("", "lxd_lxd_metadata_")
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4684,7 +4684,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
data, err := yaml.Marshal(&metadata)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4693,7 +4693,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
fnam = filepath.Join(tempDir, "metadata.yaml")
err = ioutil.WriteFile(fnam, data, 0644)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -4702,7 +4702,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
// Include metadata.yaml in the tarball.
fi, err := os.Lstat(fnam)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Debugf("Error statting %s during export", fnam)
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -4710,12 +4710,12 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
if properties != nil {
tmpOffset := len(path.Dir(fnam)) + 1
- err = ctw.WriteFile(tmpOffset, fnam, fi)
+ err = tarWriter.WriteFile(tmpOffset, fnam, fi)
} else {
- err = ctw.WriteFile(offset, fnam, fi)
+ err = tarWriter.WriteFile(offset, fnam, fi)
}
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Debugf("Error writing to tarfile: %s", err)
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -4740,7 +4740,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
}
}
- err = ctw.Close()
+ err = tarWriter.Close()
if err != nil {
logger.Error("Failed exporting instance", ctxMap)
return err
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 10d1b86c29..8cf04c6fb0 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -50,7 +50,7 @@ import (
"github.com/lxc/lxd/lxd/vsock"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
- "github.com/lxc/lxd/shared/containerwriter"
+ "github.com/lxc/lxd/shared/instancewriter"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/lxc/lxd/shared/osarch"
@@ -2986,7 +2986,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
}
// Create the tarball.
- ctw := containerwriter.NewContainerTarWriter(w, nil)
+ tarWriter := instancewriter.NewInstanceTarWriter(w, nil)
// Path inside the tar image is the pathname starting after cDir.
cDir := vm.Path()
@@ -2997,7 +2997,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = ctw.WriteFile(offset, path, fi)
+ err = tarWriter.WriteFile(offset, path, fi)
if err != nil {
logger.Debugf("Error tarring up %s: %s", path, err)
return err
@@ -3012,7 +3012,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
// Generate a new metadata.yaml.
tempDir, err := ioutil.TempDir("", "lxd_lxd_metadata_")
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3024,7 +3024,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
parentName, _, _ := shared.InstanceGetParentAndSnapshotName(vm.name)
parent, err := instance.LoadByProjectAndName(vm.state, vm.project, parentName)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3050,7 +3050,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
data, err := yaml.Marshal(&meta)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3059,21 +3059,21 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
fnam = filepath.Join(tempDir, "metadata.yaml")
err = ioutil.WriteFile(fnam, data, 0644)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
fi, err := os.Lstat(fnam)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
tmpOffset := len(filepath.Dir(fnam)) + 1
- if err := ctw.WriteFile(tmpOffset, fnam, fi); err != nil {
- ctw.Close()
+ if err := tarWriter.WriteFile(tmpOffset, fnam, fi); err != nil {
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3082,7 +3082,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
// Parse the metadata.
content, err := ioutil.ReadFile(fnam)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3090,7 +3090,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
metadata := new(api.ImageMetadata)
err = yaml.Unmarshal(content, &metadata)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3099,7 +3099,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
// Generate a new metadata.yaml.
tempDir, err := ioutil.TempDir("", "lxd_lxd_metadata_")
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3107,7 +3107,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
data, err := yaml.Marshal(&metadata)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3116,7 +3116,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
fnam = filepath.Join(tempDir, "metadata.yaml")
err = ioutil.WriteFile(fnam, data, 0644)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
}
@@ -3125,7 +3125,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
// Include metadata.yaml in the tarball.
fi, err := os.Lstat(fnam)
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Debugf("Error statting %s during export", fnam)
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -3133,12 +3133,12 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
if properties != nil {
tmpOffset := len(filepath.Dir(fnam)) + 1
- err = ctw.WriteFile(tmpOffset, fnam, fi)
+ err = tarWriter.WriteFile(tmpOffset, fnam, fi)
} else {
- err = ctw.WriteFile(offset, fnam, fi)
+ err = tarWriter.WriteFile(offset, fnam, fi)
}
if err != nil {
- ctw.Close()
+ tarWriter.Close()
logger.Debugf("Error writing to tarfile: %s", err)
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -3174,7 +3174,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = ctw.WriteFile(len(tmpPath)+1, fPath, fi)
+ err = tarWriter.WriteFile(len(tmpPath)+1, fPath, fi)
if err != nil {
return err
}
@@ -3189,7 +3189,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
}
}
- err = ctw.Close()
+ err = tarWriter.Close()
if err != nil {
logger.Error("Failed exporting instance", ctxMap)
return err
From f0705d11e03f0b93da204d03774b91e08f23c0d2 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 13:29:22 +0000
Subject: [PATCH 03/23] shared/instancewriter/instance/tar/writer: Modifies
WriteFile to accept a file name arg
Rather than an offset location from the srcPath name.
This way files can be added to the tarball using an arbitrary name.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/instancewriter/instance_tar_writer.go | 24 ++++++++++----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/shared/instancewriter/instance_tar_writer.go b/shared/instancewriter/instance_tar_writer.go
index 2199cbadfe..f0390eeed7 100644
--- a/shared/instancewriter/instance_tar_writer.go
+++ b/shared/instancewriter/instance_tar_writer.go
@@ -28,8 +28,8 @@ func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *InstanceT
return ctw
}
-// WriteFile adds a file to the tarball.
-func (ctw *InstanceTarWriter) WriteFile(offset int, path string, fi os.FileInfo) error {
+// WriteFile adds a file to the tarball with the specified name using the srcPath file as the contents of the file.
+func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo) error {
var err error
var major, minor uint32
var nlink int
@@ -37,9 +37,9 @@ func (ctw *InstanceTarWriter) WriteFile(offset int, path string, fi os.FileInfo)
link := ""
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
- link, err = os.Readlink(path)
+ link, err = os.Readlink(srcPath)
if err != nil {
- return errors.Wrap(err, "Failed to resolve symlink")
+ return errors.Wrapf(err, "Failed to resolve symlink for %q", srcPath)
}
}
@@ -53,16 +53,16 @@ func (ctw *InstanceTarWriter) WriteFile(offset int, path string, fi os.FileInfo)
return errors.Wrap(err, "Failed to create tar info header")
}
- hdr.Name = path[offset:]
+ hdr.Name = name
if fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink {
hdr.Size = 0
} else {
hdr.Size = fi.Size()
}
- hdr.Uid, hdr.Gid, major, minor, ino, nlink, err = shared.GetFileStat(path)
+ hdr.Uid, hdr.Gid, major, minor, ino, nlink, err = shared.GetFileStat(srcPath)
if err != nil {
- return errors.Wrap(err, "Failed to get file stat")
+ return errors.Wrapf(err, "Failed to get file stat %q", srcPath)
}
// Unshift the id under rootfs/ for unpriv containers.
@@ -91,9 +91,9 @@ func (ctw *InstanceTarWriter) WriteFile(offset int, path string, fi os.FileInfo)
// Handle xattrs (for real files only).
if link == "" {
- hdr.Xattrs, err = shared.GetAllXattr(path)
+ hdr.Xattrs, err = shared.GetAllXattr(srcPath)
if err != nil {
- return errors.Wrapf(err, "Failed to read xattr for %q", path)
+ return errors.Wrapf(err, "Failed to read xattr for %q", srcPath)
}
}
@@ -102,14 +102,14 @@ func (ctw *InstanceTarWriter) WriteFile(offset int, path string, fi os.FileInfo)
}
if hdr.Typeflag == tar.TypeReg {
- f, err := os.Open(path)
+ f, err := os.Open(srcPath)
if err != nil {
- return errors.Wrap(err, "Failed to open the file")
+ return errors.Wrapf(err, "Failed to open file %q", srcPath)
}
defer f.Close()
if _, err := io.Copy(ctw.tarWriter, f); err != nil {
- return errors.Wrap(err, "Failed to copy file content")
+ return errors.Wrapf(err, "Failed to copy file content %q", srcPath)
}
}
From b52ee57769b9c996f4aaf1332d8ba29efd440f1e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 13:30:11 +0000
Subject: [PATCH 04/23] shared/instancewriter/instance/tar/writer: Adds
ResetHardLinkMap function
Allows multiple instances and/or snapshots to be added to same tarball correctly by allowing the hardlink map to be reset between volumes.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/instancewriter/instance_tar_writer.go | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/shared/instancewriter/instance_tar_writer.go b/shared/instancewriter/instance_tar_writer.go
index f0390eeed7..52a422f9ab 100644
--- a/shared/instancewriter/instance_tar_writer.go
+++ b/shared/instancewriter/instance_tar_writer.go
@@ -28,6 +28,12 @@ func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *InstanceT
return ctw
}
+// ResetHardLinkMap resets the hard link map. Use when copying multiple instances (or snapshots) into a tarball.
+// So that the hard link map doesn't work across different instances/snapshots.
+func (ctw *InstanceTarWriter) ResetHardLinkMap() {
+ ctw.linkMap = map[uint64]string{}
+}
+
// WriteFile adds a file to the tarball with the specified name using the srcPath file as the contents of the file.
func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo) error {
var err error
From e964ea702dca62a1a8eb71b18b60384acbd3ae13 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 13:53:29 +0000
Subject: [PATCH 05/23] lxd/instance/drivers: instancetarwriter.WriteFile name
arg usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instance/drivers/driver_lxc.go | 8 ++++----
lxd/instance/drivers/driver_qemu.go | 11 ++++++-----
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/lxd/instance/drivers/driver_lxc.go b/lxd/instance/drivers/driver_lxc.go
index 6b052edf60..4f1e2af9ad 100644
--- a/lxd/instance/drivers/driver_lxc.go
+++ b/lxd/instance/drivers/driver_lxc.go
@@ -4574,7 +4574,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = tarWriter.WriteFile(offset, path, fi)
+ err = tarWriter.WriteFile(path[offset:], path, fi)
if err != nil {
logger.Debugf("Error tarring up %s: %s", path, err)
return err
@@ -4648,7 +4648,7 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
}
tmpOffset := len(path.Dir(fnam)) + 1
- if err := tarWriter.WriteFile(tmpOffset, fnam, fi); err != nil {
+ if err := tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi); err != nil {
tarWriter.Close()
logger.Debugf("Error writing to tarfile: %s", err)
logger.Error("Failed exporting instance", ctxMap)
@@ -4710,9 +4710,9 @@ func (c *lxc) Export(w io.Writer, properties map[string]string) error {
if properties != nil {
tmpOffset := len(path.Dir(fnam)) + 1
- err = tarWriter.WriteFile(tmpOffset, fnam, fi)
+ err = tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi)
} else {
- err = tarWriter.WriteFile(offset, fnam, fi)
+ err = tarWriter.WriteFile(fnam[offset:], fnam, fi)
}
if err != nil {
tarWriter.Close()
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 8cf04c6fb0..d89ff7d6eb 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -2997,7 +2997,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = tarWriter.WriteFile(offset, path, fi)
+ err = tarWriter.WriteFile(path[offset:], path, fi)
if err != nil {
logger.Debugf("Error tarring up %s: %s", path, err)
return err
@@ -3072,7 +3072,7 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
}
tmpOffset := len(filepath.Dir(fnam)) + 1
- if err := tarWriter.WriteFile(tmpOffset, fnam, fi); err != nil {
+ if err := tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi); err != nil {
tarWriter.Close()
logger.Error("Failed exporting instance", ctxMap)
return err
@@ -3133,9 +3133,9 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
if properties != nil {
tmpOffset := len(filepath.Dir(fnam)) + 1
- err = tarWriter.WriteFile(tmpOffset, fnam, fi)
+ err = tarWriter.WriteFile(fnam[tmpOffset:], fnam, fi)
} else {
- err = tarWriter.WriteFile(offset, fnam, fi)
+ err = tarWriter.WriteFile(fnam[offset:], fnam, fi)
}
if err != nil {
tarWriter.Close()
@@ -3174,7 +3174,8 @@ func (vm *qemu) Export(w io.Writer, properties map[string]string) error {
return err
}
- err = tarWriter.WriteFile(len(tmpPath)+1, fPath, fi)
+ imgOffset := len(tmpPath) + 1
+ err = tarWriter.WriteFile(fPath[imgOffset:], fPath, fi)
if err != nil {
return err
}
From c6c97488d4de2e5645de75f606d55629b290884c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 13:42:21 +0000
Subject: [PATCH 06/23] lxd/db/containers: Renames ContainerBackupCreate and
ContainerBackupRemove
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/db/containers.go | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/lxd/db/containers.go b/lxd/db/containers.go
index 89bc4aad43..47476e0046 100644
--- a/lxd/db/containers.go
+++ b/lxd/db/containers.go
@@ -1212,8 +1212,8 @@ WHERE projects.name=? AND instances.name=?`
return result, nil
}
-// ContainerBackupCreate creates a new backup
-func (c *Cluster) ContainerBackupCreate(args InstanceBackupArgs) error {
+// InstanceBackupCreate creates a new backup.
+func (c *Cluster) InstanceBackupCreate(args InstanceBackupArgs) error {
_, err := c.ContainerBackupID(args.Name)
if err == nil {
return ErrAlreadyDefined
@@ -1245,7 +1245,7 @@ func (c *Cluster) ContainerBackupCreate(args InstanceBackupArgs) error {
_, err = result.LastInsertId()
if err != nil {
- return fmt.Errorf("Error inserting %s into database", args.Name)
+ return fmt.Errorf("Error inserting %q into database", args.Name)
}
return nil
@@ -1254,9 +1254,8 @@ func (c *Cluster) ContainerBackupCreate(args InstanceBackupArgs) error {
return err
}
-// ContainerBackupRemove removes the container backup with the given name from
-// the database.
-func (c *Cluster) ContainerBackupRemove(name string) error {
+// InstanceBackupRemove removes the container backup with the given name from the database.
+func (c *Cluster) InstanceBackupRemove(name string) error {
id, err := c.ContainerBackupID(name)
if err != nil {
return err
From e03c52f7a79529276d03ce727b8c83fb8484068a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 13:43:08 +0000
Subject: [PATCH 07/23] lxd/backup: Updates instance backup to use tar writer
rather than tar cmd
Also:
Use of renamed functions
Error message quoting
Improves logging
Uses revert package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup.go | 255 ++++++++++++++++++++++++++------------------------
1 file changed, 131 insertions(+), 124 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index cccdf585da..177a21c385 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -2,6 +2,7 @@ package main
import (
"fmt"
+ "io"
"io/ioutil"
"os"
"path/filepath"
@@ -9,6 +10,7 @@ import (
"context"
+ "github.com/pkg/errors"
"gopkg.in/yaml.v2"
"github.com/lxc/lxd/lxd/backup"
@@ -18,34 +20,48 @@ import (
"github.com/lxc/lxd/lxd/instance/instancetype"
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/project"
+ "github.com/lxc/lxd/lxd/revert"
"github.com/lxc/lxd/lxd/state"
storagePools "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/lxd/task"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/idmap"
+ "github.com/lxc/lxd/shared/instancewriter"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
- "github.com/pkg/errors"
+ "github.com/lxc/lxd/shared/logging"
)
// Create a new backup.
func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instance.Instance) error {
+ logger := logging.AddContext(logger.Log, log.Ctx{"project": sourceInst.Project(), "instance": sourceInst.Name(), "name": args.Name})
+ logger.Debug("Instance backup started")
+ defer logger.Debug("Instance backup finished")
+
+ if sourceInst.Type() != instancetype.Container {
+ return fmt.Errorf("Instance type must be container")
+ }
+
+ revert := revert.New()
+ defer revert.Fail()
+
+ // Get storage pool.
+ pool, err := storagePools.GetPoolByInstance(s, sourceInst)
+ if err != nil {
+ return errors.Wrap(err, "Load instance storage pool")
+ }
+
// Create the database entry.
- err := s.Cluster.ContainerBackupCreate(args)
+ err = s.Cluster.InstanceBackupCreate(args)
if err != nil {
if err == db.ErrAlreadyDefined {
- return fmt.Errorf("backup '%s' already exists", args.Name)
+ return fmt.Errorf("Backup %q already exists", args.Name)
}
return errors.Wrap(err, "Insert backup info into database")
}
- revert := true
- defer func() {
- if !revert {
- return
- }
- s.Cluster.ContainerBackupRemove(args.Name)
- }()
+ revert.Add(func() { s.Cluster.InstanceBackupRemove(args.Name) })
// Get the backup struct.
b, err := instance.BackupLoadByName(s, sourceInst.Project(), args.Name)
@@ -53,170 +69,161 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
return errors.Wrap(err, "Load backup object")
}
+ // Detect compression method.
+ var compress string
b.SetCompressionAlgorithm(args.CompressionAlgorithm)
-
- // Create a temporary path for the backup.
- tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_")
- if err != nil {
- return err
- }
- defer os.RemoveAll(tmpPath)
-
- pool, err := storagePools.GetPoolByInstance(s, sourceInst)
- if err != nil {
- return errors.Wrap(err, "Load instance storage pool")
+ if b.CompressionAlgorithm() != "" {
+ compress = b.CompressionAlgorithm()
+ } else {
+ compress, err = cluster.ConfigGetString(s.Cluster, "backups.compression_algorithm")
+ if err != nil {
+ return err
+ }
}
- err = pool.BackupInstance(sourceInst, tmpPath, b.OptimizedStorage(), !b.InstanceOnly(), nil)
- if err != nil {
- return errors.Wrap(err, "Backup create")
- }
+ // Create the target path if needed.
+ backupsPath := shared.VarPath("backups", project.Instance(sourceInst.Project(), sourceInst.Name()))
+ if !shared.PathExists(backupsPath) {
+ err := os.MkdirAll(backupsPath, 0700)
+ if err != nil {
+ return err
+ }
- // Pack the backup.
- err = backupCreateTarball(s, tmpPath, *b, sourceInst)
- if err != nil {
- return err
+ revert.Add(func() { os.Remove(backupsPath) })
}
- revert = false
- return nil
-}
+ target := shared.VarPath("backups", project.Instance(sourceInst.Project(), b.Name()))
-func backupCreateTarball(s *state.State, path string, b backup.Backup, c instance.Instance) error {
- // Create the index
- poolName, err := c.StoragePool()
+ // Create temp dir for storing transient files that will be removed at end.
+ tmpDirPath := fmt.Sprintf("%s_tmp", target)
+ logger.Debug("Creating temporary backup directory", log.Ctx{"path": tmpDirPath})
+ err = os.Mkdir(tmpDirPath, 0700)
if err != nil {
return err
}
+ defer os.RemoveAll(tmpDirPath)
- if c.Type() != instancetype.Container {
- return fmt.Errorf("Instance type must be container")
- }
-
- indexFile := backup.Info{
- Name: c.Name(),
- Privileged: c.IsPrivileged(),
- Pool: poolName,
- Snapshots: []string{},
- }
-
- pool, err := storagePools.GetPoolByInstance(s, c)
+ // Setup the tarball writer.
+ logger.Debug("Opening backup tarball for writing", log.Ctx{"path": target})
+ tarFileWriter, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
- return err
+ return errors.Wrapf(err, "Error opening backup tarball for writing %q", target)
}
-
- info := pool.Driver().Info()
- indexFile.Backend = info.Name
-
- if !b.InstanceOnly() {
- snaps, err := c.Snapshots()
+ defer tarFileWriter.Close()
+ revert.Add(func() { os.Remove(target) })
+
+ // Get IDMap to unshift container as the tarball is created.
+ var idmap *idmap.IdmapSet
+ if sourceInst.Type() == instancetype.Container {
+ c := sourceInst.(instance.Container)
+ idmap, err = c.DiskIdmap()
if err != nil {
- return err
+ return errors.Wrap(err, "Error getting container IDMAP")
}
+ }
- for _, snap := range snaps {
- _, snapName, _ := shared.InstanceGetParentAndSnapshotName(snap.Name())
- indexFile.Snapshots = append(indexFile.Snapshots, snapName)
+ // Create the tarball.
+ tarPipeReader, tarPipeWriter := io.Pipe()
+ defer tarPipeWriter.Close() // Ensure that go routine below always ends.
+ tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, idmap)
+
+ // Setup tar writer go routine, with optional compression.
+ tarWriterRes := make(chan error, 0)
+ go func(resCh chan<- error) {
+ logger.Debug("Started backup tarball writer")
+ defer logger.Debug("Finished backup tarball writer")
+ if compress != "none" {
+ err = compressFile(compress, tarPipeReader, tarFileWriter)
+ } else {
+ _, err = io.Copy(tarFileWriter, tarPipeReader)
}
- }
+ resCh <- err
+ }(tarWriterRes)
- data, err := yaml.Marshal(&indexFile)
+ // Write index file.
+ indexFile := filepath.Join(tmpDirPath, "index.yaml")
+ logger.Debug("Adding backup index file", log.Ctx{"path": indexFile})
+ err = backupWriteIndex(sourceInst, pool, b.InstanceOnly(), indexFile, tarWriter)
if err != nil {
- return err
+ return errors.Wrapf(err, "Error writing backup index file")
}
- file, err := os.Create(filepath.Join(path, "index.yaml"))
- if err != nil {
- return err
- }
+ // Path inside the tar image is the pathname starting after cDir.
+ //iDir := sourceInst.Path()
+ //offset := len(iDir) + 1
- _, err = file.Write(data)
- file.Close()
+ err = pool.BackupInstance(sourceInst, tarWriter, b.OptimizedStorage(), !b.InstanceOnly(), nil)
if err != nil {
- return err
+ return errors.Wrap(err, "Backup create")
}
- // Create the target path if needed
- backupsPath := shared.VarPath("backups", project.Instance(c.Project(), c.Name()))
- if !shared.PathExists(backupsPath) {
- err := os.MkdirAll(backupsPath, 0700)
- if err != nil {
- return err
- }
+ // Close off the tarball file.
+ err = tarWriter.Close()
+ if err != nil {
+ return errors.Wrap(err, "Error closing tarball writer")
}
- // Create the tarball
- backupPath := shared.VarPath("backups", project.Instance(c.Project(), b.Name()))
- success := false
- defer func() {
- if success {
- return
- }
-
- os.RemoveAll(backupPath)
- }()
-
- args := []string{"-cf", backupPath, "--numeric-owner", "--xattrs", "-C", path, "--transform", "s,^./,backup/,S", "."}
- _, err = shared.RunCommand("tar", args...)
+ // Close off the tarball pipe writer (this will end the go routine above).
+ err = tarPipeWriter.Close()
if err != nil {
- return err
+ return errors.Wrap(err, "Error closing tarball pipe writer")
}
- err = os.RemoveAll(path)
+ err = <-tarWriterRes
if err != nil {
- return err
+ return errors.Wrap(err, "Error writing tarball")
}
- var compress string
+ revert.Success()
+ return nil
+}
- if b.CompressionAlgorithm() != "" {
- compress = b.CompressionAlgorithm()
- } else {
- compress, err = cluster.ConfigGetString(s.Cluster, "backups.compression_algorithm")
- if err != nil {
- return err
- }
+// backupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball.
+func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, instanceOnly bool, indexFile string, tarWriter *instancewriter.InstanceTarWriter) error {
+ indexInfo := backup.Info{
+ Name: sourceInst.Name(),
+ Privileged: sourceInst.IsPrivileged(),
+ Pool: pool.Name(),
+ Snapshots: []string{},
+ Backend: pool.Driver().Info().Name,
}
- if compress != "none" {
- infile, err := os.Open(backupPath)
+ if !instanceOnly {
+ snaps, err := sourceInst.Snapshots()
if err != nil {
return err
}
- defer infile.Close()
- compressed, err := os.Create(backupPath + ".compressed")
- if err != nil {
- return err
+ for _, snap := range snaps {
+ _, snapName, _ := shared.InstanceGetParentAndSnapshotName(snap.Name())
+ indexInfo.Snapshots = append(indexInfo.Snapshots, snapName)
}
- compressedName := compressed.Name()
-
- defer compressed.Close()
- defer os.Remove(compressedName)
+ }
- err = compressFile(compress, infile, compressed)
- if err != nil {
- return err
- }
+ // Convert to JSON.
+ indexData, err := yaml.Marshal(&indexInfo)
+ if err != nil {
+ return err
+ }
- err = os.Remove(backupPath)
- if err != nil {
- return err
- }
+ // Write index JSON to file.
+ err = ioutil.WriteFile(indexFile, indexData, 0644)
+ if err != nil {
+ return err
+ }
+ defer os.Remove(indexFile)
- err = os.Rename(compressedName, backupPath)
- if err != nil {
- return err
- }
+ indexFileInfo, err := os.Lstat(indexFile)
+ if err != nil {
+ return err
}
- // Set permissions
- err = os.Chmod(backupPath, 0600)
+ // Write to tarball.
+ err = tarWriter.WriteFile("backup/index.yaml", indexFile, indexFileInfo)
if err != nil {
return err
}
- success = true
return nil
}
From a82df288950bf1e793551983384f10c9e28f0bcc Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 13:45:15 +0000
Subject: [PATCH 08/23] lxd/backup: InstanceBackupRemove usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup/backup.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go
index 440e8fdb4d..f0cec28c62 100644
--- a/lxd/backup/backup.go
+++ b/lxd/backup/backup.go
@@ -239,7 +239,7 @@ func DoBackupDelete(s *state.State, projectName, backupName, containerName strin
}
// Remove the database record
- err := s.Cluster.ContainerBackupRemove(backupName)
+ err := s.Cluster.InstanceBackupRemove(backupName)
if err != nil {
return err
}
From 1637097f630caa19f6961b04322adca419e29b96 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:53:12 +0000
Subject: [PATCH 09/23] lxd/storage/drivers/utils: Minor tweak to copyDevice
error message
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/utils.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/utils.go b/lxd/storage/drivers/utils.go
index a0752f7416..fdd1bfb29b 100644
--- a/lxd/storage/drivers/utils.go
+++ b/lxd/storage/drivers/utils.go
@@ -588,7 +588,7 @@ func copyDevice(inputPath, outputPath string) error {
to, err := os.OpenFile(outputPath, os.O_WRONLY, 0)
if err != nil {
- return errors.Wrapf(err, "Error opening file writing %q", outputPath)
+ return errors.Wrapf(err, "Error opening file for writing %q", outputPath)
}
defer to.Close()
From 1411d47d1ce6594ee719559c6c02bce31a2bc376 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:52:16 +0000
Subject: [PATCH 10/23] lxd/stroage/drivers/generic: Tweak error message of
genericCreateVolumeFromMigration
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/generic.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/generic.go b/lxd/storage/drivers/generic.go
index 02eb795e4d..ca18a04fc1 100644
--- a/lxd/storage/drivers/generic.go
+++ b/lxd/storage/drivers/generic.go
@@ -190,7 +190,7 @@ func genericCreateVolumeFromMigration(d Driver, initVolume func(vol Volume) (fun
to, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0)
if err != nil {
- return errors.Wrapf(err, "Error opening file writing %q", path)
+ return errors.Wrapf(err, "Error opening file for writing %q", path)
}
defer to.Close()
From 56d86884c1760187ae5ef77cd0b2b3a96f3f0c3c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 17 Mar 2020 15:52:54 +0000
Subject: [PATCH 11/23] lxd/storage/drivers/generic/vfs: Switches
genericVFSBackupVolume to tar writer
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/generic_vfs.go | 80 ++++++++++++++++++------------
1 file changed, 48 insertions(+), 32 deletions(-)
diff --git a/lxd/storage/drivers/generic_vfs.go b/lxd/storage/drivers/generic_vfs.go
index 93a41e74c3..076e1d9361 100644
--- a/lxd/storage/drivers/generic_vfs.go
+++ b/lxd/storage/drivers/generic_vfs.go
@@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "strings"
"github.com/pkg/errors"
@@ -15,6 +16,7 @@ import (
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/ioprogress"
log "github.com/lxc/lxd/shared/log15"
)
@@ -274,16 +276,50 @@ func genericVFSGetVolumeDiskPath(vol Volume) (string, error) {
}
// genericVFSBackupVolume is a generic BackupVolume implementation for VFS-only drivers.
-func genericVFSBackupVolume(d Driver, vol Volume, targetPath string, snapshots bool, op *operations.Operation) error {
- bwlimit := d.Config()["rsync.bwlimit"]
-
+func genericVFSBackupVolume(d Driver, vol Volume, tarWriter *instancewriter.InstanceTarWriter, snapshots bool, op *operations.Operation) error {
// Backups only implemented for containers currently.
if vol.volType != VolumeTypeContainer {
return ErrNotImplemented
}
+
+ // Define a function that can copy a volume into the backup target location.
+ backupVolume := func(v Volume, prefix string) error {
+ return v.MountTask(func(mountPath string, op *operations.Operation) error {
+ // Reset hard link cache as we are copying a new volume (instance or snapshot).
+ tarWriter.ResetHardLinkMap()
+
+ writeToTar := func(srcPath string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+
+ name := filepath.Join(prefix, strings.TrimPrefix(srcPath, mountPath))
+
+ // tomp TODO check the tar arguments for exclusions in old implementation.
+ err = tarWriter.WriteFile(name, srcPath, fi)
+ if err != nil {
+ return errors.Wrapf(err, "Error adding %q as %q to tarball", srcPath, name)
+ }
+
+ return nil
+ }
+
+ d.Logger().Debug("Copying container filesystem volume", log.Ctx{"sourcePath": mountPath, "prefix": prefix})
+ err := filepath.Walk(mountPath, writeToTar)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }, op)
+ }
+
// Handle snapshots.
if snapshots {
- snapshotsPath := filepath.Join(targetPath, "snapshots")
+ snapshotsPrefix := "backup/snapshots"
+ if vol.IsVMBlock() {
+ snapshotsPrefix = "backup/virtual-machine-snapshots"
+ }
// List the snapshots.
snapshots, err := vol.Snapshots(op)
@@ -291,43 +327,23 @@ func genericVFSBackupVolume(d Driver, vol Volume, targetPath string, snapshots b
return err
}
- // Create the snapshot path.
- if len(snapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return errors.Wrapf(err, "Failed to create directory '%s'", snapshotsPath)
- }
- }
-
for _, snapshot := range snapshots {
_, snapName, _ := shared.InstanceGetParentAndSnapshotName(snapshot.Name())
- target := filepath.Join(snapshotsPath, snapName)
-
- // Copy the snapshot.
- err = snapshot.MountTask(func(mountPath string, op *operations.Operation) error {
- _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
- if err != nil {
- return err
- }
-
- return nil
- }, op)
+ prefix := filepath.Join(snapshotsPrefix, snapName)
+ err := backupVolume(snapshot, prefix)
if err != nil {
return err
}
}
}
- // Copy the parent volume itself.
- target := filepath.Join(targetPath, "container")
- err := vol.MountTask(func(mountPath string, op *operations.Operation) error {
- _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
- if err != nil {
- return err
- }
+ // Copy the main volume itself.
+ prefix := "backup/container"
+ if vol.IsVMBlock() {
+ prefix = "backup/virtual-machine"
+ }
- return nil
- }, op)
+ err := backupVolume(vol, prefix)
if err != nil {
return err
}
From 46bbd11eb19969a6ecfdfaaecee552f861fa6a1c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 13:58:22 +0000
Subject: [PATCH 12/23] lxd/images: Fixes unhandled error
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/images.go | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/lxd/images.go b/lxd/images.go
index 0941eac153..6c11850b43 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -137,7 +137,6 @@ func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
if err != nil {
return err
}
-
} else {
args := []string{"-c"}
if shared.StringInSlice(compress, reproducible) {
@@ -147,7 +146,10 @@ func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
cmd := exec.Command(compress, args...)
cmd.Stdin = infile
cmd.Stdout = outfile
- cmd.Run()
+ err := cmd.Run()
+ if err != nil {
+ return err
+ }
}
return nil
From 7aecc039bfb765cf0eb2af86db159f30c9baa751 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:00:43 +0000
Subject: [PATCH 13/23] lxd/storage/backend/lxd: Adds tarWriter to
BackupInstance function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/backend_lxd.go | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lxd/storage/backend_lxd.go b/lxd/storage/backend_lxd.go
index 85ca108c03..8b2403d9c0 100644
--- a/lxd/storage/backend_lxd.go
+++ b/lxd/storage/backend_lxd.go
@@ -25,6 +25,7 @@ import (
"github.com/lxc/lxd/lxd/storage/memorypipe"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/ioprogress"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
@@ -1397,8 +1398,8 @@ func (b *lxdBackend) MigrateInstance(inst instance.Instance, conn io.ReadWriteCl
}
// BackupInstance creates an instance backup.
-func (b *lxdBackend) BackupInstance(inst instance.Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
- logger := logging.AddContext(b.logger, log.Ctx{"project": inst.Project(), "instance": inst.Name(), "targetPath": targetPath, "optimized": optimized, "snapshots": snapshots})
+func (b *lxdBackend) BackupInstance(inst instance.Instance, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
+ logger := logging.AddContext(b.logger, log.Ctx{"project": inst.Project(), "instance": inst.Name(), "optimized": optimized, "snapshots": snapshots})
logger.Debug("BackupInstance started")
defer logger.Debug("BackupInstance finished")
@@ -1425,7 +1426,7 @@ func (b *lxdBackend) BackupInstance(inst instance.Instance, targetPath string, o
}
vol := b.newVolume(volType, contentType, volStorageName, rootDiskConf)
- err = b.driver.BackupVolume(vol, targetPath, optimized, snapshots, op)
+ err = b.driver.BackupVolume(vol, tarWriter, optimized, snapshots, op)
if err != nil {
return err
}
From 5826da7416d346c0d77021ff77fbdce9abbaac5c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:01:43 +0000
Subject: [PATCH 14/23] lxd/storage/backend/mock: Adds tarWriter to
BackupInstance function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/backend_mock.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/storage/backend_mock.go b/lxd/storage/backend_mock.go
index 9753ddbb39..b5dc13083d 100644
--- a/lxd/storage/backend_mock.go
+++ b/lxd/storage/backend_mock.go
@@ -11,6 +11,7 @@ import (
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/storage/drivers"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/logger"
)
@@ -114,7 +115,7 @@ func (b *mockBackend) RefreshInstance(i instance.Instance, src instance.Instance
return nil
}
-func (b *mockBackend) BackupInstance(inst instance.Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+func (b *mockBackend) BackupInstance(inst instance.Instance, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
return nil
}
From 2437483a13dc75cbb91be011c8b298674ec7465d Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:02:43 +0000
Subject: [PATCH 15/23] lxd/storage/drivers/driver/ceph/volumes: Adds tarWriter
arg to BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_ceph_volumes.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lxd/storage/drivers/driver_ceph_volumes.go b/lxd/storage/drivers/driver_ceph_volumes.go
index b3b2e507ca..f416d8d2e9 100644
--- a/lxd/storage/drivers/driver_ceph_volumes.go
+++ b/lxd/storage/drivers/driver_ceph_volumes.go
@@ -19,6 +19,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/revert"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/ioprogress"
"github.com/lxc/lxd/shared/units"
)
@@ -1012,8 +1013,8 @@ func (d *ceph) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *mi
}
// BackupVolume creates an exported version of a volume.
-func (d *ceph) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
- return genericVFSBackupVolume(d, vol, targetPath, snapshots, op)
+func (d *ceph) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
+ return genericVFSBackupVolume(d, vol, tarWriter, snapshots, op)
}
// CreateVolumeSnapshot creates a snapshot of a volume.
From 4a04501f83bd1fc57ef3fc7837c04110901eff7a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:03:22 +0000
Subject: [PATCH 16/23] lxd/storage/drivers/driver/cephfs/volumes: Adds
tarWriter arg to BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_cephfs_volumes.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/driver_cephfs_volumes.go b/lxd/storage/drivers/driver_cephfs_volumes.go
index 5b384c0321..0e5b720c62 100644
--- a/lxd/storage/drivers/driver_cephfs_volumes.go
+++ b/lxd/storage/drivers/driver_cephfs_volumes.go
@@ -13,6 +13,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/rsync"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/ioprogress"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/units"
@@ -433,7 +434,7 @@ func (d *cephfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *
}
// BackupVolume creates an exported version of a volume.
-func (d *cephfs) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+func (d *cephfs) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
return ErrNotImplemented
}
From beca293b51ba4e39dd941dc3f1c901f027474a28 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:03:58 +0000
Subject: [PATCH 17/23] lxd/storage/drivers/driver/dir/volumes: Adds tarWriter
arg to BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_dir_volumes.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lxd/storage/drivers/driver_dir_volumes.go b/lxd/storage/drivers/driver_dir_volumes.go
index 5160ffe8bd..bb2ca165ba 100644
--- a/lxd/storage/drivers/driver_dir_volumes.go
+++ b/lxd/storage/drivers/driver_dir_volumes.go
@@ -13,6 +13,7 @@ import (
"github.com/lxc/lxd/lxd/rsync"
"github.com/lxc/lxd/lxd/storage/quota"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/instancewriter"
log "github.com/lxc/lxd/shared/log15"
)
@@ -303,8 +304,8 @@ func (d *dir) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *mig
// BackupVolume copies a volume (and optionally its snapshots) to a specified target path.
// This driver does not support optimized backups.
-func (d *dir) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
- return genericVFSBackupVolume(d, vol, targetPath, snapshots, op)
+func (d *dir) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
+ return genericVFSBackupVolume(d, vol, tarWriter, snapshots, op)
}
// CreateVolumeSnapshot creates a snapshot of a volume.
From 91b5f7a065d6d48fd9906b6ea3180e277e3b001f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:04:24 +0000
Subject: [PATCH 18/23] lxd/storage/drivers/driver/lvm/volumes: Adds tarWriter
arg to BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_lvm_volumes.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lxd/storage/drivers/driver_lvm_volumes.go b/lxd/storage/drivers/driver_lvm_volumes.go
index 9eadf49623..6527de2364 100644
--- a/lxd/storage/drivers/driver_lvm_volumes.go
+++ b/lxd/storage/drivers/driver_lvm_volumes.go
@@ -18,6 +18,7 @@ import (
"github.com/lxc/lxd/lxd/revert"
"github.com/lxc/lxd/lxd/rsync"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/instancewriter"
log "github.com/lxc/lxd/shared/log15"
)
@@ -529,8 +530,8 @@ func (d *lvm) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *mig
// BackupVolume copies a volume (and optionally its snapshots) to a specified target path.
// This driver does not support optimized backups.
-func (d *lvm) BackupVolume(vol Volume, targetPath string, _, snapshots bool, op *operations.Operation) error {
- return genericVFSBackupVolume(d, vol, targetPath, snapshots, op)
+func (d *lvm) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, _, snapshots bool, op *operations.Operation) error {
+ return genericVFSBackupVolume(d, vol, tarWriter, snapshots, op)
}
// CreateVolumeSnapshot creates a snapshot of a volume.
From 05d3cafd53b6a28de6d0df91389acf888bcf94bc Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:04:53 +0000
Subject: [PATCH 19/23] lxd/storage/drivers/drivers/mock: Adds tarWriter arg to
BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/drivers_mock.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/drivers_mock.go b/lxd/storage/drivers/drivers_mock.go
index b3b75cd682..b69a6edc5e 100644
--- a/lxd/storage/drivers/drivers_mock.go
+++ b/lxd/storage/drivers/drivers_mock.go
@@ -6,6 +6,7 @@ import (
"github.com/lxc/lxd/lxd/migration"
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/instancewriter"
)
type mock struct {
@@ -163,7 +164,7 @@ func (d *mock) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *mi
// BackupVolume copies a volume (and optionally its snapshots) to a specified target path.
// This driver does not support optimized backups.
-func (d *mock) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+func (d *mock) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
return nil
}
From c0aed5661fb60f4c6511f14d0c009a3ba3b432e9 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:07:38 +0000
Subject: [PATCH 20/23] lxd/storage/drivers/interface: Adds tarWriter arg to
BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/interface.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/interface.go b/lxd/storage/drivers/interface.go
index 5d57b5fcae..2dd1b580e0 100644
--- a/lxd/storage/drivers/interface.go
+++ b/lxd/storage/drivers/interface.go
@@ -7,6 +7,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/logger"
)
@@ -82,6 +83,6 @@ type Driver interface {
CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error
// Backup.
- BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error
+ BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error
CreateVolumeFromBackup(vol Volume, snapshots []string, srcData io.ReadSeeker, optimizedStorage bool, op *operations.Operation) (func(vol Volume) error, func(), error)
}
From 10d7d3db5fb8fd6511c7d6793dbe3f3acd66682e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:08:00 +0000
Subject: [PATCH 21/23] lxd/storage/pool/interace: Adds tarWriter arg to
BackupInstance
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/pool_interface.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/storage/pool_interface.go b/lxd/storage/pool_interface.go
index cc2d14d034..bca6c57fd2 100644
--- a/lxd/storage/pool_interface.go
+++ b/lxd/storage/pool_interface.go
@@ -10,6 +10,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/storage/drivers"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/instancewriter"
)
// Pool represents a LXD storage pool.
@@ -42,7 +43,7 @@ type Pool interface {
MigrateInstance(inst instance.Instance, conn io.ReadWriteCloser, args *migration.VolumeSourceArgs, op *operations.Operation) error
RefreshInstance(inst instance.Instance, src instance.Instance, srcSnapshots []instance.Instance, op *operations.Operation) error
- BackupInstance(inst instance.Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error
+ BackupInstance(inst instance.Instance, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error
GetInstanceUsage(inst instance.Instance) (int64, error)
SetInstanceQuota(inst instance.Instance, size string, op *operations.Operation) error
From 9d817035a48f6945310ff84996144971176387ec Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:10:36 +0000
Subject: [PATCH 22/23] lxd/storage/drivers/driver/btrfs/volumes: Adds
tarWriter arg to BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_volumes.go | 161 ++++++++++----------
1 file changed, 82 insertions(+), 79 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index ee8994b1fb..28cb3dc7a6 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -14,6 +14,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/revert"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/ioprogress"
"github.com/lxc/lxd/shared/units"
)
@@ -618,108 +619,110 @@ func (d *btrfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *m
// BackupVolume copies a volume (and optionally its snapshots) to a specified target path.
// This driver does not support optimized backups.
-func (d *btrfs) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+func (d *btrfs) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
// Handle the non-optimized tarballs through the generic packer.
if !optimized {
- return genericVFSBackupVolume(d, vol, targetPath, snapshots, op)
+ return genericVFSBackupVolume(d, vol, tarWriter, snapshots, op)
}
- // Handle the optimized tarballs.
- sendToFile := func(path string, parent string, file string) error {
- // Prepare btrfs send arguments.
- args := []string{"send"}
- if parent != "" {
- args = append(args, "-p", parent)
- }
- args = append(args, path)
-
- // Create the file.
- fd, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644)
- if err != nil {
- return errors.Wrapf(err, "Failed to open '%s'", file)
- }
- defer fd.Close()
-
- // Write the subvolume to the file.
- err = shared.RunCommandWithFds(nil, fd, "btrfs", args...)
- if err != nil {
- return err
- }
+ /*
+ // Handle the optimized tarballs.
+ sendToFile := func(path string, parent string, file string) error {
+ // Prepare btrfs send arguments.
+ args := []string{"send"}
+ if parent != "" {
+ args = append(args, "-p", parent)
+ }
+ args = append(args, path)
- return nil
- }
+ // Create the file.
+ fd, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return errors.Wrapf(err, "Failed to open '%s'", file)
+ }
+ defer fd.Close()
- // Handle snapshots.
- finalParent := ""
- if snapshots {
- snapshotsPath := fmt.Sprintf("%s/snapshots", targetPath)
+ // Write the subvolume to the file.
+ err = shared.RunCommandWithFds(nil, fd, "btrfs", args...)
+ if err != nil {
+ return err
+ }
- // Retrieve the snapshots.
- volSnapshots, err := d.VolumeSnapshots(vol, op)
- if err != nil {
- return err
+ return nil
}
- // Create the snapshot path.
- if len(volSnapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
+ // Handle snapshots.
+ finalParent := ""
+ if snapshots {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", targetPath)
+
+ // Retrieve the snapshots.
+ volSnapshots, err := d.VolumeSnapshots(vol, op)
if err != nil {
- return errors.Wrapf(err, "Failed to create directory '%s'", snapshotsPath)
+ return err
}
- }
-
- for i, snap := range volSnapshots {
- fullSnapshotName := GetSnapshotVolumeName(vol.name, snap)
- // Figure out parent and current subvolumes.
- parent := ""
- if i > 0 {
- parent = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSnapshots[i-1]))
+ // Create the snapshot path.
+ if len(volSnapshots) > 0 {
+ err = os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return errors.Wrapf(err, "Failed to create directory '%s'", snapshotsPath)
+ }
}
- cur := GetVolumeMountPath(d.name, vol.volType, fullSnapshotName)
+ for i, snap := range volSnapshots {
+ fullSnapshotName := GetSnapshotVolumeName(vol.name, snap)
- // Make a binary btrfs backup.
- target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snap)
+ // Figure out parent and current subvolumes.
+ parent := ""
+ if i > 0 {
+ parent = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSnapshots[i-1]))
+ }
- err := sendToFile(cur, parent, target)
- if err != nil {
- return err
- }
+ cur := GetVolumeMountPath(d.name, vol.volType, fullSnapshotName)
+
+ // Make a binary btrfs backup.
+ target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snap)
+
+ err := sendToFile(cur, parent, target)
+ if err != nil {
+ return err
+ }
- finalParent = cur
+ finalParent = cur
+ }
}
- }
- // Make a temporary copy of the container.
- sourceVolume := vol.MountPath()
- containersPath := GetVolumeMountPath(d.name, vol.volType, "")
+ // Make a temporary copy of the container.
+ sourceVolume := vol.MountPath()
+ containersPath := GetVolumeMountPath(d.name, vol.volType, "")
- tmpContainerMntPoint, err := ioutil.TempDir(containersPath, "backup.")
- if err != nil {
- return errors.Wrapf(err, "Failed to create temporary directory under '%s'", containersPath)
- }
- defer os.RemoveAll(tmpContainerMntPoint)
+ tmpContainerMntPoint, err := ioutil.TempDir(containersPath, "backup.")
+ if err != nil {
+ return errors.Wrapf(err, "Failed to create temporary directory under '%s'", containersPath)
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
- err = os.Chmod(tmpContainerMntPoint, 0100)
- if err != nil {
- return errors.Wrapf(err, "Failed to chmod '%s'", tmpContainerMntPoint)
- }
+ err = os.Chmod(tmpContainerMntPoint, 0100)
+ if err != nil {
+ return errors.Wrapf(err, "Failed to chmod '%s'", tmpContainerMntPoint)
+ }
- // Create the read-only snapshot.
- targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
- err = d.snapshotSubvolume(sourceVolume, targetVolume, true, true)
- if err != nil {
- return err
- }
- defer d.deleteSubvolume(targetVolume, true)
+ // Create the read-only snapshot.
+ targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
+ err = d.snapshotSubvolume(sourceVolume, targetVolume, true, true)
+ if err != nil {
+ return err
+ }
+ defer d.deleteSubvolume(targetVolume, true)
- // Dump the container to a file.
- fsDump := fmt.Sprintf("%s/container.bin", targetPath)
- err = sendToFile(targetVolume, finalParent, fsDump)
- if err != nil {
- return err
- }
+ // Dump the container to a file.
+ fsDump := fmt.Sprintf("%s/container.bin", targetPath)
+ err = sendToFile(targetVolume, finalParent, fsDump)
+ if err != nil {
+ return err
+ }
+ */
return nil
}
From f9d5850f153fd4786c8e5d4be560a686f92a1dbf Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 19 Mar 2020 14:11:01 +0000
Subject: [PATCH 23/23] lxd/storage/drivers/driver/zfs/volumes: Adds tarWriter
arg to BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_zfs_volumes.go | 136 +++++++++++-----------
1 file changed, 70 insertions(+), 66 deletions(-)
diff --git a/lxd/storage/drivers/driver_zfs_volumes.go b/lxd/storage/drivers/driver_zfs_volumes.go
index 7b76cbed60..a2097ad704 100644
--- a/lxd/storage/drivers/driver_zfs_volumes.go
+++ b/lxd/storage/drivers/driver_zfs_volumes.go
@@ -19,6 +19,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
"github.com/lxc/lxd/lxd/revert"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/instancewriter"
"github.com/lxc/lxd/shared/ioprogress"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/units"
@@ -1144,92 +1145,95 @@ func (d *zfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *mig
}
// BackupVolume creates an exported version of a volume.
-func (d *zfs) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+func (d *zfs) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, op *operations.Operation) error {
// Handle the non-optimized tarballs through the generic packer.
if !optimized {
- return genericVFSBackupVolume(d, vol, targetPath, snapshots, op)
+ return genericVFSBackupVolume(d, vol, tarWriter, snapshots, op)
}
- // Handle the optimized tarballs.
- sendToFile := func(path string, parent string, file string) error {
- // Prepare zfs send arguments.
- args := []string{"send"}
- if parent != "" {
- args = append(args, "-i", parent)
- }
- args = append(args, path)
-
- // Create the file.
- fd, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644)
- if err != nil {
- return errors.Wrapf(err, "Failed to open '%s'", file)
- }
- defer fd.Close()
+ /*
- // Write the subvolume to the file.
- err = shared.RunCommandWithFds(nil, fd, "zfs", args...)
- if err != nil {
- return err
- }
+ // Handle the optimized tarballs.
+ sendToFile := func(path string, parent string, file string) error {
+ // Prepare zfs send arguments.
+ args := []string{"send"}
+ if parent != "" {
+ args = append(args, "-i", parent)
+ }
+ args = append(args, path)
- return nil
- }
+ // Create the file.
+ fd, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return errors.Wrapf(err, "Failed to open '%s'", file)
+ }
+ defer fd.Close()
- // Handle snapshots.
- finalParent := ""
- if snapshots {
- snapshotsPath := fmt.Sprintf("%s/snapshots", targetPath)
+ // Write the subvolume to the file.
+ err = shared.RunCommandWithFds(nil, fd, "zfs", args...)
+ if err != nil {
+ return err
+ }
- // Retrieve the snapshots.
- volSnapshots, err := d.VolumeSnapshots(vol, op)
- if err != nil {
- return err
+ return nil
}
- // Create the snapshot path.
- if len(volSnapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
+ // Handle snapshots.
+ finalParent := ""
+ if snapshots {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", targetPath)
+
+ // Retrieve the snapshots.
+ volSnapshots, err := d.VolumeSnapshots(vol, op)
if err != nil {
- return errors.Wrapf(err, "Failed to create directory '%s'", snapshotsPath)
+ return err
}
- }
- for i, snapName := range volSnapshots {
- snapshot, _ := vol.NewSnapshot(snapName)
-
- // Figure out parent and current subvolumes.
- parent := ""
- if i > 0 {
- oldSnapshot, _ := vol.NewSnapshot(volSnapshots[i-1])
- parent = d.dataset(oldSnapshot, false)
+ // Create the snapshot path.
+ if len(volSnapshots) > 0 {
+ err = os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return errors.Wrapf(err, "Failed to create directory '%s'", snapshotsPath)
+ }
}
- // Make a binary zfs backup.
- target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snapName)
+ for i, snapName := range volSnapshots {
+ snapshot, _ := vol.NewSnapshot(snapName)
- err := sendToFile(d.dataset(snapshot, false), parent, target)
- if err != nil {
- return err
- }
+ // Figure out parent and current subvolumes.
+ parent := ""
+ if i > 0 {
+ oldSnapshot, _ := vol.NewSnapshot(volSnapshots[i-1])
+ parent = d.dataset(oldSnapshot, false)
+ }
- finalParent = d.dataset(snapshot, false)
+ // Make a binary zfs backup.
+ target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snapName)
+
+ err := sendToFile(d.dataset(snapshot, false), parent, target)
+ if err != nil {
+ return err
+ }
+
+ finalParent = d.dataset(snapshot, false)
+ }
}
- }
- // Create a temporary read-only snapshot.
- srcSnapshot := fmt.Sprintf("%s at backup-%s", d.dataset(vol, false), uuid.NewRandom().String())
- _, err := shared.RunCommand("zfs", "snapshot", srcSnapshot)
- if err != nil {
- return err
- }
- defer shared.RunCommand("zfs", "destroy", srcSnapshot)
+ // Create a temporary read-only snapshot.
+ srcSnapshot := fmt.Sprintf("%s at backup-%s", d.dataset(vol, false), uuid.NewRandom().String())
+ _, err := shared.RunCommand("zfs", "snapshot", srcSnapshot)
+ if err != nil {
+ return err
+ }
+ defer shared.RunCommand("zfs", "destroy", srcSnapshot)
- // Dump the container to a file.
- fsDump := fmt.Sprintf("%s/container.bin", targetPath)
- err = sendToFile(srcSnapshot, finalParent, fsDump)
- if err != nil {
- return err
- }
+ // Dump the container to a file.
+ fsDump := fmt.Sprintf("%s/container.bin", targetPath)
+ err = sendToFile(srcSnapshot, finalParent, fsDump)
+ if err != nil {
+ return err
+ }
+ */
return nil
}
More information about the lxc-devel
mailing list