[lxc-devel] [lxd/master] Backup: Simplifies squashfs restore handling
tomponline on Github
lxc-bot at linuxcontainers.org
Wed Nov 27 16:16:45 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 346 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191127/1e77dca1/attachment-0001.bin>
-------------- next part --------------
From 272104730cd03f8263074f14bd9cb4b94d0c571c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 12:25:05 +0000
Subject: [PATCH 1/8] lxd/backup/backup/instance/config: Adds instance config
backup.yml tools
Moved from main package and cleaned up.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup/backup_instance_config.go | 122 +++++++++++++++++++++++++++
1 file changed, 122 insertions(+)
create mode 100644 lxd/backup/backup_instance_config.go
diff --git a/lxd/backup/backup_instance_config.go b/lxd/backup/backup_instance_config.go
new file mode 100644
index 0000000000..a871ad0a7e
--- /dev/null
+++ b/lxd/backup/backup_instance_config.go
@@ -0,0 +1,122 @@
+package backup
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+
+ "gopkg.in/yaml.v2"
+
+ "github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/lxd/project"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+)
+
+// InstanceConfig represents the config of an instance that can be stored in a backup.yaml file.
+type InstanceConfig struct {
+ Container *api.Instance `yaml:"container"`
+ Snapshots []*api.InstanceSnapshot `yaml:"snapshots"`
+ Pool *api.StoragePool `yaml:"pool"`
+ Volume *api.StorageVolume `yaml:"volume"`
+}
+
+// ParseInstanceConfigYamlFile decodes the yaml file at path specified into an InstanceConfig.
+func ParseInstanceConfigYamlFile(path string) (*InstanceConfig, error) {
+ data, err := ioutil.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+
+ backup := InstanceConfig{}
+ if err := yaml.Unmarshal(data, &backup); err != nil {
+ return nil, err
+ }
+
+ return &backup, nil
+}
+
+// updateRootDevicePool updates the root disk device in the supplied list of devices to the pool
+// specified. Returns true if a root disk device has been found and updated otherwise false.
+func updateRootDevicePool(devices map[string]map[string]string, poolName string) bool {
+ if devices != nil {
+ devName, _, err := shared.GetRootDiskDevice(devices)
+ if err == nil {
+ devices[devName]["pool"] = poolName
+ return true
+ }
+ }
+
+ return false
+}
+
+// UpdateInstanceConfigStoragePool changes the pool information in the backup.yaml to the pool
+// specified in b.Pool.
+func UpdateInstanceConfigStoragePool(c *db.Cluster, b Info) error {
+ // Load the storage pool.
+ _, pool, err := c.StoragePoolGet(b.Pool)
+ if err != nil {
+ return err
+ }
+
+ f := func(path string) error {
+ // Read in the backup.yaml file.
+ backup, err := ParseInstanceConfigYamlFile(path)
+ if err != nil {
+ return err
+ }
+
+ rootDiskDeviceFound := false
+
+ // Change the pool in the backup.yaml.
+ backup.Pool = pool
+
+ if updateRootDevicePool(backup.Container.Devices, pool.Name) {
+ rootDiskDeviceFound = true
+ }
+
+ if updateRootDevicePool(backup.Container.ExpandedDevices, pool.Name) {
+ rootDiskDeviceFound = true
+ }
+
+ for _, snapshot := range backup.Snapshots {
+ updateRootDevicePool(snapshot.Devices, pool.Name)
+ updateRootDevicePool(snapshot.ExpandedDevices, pool.Name)
+ }
+
+ if !rootDiskDeviceFound {
+ return fmt.Errorf("No root device could be found")
+ }
+
+ file, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ data, err := yaml.Marshal(&backup)
+ if err != nil {
+ return err
+ }
+
+ _, err = file.Write(data)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ err = f(shared.VarPath("storage-pools", pool.Name, "containers", project.Prefix(b.Project, b.Name), "backup.yaml"))
+ if err != nil {
+ return err
+ }
+
+ for _, snap := range b.Snapshots {
+ err = f(shared.VarPath("storage-pools", pool.Name, "containers-snapshots", project.Prefix(b.Project, b.Name), snap, "backup.yaml"))
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
From d8a2ef2d0523113ef24d741162ebb0f4bdef5a97 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 12:26:08 +0000
Subject: [PATCH 2/8] lxd/api/internal: Removes slurpBackupFile and switches to
backup.ParseInstanceConfigYamlFile
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/api_internal.go | 20 ++------------------
1 file changed, 2 insertions(+), 18 deletions(-)
diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index 957b0107b7..38ee0da172 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -4,7 +4,6 @@ import (
"database/sql"
"encoding/json"
"fmt"
- "io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -15,8 +14,8 @@ import (
"github.com/gorilla/mux"
"github.com/pkg/errors"
- "gopkg.in/yaml.v2"
+ "github.com/lxc/lxd/lxd/backup"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/db/cluster"
"github.com/lxc/lxd/lxd/db/node"
@@ -390,21 +389,6 @@ func internalSQLExec(tx *sql.Tx, query string, result *internalSQLResult) error
return nil
}
-func slurpBackupFile(path string) (*backupFile, error) {
- data, err := ioutil.ReadFile(path)
- if err != nil {
- return nil, err
- }
-
- backup := backupFile{}
-
- if err := yaml.Unmarshal(data, &backup); err != nil {
- return nil, err
- }
-
- return &backup, nil
-}
-
type internalImportPost struct {
Name string `json:"name" yaml:"name"`
Force bool `json:"force" yaml:"force"`
@@ -476,7 +460,7 @@ func internalImport(d *Daemon, r *http.Request) response.Response {
// Read in the backup.yaml file.
backupYamlPath := filepath.Join(containerMntPoint, "backup.yaml")
- backup, err := slurpBackupFile(backupYamlPath)
+ backup, err := backup.ParseInstanceConfigYamlFile(backupYamlPath)
if err != nil {
return response.SmartError(err)
}
From 50837149c75edcc9fa90f5a86e50d2fe5dada67a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 12:26:38 +0000
Subject: [PATCH 3/8] lxd/backup: Removes backupFixStoragePool
Replaced by backup.UpdateInstanceConfigStoragePool
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup.go | 94 ---------------------------------------------------
1 file changed, 94 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index be3b24511b..15189bffd7 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -101,100 +101,6 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
return nil
}
-// fixBackupStoragePool changes the pool information in the backup.yaml. This
-// is done only if the provided pool doesn't exist. In this case, the pool of
-// the default profile will be used.
-func backupFixStoragePool(c *db.Cluster, b backup.Info, useDefaultPool bool) error {
- var poolName string
-
- if useDefaultPool {
- // Get the default profile
- _, profile, err := c.ProfileGet("default", "default")
- if err != nil {
- return err
- }
-
- _, v, err := shared.GetRootDiskDevice(profile.Devices)
- if err != nil {
- return err
- }
-
- poolName = v["pool"]
- } else {
- poolName = b.Pool
- }
-
- // Get the default's profile pool
- _, pool, err := c.StoragePoolGet(poolName)
- if err != nil {
- return err
- }
-
- f := func(path string) error {
- // Read in the backup.yaml file.
- backup, err := slurpBackupFile(path)
- if err != nil {
- return err
- }
-
- rootDiskDeviceFound := false
-
- // Change the pool in the backup.yaml
- backup.Pool = pool
- if backup.Container.Devices != nil {
- devName, _, err := shared.GetRootDiskDevice(backup.Container.Devices)
- if err == nil {
- backup.Container.Devices[devName]["pool"] = poolName
- rootDiskDeviceFound = true
- }
- }
-
- if backup.Container.ExpandedDevices != nil {
- devName, _, err := shared.GetRootDiskDevice(backup.Container.ExpandedDevices)
- if err == nil {
- backup.Container.ExpandedDevices[devName]["pool"] = poolName
- rootDiskDeviceFound = true
- }
- }
-
- if !rootDiskDeviceFound {
- return fmt.Errorf("No root device could be found")
- }
-
- file, err := os.Create(path)
- if err != nil {
- return err
- }
- defer file.Close()
-
- data, err := yaml.Marshal(&backup)
- if err != nil {
- return err
- }
-
- _, err = file.Write(data)
- if err != nil {
- return err
- }
-
- return nil
- }
-
- err = f(shared.VarPath("storage-pools", pool.Name, "containers", project.Prefix(b.Project, b.Name), "backup.yaml"))
- if err != nil {
- return err
- }
-
- for _, snap := range b.Snapshots {
- err = f(shared.VarPath("storage-pools", pool.Name, "containers-snapshots", project.Prefix(b.Project, b.Name), snap,
- "backup.yaml"))
- if err != nil {
- return err
- }
- }
- return nil
-}
-
func backupCreateTarball(s *state.State, path string, b backup.Backup, c instance.Instance) error {
// Create the index
pool, err := c.StoragePool()
From cfc105ae56a054510a2ead417cf71f46314e58ed Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 12:28:25 +0000
Subject: [PATCH 4/8] lxd/containers/post: Updates instanceCreateFromBackup
usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/containers_post.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/containers_post.go b/lxd/containers_post.go
index eae96c5597..c3440ce4fb 100644
--- a/lxd/containers_post.go
+++ b/lxd/containers_post.go
@@ -657,7 +657,7 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
// Dump tarball to storage.
f.Seek(0, 0)
- revertFunc, err := instanceCreateFromBackup(d.State(), *bInfo, f, pool != "")
+ revertFunc, err := instanceCreateFromBackup(d.State(), *bInfo, f)
if err != nil {
return errors.Wrap(err, "Create instance from backup")
}
From ed788bd0682a20aa1710b1358ad76fce614a33d9 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 15:43:27 +0000
Subject: [PATCH 5/8] lxd/container: Updates instanceCreateFromBackup signature
- Removes need for customBool variable and instead will just always update backup.yaml file to use current storage pool.
- This simplifies the function signature needed later for pool.CreateInstanceFromBackup().
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 27 ++++++++++-----------------
1 file changed, 10 insertions(+), 17 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index 125e5c4b8e..625c57ffd4 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -307,13 +307,10 @@ func instanceCreateAsEmpty(d *Daemon, args db.InstanceArgs) (instance.Instance,
// instanceCreateFromBackup imports a backup file to restore an instance. Returns a revert function
// that can be called to delete the files created during import if subsequent steps fail.
-func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadSeeker, customPool bool) (func(), error) {
- var fixBackupFile = false
- poolName := info.Pool
-
+func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadSeeker) (func(), error) {
// Check storage pool from index.yaml exists. If the storage pool in the index.yaml doesn't
- // exist, try and use the default profile's storage pool.
- _, _, err := s.Cluster.StoragePoolGet(poolName)
+ // exist, try and use the project's default profile storage pool.
+ _, _, err := s.Cluster.StoragePoolGet(info.Pool)
if errors.Cause(err) == db.ErrNoSuchObject {
// The backup is in binary format so we cannot alter the backup.yaml.
if info.HasBinaryFormat {
@@ -332,10 +329,7 @@ func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadS
}
// Use the default-profile's root pool.
- poolName = v["pool"]
-
- // Mark requirement to change the pool information in the backup.yaml file.
- fixBackupFile = true
+ info.Pool = v["pool"]
} else if err != nil {
return nil, err
}
@@ -385,7 +379,7 @@ func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadS
srcData = tarData
}
- pool, err := storagePoolInit(s, poolName)
+ pool, err := storagePoolInit(s, info.Pool)
if err != nil {
return nil, err
}
@@ -396,12 +390,11 @@ func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadS
return nil, err
}
- if fixBackupFile || customPool {
- // Update pool information in the backup.yaml file.
- err = backupFixStoragePool(s.Cluster, info, !customPool)
- if err != nil {
- return nil, err
- }
+ // Update pool information in the backup.yaml file.
+ // Requires the volume and snapshots be mounted from pool.ContainerBackupLoad().
+ err = backup.UpdateInstanceConfigStoragePool(s.Cluster, info)
+ if err != nil {
+ return nil, err
}
// Create revert function to remove the files created so far.
From a0032cd552845c9c02af7678d4fc51f2951a91e3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 16:12:29 +0000
Subject: [PATCH 6/8] lxd/backup/backup: Removes squashfs handling from GetInfo
Expects to always be passed an (optionally compressed) tarball now.
Squashfs files should be converted to a tarball before being passed to this function.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup/backup.go | 23 +----------------------
1 file changed, 1 insertion(+), 22 deletions(-)
diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go
index 421517d1f4..8f3fdc1067 100644
--- a/lxd/backup/backup.go
+++ b/lxd/backup/backup.go
@@ -4,7 +4,6 @@ import (
"archive/tar"
"fmt"
"io"
- "io/ioutil"
"os"
"os/exec"
"strings"
@@ -48,7 +47,7 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
// Extract
r.Seek(0, 0)
- _, algo, unpacker, err := shared.DetectCompressionFile(r)
+ _, _, unpacker, err := shared.DetectCompressionFile(r)
if err != nil {
return nil, err
}
@@ -59,26 +58,6 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
}
if len(unpacker) > 0 {
- if algo == ".squashfs" {
- // 'sqfs2tar' tool does not support reading from stdin. So
- // create a temporary file to write the compressed data and
- // pass it as program argument
- tempfile, err := ioutil.TempFile("", "lxd_decompress_")
- if err != nil {
- return nil, err
- }
- defer tempfile.Close()
- defer os.Remove(tempfile.Name())
-
- // Write compressed data
- _, err = io.Copy(tempfile, r)
- if err != nil {
- return nil, err
- }
-
- // Prepare to pass the temporary file as program argument
- unpacker = append(unpacker, tempfile.Name())
- }
cmd := exec.Command(unpacker[0], unpacker[1:]...)
cmd.Stdin = r
From 0cbcbc78827437ee36e9edb0f48bc0181d8f241c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 16:14:08 +0000
Subject: [PATCH 7/8] lxd/container: Removes squashfs handling from
instanceCreateBackup
Expects to always be passed an (optionally compressed) tarball now.
Squashfs files should be converted to a tarball before being passed to this function.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 41 +----------------------------------------
1 file changed, 1 insertion(+), 40 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index 625c57ffd4..0b94e16673 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -335,50 +334,12 @@ func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadS
}
// Find the compression algorithm.
- tarArgs, algo, decomArgs, err := shared.DetectCompressionFile(srcData)
+ tarArgs, _, _, err := shared.DetectCompressionFile(srcData)
if err != nil {
return nil, err
}
srcData.Seek(0, 0)
- if algo == ".squashfs" {
- // The 'sqfs2tar' tool does not support reading from stdin. So we need to create a
- // temporary file, write the compressed data to it and pass it as program argument.
- tempFile, err := ioutil.TempFile("", "lxd_decompress_")
- if err != nil {
- return nil, err
- }
- defer tempFile.Close()
- defer os.Remove(tempFile.Name())
-
- // Copy the compressed data stream to the temporart file.
- _, err = io.Copy(tempFile, srcData)
- if err != nil {
- return nil, err
- }
-
- // Pass the temporary file as program argument to the decompression command.
- decomArgs := append(decomArgs, tempFile.Name())
-
- // Create another temporary file to write the decompressed data.
- tarData, err := ioutil.TempFile("", "lxd_decompress_")
- if err != nil {
- return nil, err
- }
- defer tarData.Close()
- defer os.Remove(tarData.Name())
-
- // Decompress to tarData temporary file.
- err = shared.RunCommandWithFds(nil, tarData, decomArgs[0], decomArgs[1:]...)
- if err != nil {
- return nil, err
- }
- tarData.Seek(0, 0)
-
- // Set source data stream to newly created tar file handle.
- srcData = tarData
- }
-
pool, err := storagePoolInit(s, info.Pool)
if err != nil {
return nil, err
From dd16c276448c2e8cf125e349e71b601b5a06ae22 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 27 Nov 2019 16:14:54 +0000
Subject: [PATCH 8/8] lxd/containers/post: Moves backup restore squashfs
handling to createFromBackup
This avoids copying the uploaded backup file to multiple temporary files and simplifies the downstream logic.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/containers_post.go | 58 ++++++++++++++++++++++++++++++++++--------
1 file changed, 47 insertions(+), 11 deletions(-)
diff --git a/lxd/containers_post.go b/lxd/containers_post.go
index c3440ce4fb..632fe34a24 100644
--- a/lxd/containers_post.go
+++ b/lxd/containers_post.go
@@ -625,24 +625,59 @@ func createFromCopy(d *Daemon, project string, req *api.InstancesPost) response.
}
func createFromBackup(d *Daemon, project string, data io.Reader, pool string) response.Response {
- // Write the data to a temp file.
- f, err := ioutil.TempFile("", "lxd_backup_")
+ // Create temporary file to store uploaded backup data.
+ backupFile, err := ioutil.TempFile("", "lxd_backup_")
if err != nil {
return response.InternalError(err)
}
- defer os.Remove(f.Name())
+ defer os.Remove(backupFile.Name())
- _, err = io.Copy(f, data)
+ // Stream uploaded backup data into temporary file.
+ _, err = io.Copy(backupFile, data)
if err != nil {
- f.Close()
+ backupFile.Close()
return response.InternalError(err)
}
+ // Detect squashfs compression and convert to tarball.
+ backupFile.Seek(0, 0)
+ _, algo, decomArgs, err := shared.DetectCompressionFile(backupFile)
+ if err != nil {
+ backupFile.Close()
+ return response.InternalError(err)
+ }
+
+ if algo == ".squashfs" {
+ // Pass the temporary file as program argument to the decompression command.
+ decomArgs := append(decomArgs, backupFile.Name())
+
+ // Create temporary file to store the decompressed tarball in.
+ tarFile, err := ioutil.TempFile("", "lxd_decompress_")
+ if err != nil {
+ backupFile.Close()
+ return response.InternalError(err)
+ }
+ defer os.Remove(tarFile.Name())
+
+ // Decompress to tarData temporary file.
+ err = shared.RunCommandWithFds(nil, tarFile, decomArgs[0], decomArgs[1:]...)
+ if err != nil {
+ return response.InternalError(err)
+ }
+
+ // We don't need the original squashfs file anymore.
+ backupFile.Close()
+ os.Remove(backupFile.Name())
+
+ // Replace the backup file handle with the handle to the tar file.
+ backupFile = tarFile
+ }
+
// Parse the backup information.
- f.Seek(0, 0)
- bInfo, err := backup.GetInfo(f)
+ backupFile.Seek(0, 0)
+ bInfo, err := backup.GetInfo(backupFile)
if err != nil {
- f.Close()
+ backupFile.Close()
return response.BadRequest(err)
}
bInfo.Project = project
@@ -653,11 +688,11 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
}
run := func(op *operations.Operation) error {
- defer f.Close()
+ defer backupFile.Close()
// Dump tarball to storage.
- f.Seek(0, 0)
- revertFunc, err := instanceCreateFromBackup(d.State(), *bInfo, f)
+ backupFile.Seek(0, 0)
+ revertFunc, err := instanceCreateFromBackup(d.State(), *bInfo, backupFile)
if err != nil {
return errors.Wrap(err, "Create instance from backup")
}
@@ -712,6 +747,7 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
op, err := operations.OperationCreate(d.State(), project, operations.OperationClassTask, db.OperationBackupRestore,
resources, nil, run, nil, nil)
if err != nil {
+ backupFile.Close()
return response.InternalError(err)
}
More information about the lxc-devel
mailing list