[lxc-devel] [lxd/master] Storage backup import cleanup

tomponline on Github lxc-bot at linuxcontainers.org
Tue Nov 26 17:00:16 UTC 2019


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 765 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191126/1ca19878/attachment.bin>
-------------- next part --------------
From 329c4e2ad3059c17c6c0cdf859d94acf58138786 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 26 Nov 2019 16:53:25 +0000
Subject: [PATCH 1/4] shared/archive/linux: Adds some explanation to
 DetectCompressionFile

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 shared/archive_linux.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/shared/archive_linux.go b/shared/archive_linux.go
index a77a09b70a..9a07136577 100644
--- a/shared/archive_linux.go
+++ b/shared/archive_linux.go
@@ -23,6 +23,9 @@ func DetectCompression(fname string) ([]string, string, []string, error) {
 	return DetectCompressionFile(f)
 }
 
+// DetectCompressionFile detects the compression type of a file and returns the tar arguments needed
+// to unpack the file, compression type (in the form of a file extension), and the command needed
+// to decompress the file to an uncompressed tarball.
 func DetectCompressionFile(f io.ReadSeeker) ([]string, string, []string, error) {
 	// read header parts to detect compression method
 	// bz2 - 2 bytes, 'BZ' signature/magic number
@@ -50,8 +53,7 @@ func DetectCompressionFile(f io.ReadSeeker) ([]string, string, []string, error)
 	case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}):
 		return []string{"-xf"}, ".tar", []string{}, nil
 	case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}):
-		return []string{"-xf"}, ".squashfs",
-			[]string{"sqfs2tar", "--no-skip"}, nil
+		return []string{"-xf"}, ".squashfs", []string{"sqfs2tar", "--no-skip"}, nil
 	default:
 		return nil, "", nil, fmt.Errorf("Unsupported compression")
 	}

From 097bd67c51200665b6214425bbee03fb24df5825 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 26 Nov 2019 16:53:50 +0000
Subject: [PATCH 2/4] lxd/images: Adds more output detail when tar2sqfs fails
 in compressFile

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/images.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lxd/images.go b/lxd/images.go
index cf989c4011..c8f04132af 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -122,14 +122,13 @@ func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
 		defer os.Remove(tempfile.Name())
 
 		// Prepare 'tar2sqfs' arguments
-		args := []string{"tar2sqfs", "--no-skip", "--force",
-			"--compressor", "xz", tempfile.Name()}
+		args := []string{"tar2sqfs", "--no-skip", "--force", "--compressor", "xz", tempfile.Name()}
 		cmd = exec.Command(args[0], args[1:]...)
 		cmd.Stdin = infile
 
-		err = cmd.Run()
+		output, err := cmd.CombinedOutput()
 		if err != nil {
-			return err
+			return fmt.Errorf("tar2sqfs: %v (%v)", err, strings.TrimSpace(string(output)))
 		}
 		// Replay the result to outfile
 		tempfile.Seek(0, 0)

From 010b400f716a5b7d662095c912dd3b02c407ae1c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 26 Nov 2019 16:54:48 +0000
Subject: [PATCH 3/4] lxd/container: instanceCreateFromBackup restructure so as
 not to return storage

- Lays the groundwork to support both old and new storage layers.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/container.go | 100 +++++++++++++++++++++++------------------------
 1 file changed, 50 insertions(+), 50 deletions(-)

diff --git a/lxd/container.go b/lxd/container.go
index 1406e91ebf..125e5c4b8e 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -305,26 +305,22 @@ func instanceCreateAsEmpty(d *Daemon, args db.InstanceArgs) (instance.Instance,
 	return inst, nil
 }
 
-func containerCreateFromBackup(s *state.State, info backup.Info, data io.ReadSeeker,
-	customPool bool) (storage, error) {
-	var pool storage
+// instanceCreateFromBackup imports a backup file to restore an instance. Returns a revert function
+// that can be called to delete the files created during import if subsequent steps fail.
+func instanceCreateFromBackup(s *state.State, info backup.Info, srcData io.ReadSeeker, customPool bool) (func(), error) {
 	var fixBackupFile = false
+	poolName := info.Pool
 
-	// Get storage pool from index.yaml
-	pool, storageErr := storagePoolInit(s, info.Pool)
-	if storageErr != nil && errors.Cause(storageErr) != db.ErrNoSuchObject {
-		// Unexpected error
-		return nil, storageErr
-	}
-
-	if errors.Cause(storageErr) == db.ErrNoSuchObject {
-		// The pool doesn't exist, and the backup is in binary format so we
-		// cannot alter the backup.yaml.
+	// Check storage pool from index.yaml exists. If the storage pool in the index.yaml doesn't
+	// exist, try and use the default profile's storage pool.
+	_, _, err := s.Cluster.StoragePoolGet(poolName)
+	if errors.Cause(err) == db.ErrNoSuchObject {
+		// The backup is in binary format so we cannot alter the backup.yaml.
 		if info.HasBinaryFormat {
-			return nil, storageErr
+			return nil, err
 		}
 
-		// Get the default profile
+		// Get the default profile.
 		_, profile, err := s.Cluster.ProfileGet(info.Project, "default")
 		if err != nil {
 			return nil, errors.Wrap(err, "Failed to get default profile")
@@ -335,43 +331,42 @@ func containerCreateFromBackup(s *state.State, info backup.Info, data io.ReadSee
 			return nil, errors.Wrap(err, "Failed to get root disk device")
 		}
 
-		// Use the default-profile's root pool
-		pool, err = storagePoolInit(s, v["pool"])
-		if err != nil {
-			return nil, errors.Wrap(err, "Failed to initialize storage pool")
-		}
+		// Use the default-profile's root pool.
+		poolName = v["pool"]
 
+		// Mark requirement to change the pool information in the backup.yaml file.
 		fixBackupFile = true
+	} else if err != nil {
+		return nil, err
 	}
 
-	// Find the compression algorithm
-	tarArgs, algo, decomArgs, err := shared.DetectCompressionFile(data)
+	// Find the compression algorithm.
+	tarArgs, algo, decomArgs, err := shared.DetectCompressionFile(srcData)
 	if err != nil {
 		return nil, err
 	}
-	data.Seek(0, 0)
+	srcData.Seek(0, 0)
 
 	if algo == ".squashfs" {
-		// Create a temporary file. 'sqfs2tar' tool do not support reading
-		// from stdin. So write the compressed data to the temporary file
-		// and pass it as program argument
-		tempfile, err := ioutil.TempFile("", "lxd_decompress_")
+		// The 'sqfs2tar' tool does not support reading from stdin. So we need to create a
+		// temporary file, write the compressed data to it and pass it as program argument.
+		tempFile, err := ioutil.TempFile("", "lxd_decompress_")
 		if err != nil {
 			return nil, err
 		}
-		defer tempfile.Close()
-		defer os.Remove(tempfile.Name())
+		defer tempFile.Close()
+		defer os.Remove(tempFile.Name())
 
-		// Write the compressed data
-		_, err = io.Copy(tempfile, data)
+		// Copy the compressed data stream to the temporart file.
+		_, err = io.Copy(tempFile, srcData)
 		if err != nil {
 			return nil, err
 		}
 
-		// Prepare to pass the temporary file as program argument
-		decomArgs := append(decomArgs, tempfile.Name())
+		// Pass the temporary file as program argument to the decompression command.
+		decomArgs := append(decomArgs, tempFile.Name())
 
-		// Create another temporary file to write the decompressed data
+		// Create another temporary file to write the decompressed data.
 		tarData, err := ioutil.TempFile("", "lxd_decompress_")
 		if err != nil {
 			return nil, err
@@ -379,37 +374,42 @@ func containerCreateFromBackup(s *state.State, info backup.Info, data io.ReadSee
 		defer tarData.Close()
 		defer os.Remove(tarData.Name())
 
-		// Decompress to tarData temporary file
-		err = shared.RunCommandWithFds(nil, tarData,
-			decomArgs[0], decomArgs[1:]...)
+		// Decompress to tarData temporary file.
+		err = shared.RunCommandWithFds(nil, tarData, decomArgs[0], decomArgs[1:]...)
 		if err != nil {
 			return nil, err
 		}
 		tarData.Seek(0, 0)
 
-		// Unpack tarball from decompressed temporary file
-		err = pool.ContainerBackupLoad(info, tarData, tarArgs)
-		if err != nil {
-			return nil, err
-		}
+		// Set source data stream to newly created tar file handle.
+		srcData = tarData
+	}
 
-	} else {
-		// Unpack tarball
-		err = pool.ContainerBackupLoad(info, data, tarArgs)
-		if err != nil {
-			return nil, err
-		}
+	pool, err := storagePoolInit(s, poolName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unpack tarball from the source tar stream.
+	err = pool.ContainerBackupLoad(info, srcData, tarArgs)
+	if err != nil {
+		return nil, err
 	}
 
 	if fixBackupFile || customPool {
-		// Update the pool
+		// Update pool information in the backup.yaml file.
 		err = backupFixStoragePool(s.Cluster, info, !customPool)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	return pool, nil
+	// Create revert function to remove the files created so far.
+	revertFunc := func() {
+		pool.ContainerDelete(&containerLXC{name: info.Name, project: info.Project})
+	}
+
+	return revertFunc, nil
 }
 
 func containerCreateEmptySnapshot(s *state.State, args db.InstanceArgs) (instance.Instance, error) {

From 1be89b525fc12b9fd4199f80cd6c4eb4f3ea1015 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 26 Nov 2019 16:55:42 +0000
Subject: [PATCH 4/4] lxd/containers/post: Updates createFromBackup to not need
 storage returned from instanceCreateFromBackup

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/containers_post.go | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/lxd/containers_post.go b/lxd/containers_post.go
index 8de9aabbe1..eae96c5597 100644
--- a/lxd/containers_post.go
+++ b/lxd/containers_post.go
@@ -625,7 +625,7 @@ func createFromCopy(d *Daemon, project string, req *api.InstancesPost) response.
 }
 
 func createFromBackup(d *Daemon, project string, data io.Reader, pool string) response.Response {
-	// Write the data to a temp file
+	// Write the data to a temp file.
 	f, err := ioutil.TempFile("", "lxd_backup_")
 	if err != nil {
 		return response.InternalError(err)
@@ -638,7 +638,7 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
 		return response.InternalError(err)
 	}
 
-	// Parse the backup information
+	// Parse the backup information.
 	f.Seek(0, 0)
 	bInfo, err := backup.GetInfo(f)
 	if err != nil {
@@ -647,7 +647,7 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
 	}
 	bInfo.Project = project
 
-	// Override pool
+	// Override pool.
 	if pool != "" {
 		bInfo.Pool = pool
 	}
@@ -655,19 +655,27 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
 	run := func(op *operations.Operation) error {
 		defer f.Close()
 
-		// Dump tarball to storage
+		// Dump tarball to storage.
 		f.Seek(0, 0)
-		cPool, err := containerCreateFromBackup(d.State(), *bInfo, f, pool != "")
+		revertFunc, err := instanceCreateFromBackup(d.State(), *bInfo, f, pool != "")
 		if err != nil {
-			return errors.Wrap(err, "Create container from backup")
+			return errors.Wrap(err, "Create instance from backup")
 		}
 
+		revert := true
+		defer func() {
+			if !revert {
+				return
+			}
+
+			revertFunc()
+		}()
+
 		body, err := json.Marshal(&internalImportPost{
 			Name:  bInfo.Name,
 			Force: true,
 		})
 		if err != nil {
-			cPool.ContainerDelete(&containerLXC{name: bInfo.Name, project: project})
 			return errors.Wrap(err, "Marshal internal import request")
 		}
 
@@ -680,13 +688,12 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
 		resp := internalImport(d, req)
 
 		if resp.String() != "success" {
-			cPool.ContainerDelete(&containerLXC{name: bInfo.Name, project: project})
 			return fmt.Errorf("Internal import request: %v", resp.String())
 		}
 
 		c, err := instanceLoadByProjectAndName(d.State(), project, bInfo.Name)
 		if err != nil {
-			return errors.Wrap(err, "Load container")
+			return errors.Wrap(err, "Load instance")
 		}
 
 		_, err = c.StorageStop()
@@ -694,11 +701,13 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
 			return errors.Wrap(err, "Stop storage pool")
 		}
 
+		revert = false
 		return nil
 	}
 
 	resources := map[string][]string{}
-	resources["containers"] = []string{bInfo.Name}
+	resources["instances"] = []string{bInfo.Name}
+	resources["containers"] = resources["instances"]
 
 	op, err := operations.OperationCreate(d.State(), project, operations.OperationClassTask, db.OperationBackupRestore,
 		resources, nil, run, nil, nil)


More information about the lxc-devel mailing list