[lxc-devel] [lxd/master] Progress2 multiwrite sha256

joelhockey on Github lxc-bot at linuxcontainers.org
Fri Feb 1 05:40:10 UTC 2019


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190131/79aa8958/attachment.bin>
-------------- next part --------------
From 20a55a17baf3933d10ca7aee009d54b8365cec7e Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhockey at chromium.org>
Date: Thu, 31 Jan 2019 18:52:24 -0800
Subject: [PATCH 1/2] lxd/images: change compressFile to take io.Reader and
 io.Writer

This is part 1 of a series of patches to add better progress
tracking support for export and import.

By using Reader and Writer rather than filename for compressing
the caller can provide a tracking reader/writer for progress.

Signed-off-by: Joel Hockey <joelhockey at chromium.org>
---
 lxd/backup.go  | 16 ++++++++++++++--
 lxd/images.go  | 35 ++++++++++++++++++-----------------
 lxd/patches.go | 16 ++++++++++++++--
 3 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/lxd/backup.go b/lxd/backup.go
index 46a53a3014..e469a19429 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -353,7 +353,19 @@ func backupCreateTarball(s *state.State, path string, backup backup) error {
 	}
 
 	if compress != "none" {
-		compressedPath, err := compressFile(backupPath, compress)
+		infile, err := os.Open(backupPath)
+		if err != nil {
+			return err
+		}
+		defer infile.Close()
+
+		compressed, err := os.Create(backupPath + ".compressed")
+		if err != nil {
+			return err
+		}
+		defer compressed.Close()
+
+		err = compressFile(compress, infile, compressed)
 		if err != nil {
 			return err
 		}
@@ -363,7 +375,7 @@ func backupCreateTarball(s *state.State, path string, backup backup) error {
 			return err
 		}
 
-		err = os.Rename(compressedPath, backupPath)
+		err = os.Rename(compressed.Name(), backupPath)
 		if err != nil {
 			return err
 		}
diff --git a/lxd/images.go b/lxd/images.go
index a27f4f5620..ea8c2cc69f 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -129,7 +129,7 @@ func unpackImage(imagefname string, destpath string, sType storageType, runningI
 	return nil
 }
 
-func compressFile(path string, compress string) (string, error) {
+func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
 	reproducible := []string{"gzip"}
 
 	args := []string{"-c"}
@@ -137,24 +137,11 @@ func compressFile(path string, compress string) (string, error) {
 		args = append(args, "-n")
 	}
 
-	args = append(args, path)
 	cmd := exec.Command(compress, args...)
-
-	outfile, err := os.Create(path + ".compressed")
-	if err != nil {
-		return "", err
-	}
-
-	defer outfile.Close()
+	cmd.Stdin = infile
 	cmd.Stdout = outfile
 
-	err = cmd.Run()
-	if err != nil {
-		os.Remove(outfile.Name())
-		return "", err
-	}
-
-	return outfile.Name(), nil
+	return cmd.Run()
 }
 
 /*
@@ -223,7 +210,21 @@ func imgPostContInfo(d *Daemon, r *http.Request, req api.ImagesPost, builddir st
 	}
 
 	if compress != "none" {
-		compressedPath, err = compressFile(tarfile.Name(), compress)
+		tarfile, err = os.Open(tarfile.Name())
+		if err != nil {
+			return nil, err
+		}
+		defer tarfile.Close()
+
+		compressedPath = tarfile.Name() + ".compressed"
+
+		compressed, err := os.Create(compressedPath)
+		if err != nil {
+			return nil, err
+		}
+		defer compressed.Close()
+
+		err = compressFile(compress, tarfile, compressed)
 		if err != nil {
 			return nil, err
 		}
diff --git a/lxd/patches.go b/lxd/patches.go
index 235410e66c..93b2ab8e5f 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -3154,7 +3154,19 @@ func patchMoveBackups(name string, d *Daemon) error {
 				}
 
 				// Compress it
-				compressedPath, err := compressFile(backupPath, "xz")
+				infile, err := os.Open(backupPath)
+				if err != nil {
+					return err
+				}
+				defer infile.Close()
+
+				compressed, err := os.Create(backupPath + ".compressed")
+				if err != nil {
+					return err
+				}
+				defer compressed.Close()
+
+				err = compressFile("xz", infile, compressed)
 				if err != nil {
 					return err
 				}
@@ -3164,7 +3176,7 @@ func patchMoveBackups(name string, d *Daemon) error {
 					return err
 				}
 
-				err = os.Rename(compressedPath, backupPath)
+				err = os.Rename(compressed.Name(), backupPath)
 				if err != nil {
 					return err
 				}

From a7723ab209169960b503f3ee789bb2f36050d21b Mon Sep 17 00:00:00 2001
From: Joel Hockey <joelhockey at chromium.org>
Date: Thu, 31 Jan 2019 19:03:24 -0800
Subject: [PATCH 2/2] lxd/images: calculate sha256 as image is written

This is part 2 of a series of patches to add better progress
tracking support for export and import.

Calculate sha256 as either tarfile is written (if no compression)
or as compression is done to improve performance.

Signed-off-by: Joel Hockey <joelhockey at chromium.org>
---
 lxd/images.go | 46 ++++++++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/lxd/images.go b/lxd/images.go
index ea8c2cc69f..e32488f5b9 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -191,14 +191,10 @@ func imgPostContInfo(d *Daemon, r *http.Request, req api.ImagesPost, builddir st
 	}
 	defer os.Remove(tarfile.Name())
 
-	if err := c.Export(tarfile, req.Properties); err != nil {
-		tarfile.Close()
-		return nil, err
-	}
-	tarfile.Close()
-
+	sha256 := sha256.New()
 	var compressedPath string
 	var compress string
+	var writer io.Writer
 
 	if req.CompressionAlgorithm != "" {
 		compress = req.CompressionAlgorithm
@@ -208,8 +204,23 @@ func imgPostContInfo(d *Daemon, r *http.Request, req api.ImagesPost, builddir st
 			return nil, err
 		}
 	}
+	usingCompression := compress != "none"
 
-	if compress != "none" {
+	// If there is no compression, then calculate sha256 on tarfile
+	if usingCompression {
+		writer = tarfile
+	} else {
+		writer = io.MultiWriter(tarfile, sha256)
+		compressedPath = tarfile.Name()
+	}
+
+	if err := c.Export(writer, req.Properties); err != nil {
+		tarfile.Close()
+		return nil, err
+	}
+	tarfile.Close()
+
+	if usingCompression {
 		tarfile, err = os.Open(tarfile.Name())
 		if err != nil {
 			return nil, err
@@ -222,29 +233,24 @@ func imgPostContInfo(d *Daemon, r *http.Request, req api.ImagesPost, builddir st
 		if err != nil {
 			return nil, err
 		}
+
 		defer compressed.Close()
+		defer os.Remove(compressed.Name())
 
-		err = compressFile(compress, tarfile, compressed)
+		// Calculate sha256 as we compress
+		writer := io.MultiWriter(compressed, sha256)
+
+		err = compressFile(compress, tarfile, writer)
 		if err != nil {
 			return nil, err
 		}
-	} else {
-		compressedPath = tarfile.Name()
 	}
-	defer os.Remove(compressedPath)
 
-	sha256 := sha256.New()
-	tarf, err := os.Open(compressedPath)
-	if err != nil {
-		return nil, err
-	}
-
-	info.Size, err = io.Copy(sha256, tarf)
-	tarf.Close()
+	fi, err := os.Stat(compressedPath)
 	if err != nil {
 		return nil, err
 	}
-
+	info.Size = fi.Size()
 	info.Fingerprint = fmt.Sprintf("%x", sha256.Sum(nil))
 
 	_, _, err = d.cluster.ImageGet(project, info.Fingerprint, false, true)


More information about the lxc-devel mailing list