[lxc-devel] [lxd/master] images: Use native Go tar for metadata

stgraber on Github lxc-bot at linuxcontainers.org
Fri Sep 6 06:12:20 UTC 2019


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190905/7e492c4e/attachment.bin>
-------------- next part --------------
From 67e611c4e4b8d65694c5697025bc67ef8682cf6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Sep 2019 14:52:12 -0400
Subject: [PATCH 1/2] lxd/images: Use native tar parser for metadata
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: St├ęphane Graber <stgraber at ubuntu.com>
---
 lxd/images.go | 82 ++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 61 insertions(+), 21 deletions(-)

diff --git a/lxd/images.go b/lxd/images.go
index efb35eee83..3a258d9fa9 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"crypto/sha256"
@@ -810,46 +811,85 @@ func imagesPost(d *Daemon, r *http.Request) Response {
 }
 
 func getImageMetadata(fname string) (*api.ImageMetadata, error) {
-	metadataName := "metadata.yaml"
+	var tr *tar.Reader
+	var result api.ImageMetadata
 
-	compressionArgs, _, _, err := shared.DetectCompression(fname)
+	// Open the file
+	r, err := os.Open(fname)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
 
+	// Decompress if needed
+	_, _, unpacker, err := shared.DetectCompressionFile(r)
 	if err != nil {
-		return nil, fmt.Errorf(
-			"detectCompression failed, err='%v', tarfile='%s'",
-			err,
-			fname)
+		return nil, err
 	}
+	r.Seek(0, 0)
 
-	args := []string{"-O"}
-	args = append(args, compressionArgs...)
-	args = append(args, fname, metadataName)
+	if unpacker == nil {
+		return nil, fmt.Errorf("Unsupported backup compression")
+	}
 
-	// read the metadata.yaml
-	output, err := shared.RunCommand("tar", args...)
+	// Open the tarball
+	if len(unpacker) > 0 {
+		cmd := exec.Command(unpacker[0], unpacker[1:]...)
+		cmd.Stdin = r
 
-	if err != nil {
-		outputLines := strings.Split(output, "\n")
-		return nil, fmt.Errorf("Could not extract image %s from tar: %v (%s)", metadataName, err, outputLines[0])
+		stdout, err := cmd.StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
+		defer stdout.Close()
+
+		err = cmd.Start()
+		if err != nil {
+			return nil, err
+		}
+		defer cmd.Wait()
+
+		tr = tar.NewReader(stdout)
+	} else {
+		tr = tar.NewReader(r)
 	}
 
-	metadata := api.ImageMetadata{}
-	err = yaml.Unmarshal([]byte(output), &metadata)
+	// Parse the content
+	hasMeta := false
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break // End of archive
+		}
+		if err != nil {
+			return nil, err
+		}
 
-	if err != nil {
-		return nil, fmt.Errorf("Could not parse %s: %v", metadataName, err)
+		if hdr.Name == "metadata.yaml" || hdr.Name == "./metadata.yaml" {
+			err = yaml.NewDecoder(tr).Decode(&result)
+			if err != nil {
+				return nil, err
+			}
+
+			hasMeta = true
+			break
+		}
+	}
+
+	if !hasMeta {
+		return nil, fmt.Errorf("Metadata tarball is missing metadata.yaml")
 	}
 
-	_, err = osarch.ArchitectureId(metadata.Architecture)
+	_, err = osarch.ArchitectureId(result.Architecture)
 	if err != nil {
 		return nil, err
 	}
 
-	if metadata.CreationDate == 0 {
+	if result.CreationDate == 0 {
 		return nil, fmt.Errorf("Missing creation date")
 	}
 
-	return &metadata, nil
+	return &result, nil
 }
 
 func doImagesGet(d *Daemon, recursion bool, project string, public bool) (interface{}, error) {

From 07e74d30479253fbadb09900de3cfb02f49d618a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Sep 2019 15:10:23 -0400
Subject: [PATCH 2/2] lxd/images: Tweak wrapping
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: St├ęphane Graber <stgraber at ubuntu.com>
---
 lxd/images.go | 71 +++++++++++++++++----------------------------------
 1 file changed, 24 insertions(+), 47 deletions(-)

diff --git a/lxd/images.go b/lxd/images.go
index 3a258d9fa9..eaaf6caf01 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -463,24 +463,19 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string,
 
 		imageTarf.Close()
 		if err != nil {
-			logger.Error(
-				"Failed to copy the image tarfile",
-				log.Ctx{"err": err})
+			logger.Error("Failed to copy the image tarfile", log.Ctx{"err": err})
 			return nil, err
 		}
 
 		// Get the rootfs tarball
 		part, err = mr.NextPart()
 		if err != nil {
-			logger.Error(
-				"Failed to get the next part",
-				log.Ctx{"err": err})
+			logger.Error("Failed to get the next part", log.Ctx{"err": err})
 			return nil, err
 		}
 
 		if part.FormName() != "rootfs" {
-			logger.Error(
-				"Invalid multipart image")
+			logger.Error("Invalid multipart image")
 
 			return nil, fmt.Errorf("Invalid multipart image")
 		}
@@ -497,9 +492,7 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string,
 
 		rootfsTarf.Close()
 		if err != nil {
-			logger.Error(
-				"Failed to copy the rootfs tarfile",
-				log.Ctx{"err": err})
+			logger.Error("Failed to copy the rootfs tarfile", log.Ctx{"err": err})
 			return nil, err
 		}
 
@@ -514,33 +507,27 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string,
 
 		imageMeta, err = getImageMetadata(imageTarf.Name())
 		if err != nil {
-			logger.Error(
-				"Failed to get image metadata",
-				log.Ctx{"err": err})
+			logger.Error("Failed to get image metadata", log.Ctx{"err": err})
 			return nil, err
 		}
 
 		imgfname := shared.VarPath("images", info.Fingerprint)
 		err = shared.FileMove(imageTarf.Name(), imgfname)
 		if err != nil {
-			logger.Error(
-				"Failed to move the image tarfile",
-				log.Ctx{
-					"err":    err,
-					"source": imageTarf.Name(),
-					"dest":   imgfname})
+			logger.Error("Failed to move the image tarfile", log.Ctx{
+				"err":    err,
+				"source": imageTarf.Name(),
+				"dest":   imgfname})
 			return nil, err
 		}
 
 		rootfsfname := shared.VarPath("images", info.Fingerprint+".rootfs")
 		err = shared.FileMove(rootfsTarf.Name(), rootfsfname)
 		if err != nil {
-			logger.Error(
-				"Failed to move the rootfs tarfile",
-				log.Ctx{
-					"err":    err,
-					"source": rootfsTarf.Name(),
-					"dest":   imgfname})
+			logger.Error("Failed to move the rootfs tarfile", log.Ctx{
+				"err":    err,
+				"source": rootfsTarf.Name(),
+				"dest":   imgfname})
 			return nil, err
 		}
 	} else {
@@ -549,9 +536,7 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string,
 		info.Size = size
 		logger.Debug("Tar size", log.Ctx{"size": size})
 		if err != nil {
-			logger.Error(
-				"Failed to copy the tarfile",
-				log.Ctx{"err": err})
+			logger.Error("Failed to copy the tarfile", log.Ctx{"err": err})
 			return nil, err
 		}
 
@@ -560,35 +545,26 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string,
 
 		expectedFingerprint := r.Header.Get("X-LXD-fingerprint")
 		if expectedFingerprint != "" && info.Fingerprint != expectedFingerprint {
-			logger.Error(
-				"Fingerprints don't match",
-				log.Ctx{
-					"got":      info.Fingerprint,
-					"expected": expectedFingerprint})
-			err = fmt.Errorf(
-				"fingerprints don't match, got %s expected %s",
-				info.Fingerprint,
-				expectedFingerprint)
+			logger.Error("Fingerprints don't match", log.Ctx{
+				"got":      info.Fingerprint,
+				"expected": expectedFingerprint})
+			err = fmt.Errorf("fingerprints don't match, got %s expected %s", info.Fingerprint, expectedFingerprint)
 			return nil, err
 		}
 
 		imageMeta, err = getImageMetadata(post.Name())
 		if err != nil {
-			logger.Error(
-				"Failed to get image metadata",
-				log.Ctx{"err": err})
+			logger.Error("Failed to get image metadata", log.Ctx{"err": err})
 			return nil, err
 		}
 
 		imgfname := shared.VarPath("images", info.Fingerprint)
 		err = shared.FileMove(post.Name(), imgfname)
 		if err != nil {
-			logger.Error(
-				"Failed to move the tarfile",
-				log.Ctx{
-					"err":    err,
-					"source": post.Name(),
-					"dest":   imgfname})
+			logger.Error("Failed to move the tarfile", log.Ctx{
+				"err":    err,
+				"source": post.Name(),
+				"dest":   imgfname})
 			return nil, err
 		}
 	}
@@ -612,6 +588,7 @@ func getImgPostInfo(d *Daemon, r *http.Request, builddir string, project string,
 	if err != nil {
 		return nil, err
 	}
+
 	if exists {
 		// Do not create a database entry if the request is coming from the internal
 		// cluster communications for image synchronization


More information about the lxc-devel mailing list