[lxc-devel] [lxd/master] Add support for simplestreams based delta image transfers
stgraber on Github
lxc-bot at linuxcontainers.org
Fri Aug 18 06:32:59 UTC 2017
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20170818/556ff1b3/attachment.bin>
-------------- next part --------------
From df098ccda7be31b3ce2124ab756c98acce81b14c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 18 Aug 2017 00:56:07 -0400
Subject: [PATCH 1/4] client: Cleanup code duplication in image download
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client/simplestreams_images.go | 39 +++++++++++++++++++++------------------
1 file changed, 21 insertions(+), 18 deletions(-)
diff --git a/client/simplestreams_images.go b/client/simplestreams_images.go
index fd82d9912..1ae9c6acf 100644
--- a/client/simplestreams_images.go
+++ b/client/simplestreams_images.go
@@ -2,6 +2,7 @@ package lxd
import (
"fmt"
+ "io"
"strings"
"github.com/lxc/lxd/shared/api"
@@ -57,22 +58,32 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe
// Prepare the response
resp := ImageFileResponse{}
- // Download the LXD image file
- meta, ok := files["meta"]
- if ok && req.MetaFile != nil {
+ // Download function
+ download := func(path string, filename string, sha256 string, target io.WriteSeeker) (int64, error) {
// Try over http
- url := fmt.Sprintf("http://%s/%s", strings.TrimPrefix(r.httpHost, "https://"), meta.Path)
+ url := fmt.Sprintf("http://%s/%s", strings.TrimPrefix(r.httpHost, "https://"), path)
- size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "metadata", url, meta.Sha256, req.MetaFile)
+ size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target)
if err != nil {
// Try over https
- url = fmt.Sprintf("%s/%s", r.httpHost, meta.Path)
- size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "metadata", url, meta.Sha256, req.MetaFile)
+ url = fmt.Sprintf("%s/%s", r.httpHost, path)
+ size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target)
if err != nil {
- return nil, err
+ return -1, err
}
}
+ return size, nil
+ }
+
+ // Download the LXD image file
+ meta, ok := files["meta"]
+ if ok && req.MetaFile != nil {
+ size, err := download(meta.Path, "metadata", meta.Sha256, req.MetaFile)
+ if err != nil {
+ return nil, err
+ }
+
parts := strings.Split(meta.Path, "/")
resp.MetaName = parts[len(parts)-1]
resp.MetaSize = size
@@ -81,17 +92,9 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe
// Download the rootfs
rootfs, ok := files["root"]
if ok && req.RootfsFile != nil {
- // Try over http
- url := fmt.Sprintf("http://%s/%s", strings.TrimPrefix(r.httpHost, "https://"), rootfs.Path)
-
- size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "rootfs", url, rootfs.Sha256, req.RootfsFile)
+ size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile)
if err != nil {
- // Try over https
- url = fmt.Sprintf("%s/%s", r.httpHost, rootfs.Path)
- size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, "rootfs", url, rootfs.Sha256, req.RootfsFile)
- if err != nil {
- return nil, err
- }
+ return nil, err
}
parts := strings.Split(rootfs.Path, "/")
From 3011ed610aca6ba4e71ee8236de58b2f62834233 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 18 Aug 2017 01:40:37 -0400
Subject: [PATCH 2/4] shared/simplestreams: Add delta support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
shared/simplestreams/simplestreams.go | 39 ++++++++++++++++++++++++++++++++++-
1 file changed, 38 insertions(+), 1 deletion(-)
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 0615fe011..322f4fffe 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -115,8 +115,14 @@ func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) {
var meta SimpleStreamsManifestProductVersionItem
var rootTar SimpleStreamsManifestProductVersionItem
var rootSquash SimpleStreamsManifestProductVersionItem
+ deltas := []SimpleStreamsManifestProductVersionItem{}
for _, item := range version.Items {
+ // Identify deltas
+ if item.FileType == "squashfs.vcdiff" {
+ deltas = append(deltas, item)
+ }
+
// Skip the files we don't care about
if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "squashfs"}) {
continue
@@ -223,9 +229,39 @@ func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) {
}
}
- downloads[fingerprint] = [][]string{
+ imgDownloads := [][]string{
{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)},
{rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}}
+
+ // Add the deltas
+ for _, delta := range deltas {
+ srcImage, ok := product.Versions[delta.DeltaBase]
+ if !ok {
+ continue
+ }
+
+ var srcFingerprint string
+ for _, item := range srcImage.Items {
+ if item.FileType != "lxd.tar.xz" {
+ continue
+ }
+
+ srcFingerprint = item.LXDHashSha256SquashFs
+ break
+ }
+
+ if srcFingerprint == "" {
+ continue
+ }
+
+ imgDownloads = append(imgDownloads, []string{
+ delta.Path,
+ delta.HashSha256,
+ fmt.Sprintf("root.delta-%s", srcFingerprint),
+ fmt.Sprintf("%d", delta.Size)})
+ }
+
+ downloads[fingerprint] = imgDownloads
images = append(images, image)
}
}
@@ -261,6 +297,7 @@ type SimpleStreamsManifestProductVersionItem struct {
LXDHashSha256RootXz string `json:"combined_rootxz_sha256"`
LXDHashSha256SquashFs string `json:"combined_squashfs_sha256"`
Size int64 `json:"size"`
+ DeltaBase string `json:"delta_base"`
}
type SimpleStreamsIndex struct {
From 217b609723b4a9a33b379d47d7ee3fb20ff7312f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 18 Aug 2017 02:08:52 -0400
Subject: [PATCH 3/4] client: Add support for delta image transfer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client/interfaces.go | 4 +++
client/simplestreams_images.go | 77 ++++++++++++++++++++++++++++++++++++++----
2 files changed, 75 insertions(+), 6 deletions(-)
diff --git a/client/interfaces.go b/client/interfaces.go
index ed9b001a3..f6cb6006c 100644
--- a/client/interfaces.go
+++ b/client/interfaces.go
@@ -209,6 +209,10 @@ type ImageFileRequest struct {
// A canceler that can be used to interrupt some part of the image download request
Canceler *cancel.Canceler
+
+ // Path retriever for image delta downloads
+ // If set, it must return the path to the image file or an empty string if not available
+ DeltaSourceRetriever func(fingerprint string, file string) string
}
// The ImageFileResponse struct is used as the response for image downloads
diff --git a/client/simplestreams_images.go b/client/simplestreams_images.go
index 1ae9c6acf..21c266d8f 100644
--- a/client/simplestreams_images.go
+++ b/client/simplestreams_images.go
@@ -3,8 +3,12 @@ package lxd
import (
"fmt"
"io"
+ "io/ioutil"
+ "os"
+ "os/exec"
"strings"
+ "github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
)
@@ -92,14 +96,75 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe
// Download the rootfs
rootfs, ok := files["root"]
if ok && req.RootfsFile != nil {
- size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile)
- if err != nil {
- return nil, err
+ // Look for deltas (requires xdelta3)
+ downloaded := false
+ _, err := exec.LookPath("xdelta3")
+ if err == nil && req.DeltaSourceRetriever != nil {
+ for filename, file := range files {
+ if !strings.HasPrefix(filename, "root.delta-") {
+ continue
+ }
+
+ // Check if we have the source file for the delta
+ srcFingerprint := strings.Split(filename, "root.delta-")[1]
+ srcPath := req.DeltaSourceRetriever(srcFingerprint, "rootfs")
+ if srcPath == "" {
+ continue
+ }
+
+ // Create temporary file for the delta
+ deltaFile, err := ioutil.TempFile("", "lxc_image_")
+ if err != nil {
+ return nil, err
+ }
+ defer deltaFile.Close()
+ defer os.Remove(deltaFile.Name())
+
+ // Download the delta
+ _, err = download(file.Path, "rootfs delta", file.Sha256, deltaFile)
+ if err != nil {
+ return nil, err
+ }
+ deltaFile.Close()
+
+ // Create temporary file for the delta
+ patchedFile, err := ioutil.TempFile("", "lxc_image_")
+ if err != nil {
+ return nil, err
+ }
+ defer patchedFile.Close()
+ defer os.Remove(patchedFile.Name())
+
+ // Apply it
+ _, err = shared.RunCommand("xdelta3", "-f", "-d", "-s", srcPath, deltaFile.Name(), patchedFile.Name())
+ if err != nil {
+ return nil, err
+ }
+
+ // Copy to the target
+ size, err := io.Copy(req.RootfsFile, patchedFile)
+ if err != nil {
+ return nil, err
+ }
+
+ parts := strings.Split(rootfs.Path, "/")
+ resp.RootfsName = parts[len(parts)-1]
+ resp.RootfsSize = size
+ downloaded = true
+ }
}
- parts := strings.Split(rootfs.Path, "/")
- resp.RootfsName = parts[len(parts)-1]
- resp.RootfsSize = size
+ // Download the whole file
+ if !downloaded {
+ size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile)
+ if err != nil {
+ return nil, err
+ }
+
+ parts := strings.Split(rootfs.Path, "/")
+ resp.RootfsName = parts[len(parts)-1]
+ resp.RootfsSize = size
+ }
}
return &resp, nil
From 81a19a6707c912edd03cb5522c67ea76e327cc81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 18 Aug 2017 02:30:27 -0400
Subject: [PATCH 4/4] lxd: Add support for delta images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/daemon_images.go | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go
index 659bb0d1a..4a56ca861 100644
--- a/lxd/daemon_images.go
+++ b/lxd/daemon_images.go
@@ -403,6 +403,14 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce
RootfsFile: io.WriteSeeker(destRootfs),
ProgressHandler: progress,
Canceler: canceler,
+ DeltaSourceRetriever: func(fingerprint string, file string) string {
+ path := shared.VarPath("images", fmt.Sprintf("%s.%s", fingerprint, file))
+ if shared.PathExists(path) {
+ return path
+ }
+
+ return ""
+ },
}
if secret != "" {
More information about the lxc-devel
mailing list