[lxc-devel] [lxd/master] Simplestreams handling cleanup

stgraber on Github lxc-bot at linuxcontainers.org
Tue Sep 3 21:31:07 UTC 2019


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 544 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190903/704bb241/attachment-0001.bin>
-------------- next part --------------
From 8e40f883b4d3c18f437fb287b6d958280df0b42b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 15:50:51 -0400
Subject: [PATCH 01/11] shared/simplestreams: Split out sortedImages
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/simplestreams.go | 52 +------------------------
 shared/simplestreams/sort.go          | 56 +++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 51 deletions(-)
 create mode 100644 shared/simplestreams/sort.go

diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 030da803c3..1d84701667 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -20,56 +20,6 @@ import (
 	"github.com/lxc/lxd/shared/osarch"
 )
 
-type ssSortImage []api.Image
-
-func (a ssSortImage) Len() int {
-	return len(a)
-}
-
-func (a ssSortImage) Swap(i, j int) {
-	a[i], a[j] = a[j], a[i]
-}
-
-func (a ssSortImage) Less(i, j int) bool {
-	if a[i].Properties["os"] == a[j].Properties["os"] {
-		if a[i].Properties["release"] == a[j].Properties["release"] {
-			if !shared.TimeIsSet(a[i].CreatedAt) {
-				return true
-			}
-
-			if !shared.TimeIsSet(a[j].CreatedAt) {
-				return false
-			}
-
-			if a[i].CreatedAt == a[j].CreatedAt {
-				return a[i].Properties["serial"] > a[j].Properties["serial"]
-			}
-
-			return a[i].CreatedAt.UTC().Unix() > a[j].CreatedAt.UTC().Unix()
-		}
-
-		if a[i].Properties["release"] == "" {
-			return false
-		}
-
-		if a[j].Properties["release"] == "" {
-			return true
-		}
-
-		return a[i].Properties["release"] < a[j].Properties["release"]
-	}
-
-	if a[i].Properties["os"] == "" {
-		return false
-	}
-
-	if a[j].Properties["os"] == "" {
-		return true
-	}
-
-	return a[i].Properties["os"] < a[j].Properties["os"]
-}
-
 var ssDefaultOS = map[string]string{
 	"https://cloud-images.ubuntu.com": "ubuntu",
 }
@@ -440,7 +390,7 @@ func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, erro
 func (s *SimpleStreams) applyAliases(images []api.Image) ([]api.Image, map[string]*api.ImageAliasesEntry, error) {
 	aliases := map[string]*api.ImageAliasesEntry{}
 
-	sort.Sort(ssSortImage(images))
+	sort.Sort(sortedImages(images))
 
 	defaultOS := ""
 	for k, v := range ssDefaultOS {
diff --git a/shared/simplestreams/sort.go b/shared/simplestreams/sort.go
new file mode 100644
index 0000000000..bb62abb422
--- /dev/null
+++ b/shared/simplestreams/sort.go
@@ -0,0 +1,56 @@
+package simplestreams
+
+import (
+	"github.com/lxc/lxd/shared"
+	"github.com/lxc/lxd/shared/api"
+)
+
+type sortedImages []api.Image
+
+func (a sortedImages) Len() int {
+	return len(a)
+}
+
+func (a sortedImages) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a sortedImages) Less(i, j int) bool {
+	if a[i].Properties["os"] == a[j].Properties["os"] {
+		if a[i].Properties["release"] == a[j].Properties["release"] {
+			if !shared.TimeIsSet(a[i].CreatedAt) {
+				return true
+			}
+
+			if !shared.TimeIsSet(a[j].CreatedAt) {
+				return false
+			}
+
+			if a[i].CreatedAt == a[j].CreatedAt {
+				return a[i].Properties["serial"] > a[j].Properties["serial"]
+			}
+
+			return a[i].CreatedAt.UTC().Unix() > a[j].CreatedAt.UTC().Unix()
+		}
+
+		if a[i].Properties["release"] == "" {
+			return false
+		}
+
+		if a[j].Properties["release"] == "" {
+			return true
+		}
+
+		return a[i].Properties["release"] < a[j].Properties["release"]
+	}
+
+	if a[i].Properties["os"] == "" {
+		return false
+	}
+
+	if a[j].Properties["os"] == "" {
+		return true
+	}
+
+	return a[i].Properties["os"] < a[j].Properties["os"]
+}

From 9ce8681b2f9565eb1417d4d65ba6cfa617efab22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 15:52:05 -0400
Subject: [PATCH 02/11] shared/simplestreams: Rename ssDefaultOS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/simplestreams.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 1d84701667..f629a33093 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -20,7 +20,7 @@ import (
 	"github.com/lxc/lxd/shared/osarch"
 )
 
-var ssDefaultOS = map[string]string{
+var urlDefaultOS = map[string]string{
 	"https://cloud-images.ubuntu.com": "ubuntu",
 }
 
@@ -393,7 +393,7 @@ func (s *SimpleStreams) applyAliases(images []api.Image) ([]api.Image, map[strin
 	sort.Sort(sortedImages(images))
 
 	defaultOS := ""
-	for k, v := range ssDefaultOS {
+	for k, v := range urlDefaultOS {
 		if strings.HasPrefix(s.url, k) {
 			defaultOS = v
 			break

From bb110d7211cd683ec81266c10d67d47585834525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 15:58:43 -0400
Subject: [PATCH 03/11] shared/simplestreams: Split index/manifest out
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/index.go         |  15 ++
 shared/simplestreams/manifest.go      | 250 +++++++++++++++++++++++++
 shared/simplestreams/simplestreams.go | 254 --------------------------
 3 files changed, 265 insertions(+), 254 deletions(-)
 create mode 100644 shared/simplestreams/index.go
 create mode 100644 shared/simplestreams/manifest.go

diff --git a/shared/simplestreams/index.go b/shared/simplestreams/index.go
new file mode 100644
index 0000000000..2416c489c3
--- /dev/null
+++ b/shared/simplestreams/index.go
@@ -0,0 +1,15 @@
+package simplestreams
+
+type SimpleStreamsIndex struct {
+	Format  string                              `json:"format"`
+	Index   map[string]SimpleStreamsIndexStream `json:"index"`
+	Updated string                              `json:"updated,omitempty"`
+}
+
+type SimpleStreamsIndexStream struct {
+	Updated  string   `json:"updated,omitempty"`
+	DataType string   `json:"datatype"`
+	Path     string   `json:"path"`
+	Format   string   `json:"format,omitempty"`
+	Products []string `json:"products"`
+}
diff --git a/shared/simplestreams/manifest.go b/shared/simplestreams/manifest.go
new file mode 100644
index 0000000000..c22b8721dc
--- /dev/null
+++ b/shared/simplestreams/manifest.go
@@ -0,0 +1,250 @@
+package simplestreams
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/lxc/lxd/shared"
+	"github.com/lxc/lxd/shared/api"
+	"github.com/lxc/lxd/shared/osarch"
+)
+
+type SimpleStreamsManifest struct {
+	Updated  string                                  `json:"updated,omitempty"`
+	DataType string                                  `json:"datatype"`
+	Format   string                                  `json:"format"`
+	License  string                                  `json:"license,omitempty"`
+	Products map[string]SimpleStreamsManifestProduct `json:"products"`
+}
+
+type SimpleStreamsManifestProduct struct {
+	Aliases         string                                         `json:"aliases"`
+	Architecture    string                                         `json:"arch"`
+	OperatingSystem string                                         `json:"os"`
+	Release         string                                         `json:"release"`
+	ReleaseCodename string                                         `json:"release_codename,omitempty"`
+	ReleaseTitle    string                                         `json:"release_title"`
+	Supported       bool                                           `json:"supported,omitempty"`
+	SupportedEOL    string                                         `json:"support_eol,omitempty"`
+	Version         string                                         `json:"version,omitempty"`
+	Versions        map[string]SimpleStreamsManifestProductVersion `json:"versions"`
+}
+
+type SimpleStreamsManifestProductVersion struct {
+	PublicName string                                             `json:"pubname,omitempty"`
+	Label      string                                             `json:"label,omitempty"`
+	Items      map[string]SimpleStreamsManifestProductVersionItem `json:"items"`
+}
+
+type SimpleStreamsManifestProductVersionItem struct {
+	Path                  string `json:"path"`
+	FileType              string `json:"ftype"`
+	HashMd5               string `json:"md5,omitempty"`
+	HashSha256            string `json:"sha256,omitempty"`
+	LXDHashSha256         string `json:"combined_sha256,omitempty"`
+	LXDHashSha256RootXz   string `json:"combined_rootxz_sha256,omitempty"`
+	LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"`
+	Size                  int64  `json:"size"`
+	DeltaBase             string `json:"delta_base,omitempty"`
+}
+
+func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) {
+	downloads := map[string][][]string{}
+
+	images := []api.Image{}
+	nameLayout := "20060102"
+	eolLayout := "2006-01-02"
+
+	for _, product := range s.Products {
+		// Skip unsupported architectures
+		architecture, err := osarch.ArchitectureId(product.Architecture)
+		if err != nil {
+			continue
+		}
+
+		architectureName, err := osarch.ArchitectureName(architecture)
+		if err != nil {
+			continue
+		}
+
+		for name, version := range product.Versions {
+			// Short of anything better, use the name as date (see format above)
+			if len(name) < 8 {
+				continue
+			}
+
+			creationDate, err := time.Parse(nameLayout, name[0:8])
+			if err != nil {
+				continue
+			}
+
+			var meta SimpleStreamsManifestProductVersionItem
+			var rootTar SimpleStreamsManifestProductVersionItem
+			var rootSquash SimpleStreamsManifestProductVersionItem
+			deltas := []SimpleStreamsManifestProductVersionItem{}
+
+			for _, item := range version.Items {
+				// Identify deltas
+				if item.FileType == "squashfs.vcdiff" {
+					deltas = append(deltas, item)
+				}
+
+				// Skip the files we don't care about
+				if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "lxd_combined.tar.gz", "squashfs"}) {
+					continue
+				}
+
+				if item.FileType == "lxd.tar.xz" {
+					meta = item
+				} else if item.FileType == "squashfs" {
+					rootSquash = item
+				} else if item.FileType == "root.tar.xz" {
+					rootTar = item
+				} else if item.FileType == "lxd_combined.tar.gz" {
+					meta = item
+					rootTar = item
+				}
+			}
+
+			if meta.FileType == "" || (rootTar.FileType == "" && rootSquash.FileType == "") {
+				// Invalid image
+				continue
+			}
+
+			var rootfsSize int64
+			metaPath := meta.Path
+			metaHash := meta.HashSha256
+			metaSize := meta.Size
+			rootfsPath := ""
+			rootfsHash := ""
+			fields := strings.Split(meta.Path, "/")
+			filename := fields[len(fields)-1]
+			size := meta.Size
+			fingerprint := ""
+
+			if rootSquash.FileType != "" {
+				if meta.LXDHashSha256SquashFs != "" {
+					fingerprint = meta.LXDHashSha256SquashFs
+				} else {
+					fingerprint = meta.LXDHashSha256
+				}
+				size += rootSquash.Size
+				rootfsPath = rootSquash.Path
+				rootfsHash = rootSquash.HashSha256
+				rootfsSize = rootSquash.Size
+			} else {
+				if meta == rootTar {
+					fingerprint = meta.HashSha256
+					size = meta.Size
+				} else {
+					if meta.LXDHashSha256RootXz != "" {
+						fingerprint = meta.LXDHashSha256RootXz
+					} else {
+						fingerprint = meta.LXDHashSha256
+					}
+					size += rootTar.Size
+				}
+				rootfsPath = rootTar.Path
+				rootfsHash = rootTar.HashSha256
+				rootfsSize = rootTar.Size
+			}
+
+			if size == 0 || filename == "" || fingerprint == "" {
+				// Invalid image
+				continue
+			}
+
+			// Generate the actual image entry
+			description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
+			if version.Label != "" {
+				description = fmt.Sprintf("%s (%s)", description, version.Label)
+			}
+			description = fmt.Sprintf("%s (%s)", description, name)
+
+			image := api.Image{}
+			image.Architecture = architectureName
+			image.Public = true
+			image.Size = size
+			image.CreatedAt = creationDate
+			image.UploadedAt = creationDate
+			image.Filename = filename
+			image.Fingerprint = fingerprint
+			image.Properties = map[string]string{
+				"os":           product.OperatingSystem,
+				"release":      product.Release,
+				"version":      product.Version,
+				"architecture": product.Architecture,
+				"label":        version.Label,
+				"serial":       name,
+				"description":  description,
+			}
+
+			// Add the provided aliases
+			if product.Aliases != "" {
+				image.Aliases = []api.ImageAlias{}
+				for _, entry := range strings.Split(product.Aliases, ",") {
+					image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry})
+				}
+			}
+
+			// Clear unset properties
+			for k, v := range image.Properties {
+				if v == "" {
+					delete(image.Properties, k)
+				}
+			}
+
+			// Attempt to parse the EOL
+			image.ExpiresAt = time.Unix(0, 0).UTC()
+			if product.SupportedEOL != "" {
+				eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
+				if err == nil {
+					image.ExpiresAt = eolDate
+				}
+			}
+
+			var imgDownloads [][]string
+			if meta == rootTar {
+				imgDownloads = [][]string{{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}}
+			} else {
+				imgDownloads = [][]string{
+					{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)},
+					{rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}}
+			}
+
+			// Add the deltas
+			for _, delta := range deltas {
+				srcImage, ok := product.Versions[delta.DeltaBase]
+				if !ok {
+					continue
+				}
+
+				var srcFingerprint string
+				for _, item := range srcImage.Items {
+					if item.FileType != "lxd.tar.xz" {
+						continue
+					}
+
+					srcFingerprint = item.LXDHashSha256SquashFs
+					break
+				}
+
+				if srcFingerprint == "" {
+					continue
+				}
+
+				imgDownloads = append(imgDownloads, []string{
+					delta.Path,
+					delta.HashSha256,
+					fmt.Sprintf("root.delta-%s", srcFingerprint),
+					fmt.Sprintf("%d", delta.Size)})
+			}
+
+			downloads[fingerprint] = imgDownloads
+			images = append(images, image)
+		}
+	}
+
+	return images, downloads
+}
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index f629a33093..d1ac9cf3cc 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -12,7 +12,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/lxc/lxd/shared"
 	"github.com/lxc/lxd/shared/api"
@@ -24,259 +23,6 @@ var urlDefaultOS = map[string]string{
 	"https://cloud-images.ubuntu.com": "ubuntu",
 }
 
-type SimpleStreamsManifest struct {
-	Updated  string                                  `json:"updated,omitempty"`
-	DataType string                                  `json:"datatype"`
-	Format   string                                  `json:"format"`
-	License  string                                  `json:"license,omitempty"`
-	Products map[string]SimpleStreamsManifestProduct `json:"products"`
-}
-
-func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) {
-	downloads := map[string][][]string{}
-
-	images := []api.Image{}
-	nameLayout := "20060102"
-	eolLayout := "2006-01-02"
-
-	for _, product := range s.Products {
-		// Skip unsupported architectures
-		architecture, err := osarch.ArchitectureId(product.Architecture)
-		if err != nil {
-			continue
-		}
-
-		architectureName, err := osarch.ArchitectureName(architecture)
-		if err != nil {
-			continue
-		}
-
-		for name, version := range product.Versions {
-			// Short of anything better, use the name as date (see format above)
-			if len(name) < 8 {
-				continue
-			}
-
-			creationDate, err := time.Parse(nameLayout, name[0:8])
-			if err != nil {
-				continue
-			}
-
-			var meta SimpleStreamsManifestProductVersionItem
-			var rootTar SimpleStreamsManifestProductVersionItem
-			var rootSquash SimpleStreamsManifestProductVersionItem
-			deltas := []SimpleStreamsManifestProductVersionItem{}
-
-			for _, item := range version.Items {
-				// Identify deltas
-				if item.FileType == "squashfs.vcdiff" {
-					deltas = append(deltas, item)
-				}
-
-				// Skip the files we don't care about
-				if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "lxd_combined.tar.gz", "squashfs"}) {
-					continue
-				}
-
-				if item.FileType == "lxd.tar.xz" {
-					meta = item
-				} else if item.FileType == "squashfs" {
-					rootSquash = item
-				} else if item.FileType == "root.tar.xz" {
-					rootTar = item
-				} else if item.FileType == "lxd_combined.tar.gz" {
-					meta = item
-					rootTar = item
-				}
-			}
-
-			if meta.FileType == "" || (rootTar.FileType == "" && rootSquash.FileType == "") {
-				// Invalid image
-				continue
-			}
-
-			var rootfsSize int64
-			metaPath := meta.Path
-			metaHash := meta.HashSha256
-			metaSize := meta.Size
-			rootfsPath := ""
-			rootfsHash := ""
-			fields := strings.Split(meta.Path, "/")
-			filename := fields[len(fields)-1]
-			size := meta.Size
-			fingerprint := ""
-
-			if rootSquash.FileType != "" {
-				if meta.LXDHashSha256SquashFs != "" {
-					fingerprint = meta.LXDHashSha256SquashFs
-				} else {
-					fingerprint = meta.LXDHashSha256
-				}
-				size += rootSquash.Size
-				rootfsPath = rootSquash.Path
-				rootfsHash = rootSquash.HashSha256
-				rootfsSize = rootSquash.Size
-			} else {
-				if meta == rootTar {
-					fingerprint = meta.HashSha256
-					size = meta.Size
-				} else {
-					if meta.LXDHashSha256RootXz != "" {
-						fingerprint = meta.LXDHashSha256RootXz
-					} else {
-						fingerprint = meta.LXDHashSha256
-					}
-					size += rootTar.Size
-				}
-				rootfsPath = rootTar.Path
-				rootfsHash = rootTar.HashSha256
-				rootfsSize = rootTar.Size
-			}
-
-			if size == 0 || filename == "" || fingerprint == "" {
-				// Invalid image
-				continue
-			}
-
-			// Generate the actual image entry
-			description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
-			if version.Label != "" {
-				description = fmt.Sprintf("%s (%s)", description, version.Label)
-			}
-			description = fmt.Sprintf("%s (%s)", description, name)
-
-			image := api.Image{}
-			image.Architecture = architectureName
-			image.Public = true
-			image.Size = size
-			image.CreatedAt = creationDate
-			image.UploadedAt = creationDate
-			image.Filename = filename
-			image.Fingerprint = fingerprint
-			image.Properties = map[string]string{
-				"os":           product.OperatingSystem,
-				"release":      product.Release,
-				"version":      product.Version,
-				"architecture": product.Architecture,
-				"label":        version.Label,
-				"serial":       name,
-				"description":  description,
-			}
-
-			// Add the provided aliases
-			if product.Aliases != "" {
-				image.Aliases = []api.ImageAlias{}
-				for _, entry := range strings.Split(product.Aliases, ",") {
-					image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry})
-				}
-			}
-
-			// Clear unset properties
-			for k, v := range image.Properties {
-				if v == "" {
-					delete(image.Properties, k)
-				}
-			}
-
-			// Attempt to parse the EOL
-			image.ExpiresAt = time.Unix(0, 0).UTC()
-			if product.SupportedEOL != "" {
-				eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
-				if err == nil {
-					image.ExpiresAt = eolDate
-				}
-			}
-
-			var imgDownloads [][]string
-			if meta == rootTar {
-				imgDownloads = [][]string{{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}}
-			} else {
-				imgDownloads = [][]string{
-					{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)},
-					{rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}}
-			}
-
-			// Add the deltas
-			for _, delta := range deltas {
-				srcImage, ok := product.Versions[delta.DeltaBase]
-				if !ok {
-					continue
-				}
-
-				var srcFingerprint string
-				for _, item := range srcImage.Items {
-					if item.FileType != "lxd.tar.xz" {
-						continue
-					}
-
-					srcFingerprint = item.LXDHashSha256SquashFs
-					break
-				}
-
-				if srcFingerprint == "" {
-					continue
-				}
-
-				imgDownloads = append(imgDownloads, []string{
-					delta.Path,
-					delta.HashSha256,
-					fmt.Sprintf("root.delta-%s", srcFingerprint),
-					fmt.Sprintf("%d", delta.Size)})
-			}
-
-			downloads[fingerprint] = imgDownloads
-			images = append(images, image)
-		}
-	}
-
-	return images, downloads
-}
-
-type SimpleStreamsManifestProduct struct {
-	Aliases         string                                         `json:"aliases"`
-	Architecture    string                                         `json:"arch"`
-	OperatingSystem string                                         `json:"os"`
-	Release         string                                         `json:"release"`
-	ReleaseCodename string                                         `json:"release_codename,omitempty"`
-	ReleaseTitle    string                                         `json:"release_title"`
-	Supported       bool                                           `json:"supported,omitempty"`
-	SupportedEOL    string                                         `json:"support_eol,omitempty"`
-	Version         string                                         `json:"version,omitempty"`
-	Versions        map[string]SimpleStreamsManifestProductVersion `json:"versions"`
-}
-
-type SimpleStreamsManifestProductVersion struct {
-	PublicName string                                             `json:"pubname,omitempty"`
-	Label      string                                             `json:"label,omitempty"`
-	Items      map[string]SimpleStreamsManifestProductVersionItem `json:"items"`
-}
-
-type SimpleStreamsManifestProductVersionItem struct {
-	Path                  string `json:"path"`
-	FileType              string `json:"ftype"`
-	HashMd5               string `json:"md5,omitempty"`
-	HashSha256            string `json:"sha256,omitempty"`
-	LXDHashSha256         string `json:"combined_sha256,omitempty"`
-	LXDHashSha256RootXz   string `json:"combined_rootxz_sha256,omitempty"`
-	LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"`
-	Size                  int64  `json:"size"`
-	DeltaBase             string `json:"delta_base,omitempty"`
-}
-
-type SimpleStreamsIndex struct {
-	Format  string                              `json:"format"`
-	Index   map[string]SimpleStreamsIndexStream `json:"index"`
-	Updated string                              `json:"updated,omitempty"`
-}
-
-type SimpleStreamsIndexStream struct {
-	Updated  string   `json:"updated,omitempty"`
-	DataType string   `json:"datatype"`
-	Path     string   `json:"path"`
-	Format   string   `json:"format,omitempty"`
-	Products []string `json:"products"`
-}
-
 type SimpleStreamsFile struct {
 	Path   string
 	Sha256 string

From 2eb29c1dd3d922a9f4980a983ca7f058e9a89483 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:09:08 -0400
Subject: [PATCH 04/11] shared/simplestreams: Rename index structs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/index.go         | 16 +++++++++-------
 shared/simplestreams/simplestreams.go | 16 ++++++++--------
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/shared/simplestreams/index.go b/shared/simplestreams/index.go
index 2416c489c3..91f7c197fd 100644
--- a/shared/simplestreams/index.go
+++ b/shared/simplestreams/index.go
@@ -1,15 +1,17 @@
 package simplestreams
 
-type SimpleStreamsIndex struct {
-	Format  string                              `json:"format"`
-	Index   map[string]SimpleStreamsIndexStream `json:"index"`
-	Updated string                              `json:"updated,omitempty"`
+// Stream represents the base structure of index.json
+type Stream struct {
+	Index   map[string]StreamIndex `json:"index"`
+	Updated string                 `json:"updated,omitempty"`
+	Format  string                 `json:"format"`
 }
 
-type SimpleStreamsIndexStream struct {
-	Updated  string   `json:"updated,omitempty"`
+// StreamIndex represents the Index entry inside index.json
+type StreamIndex struct {
 	DataType string   `json:"datatype"`
 	Path     string   `json:"path"`
-	Format   string   `json:"format,omitempty"`
+	Updated  string   `json:"updated,omitempty"`
 	Products []string `json:"products"`
+	Format   string   `json:"format,omitempty"`
 }
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index d1ac9cf3cc..81f4a68012 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -43,15 +43,15 @@ type SimpleStreams struct {
 	url       string
 	useragent string
 
-	cachedIndex    *SimpleStreamsIndex
+	cachedStream   *Stream
 	cachedManifest map[string]*SimpleStreamsManifest
 	cachedImages   []api.Image
 	cachedAliases  map[string]*api.ImageAliasesEntry
 }
 
-func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) {
-	if s.cachedIndex != nil {
-		return s.cachedIndex, nil
+func (s *SimpleStreams) parseIndex() (*Stream, error) {
+	if s.cachedStream != nil {
+		return s.cachedStream, nil
 	}
 
 	url := fmt.Sprintf("%s/streams/v1/index.json", s.url)
@@ -80,15 +80,15 @@ func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) {
 	}
 
 	// Parse the idnex
-	ssIndex := SimpleStreamsIndex{}
-	err = json.Unmarshal(body, &ssIndex)
+	stream := Stream{}
+	err = json.Unmarshal(body, &stream)
 	if err != nil {
 		return nil, err
 	}
 
-	s.cachedIndex = &ssIndex
+	s.cachedStream = &stream
 
-	return &ssIndex, nil
+	return &stream, nil
 }
 
 func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) {

From 498666b87688190cbec7003c86e9622515efaca5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:16:17 -0400
Subject: [PATCH 05/11] shared/simplestreams: Rename product structs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 .../{manifest.go => products.go}              | 69 ++++++++++---------
 shared/simplestreams/simplestreams.go         |  8 +--
 2 files changed, 42 insertions(+), 35 deletions(-)
 rename shared/simplestreams/{manifest.go => products.go} (74%)

diff --git a/shared/simplestreams/manifest.go b/shared/simplestreams/products.go
similarity index 74%
rename from shared/simplestreams/manifest.go
rename to shared/simplestreams/products.go
index c22b8721dc..ab89eaea48 100644
--- a/shared/simplestreams/manifest.go
+++ b/shared/simplestreams/products.go
@@ -10,46 +10,53 @@ import (
 	"github.com/lxc/lxd/shared/osarch"
 )
 
-type SimpleStreamsManifest struct {
-	Updated  string                                  `json:"updated,omitempty"`
-	DataType string                                  `json:"datatype"`
-	Format   string                                  `json:"format"`
-	License  string                                  `json:"license,omitempty"`
-	Products map[string]SimpleStreamsManifestProduct `json:"products"`
+// Products represents the base of download.json
+type Products struct {
+	ContentID string             `json:"content_id"`
+	DataType  string             `json:"datatype"`
+	Format    string             `json:"format"`
+	License   string             `json:"license,omitempty"`
+	Products  map[string]Product `json:"products"`
+	Updated   string             `json:"updated,omitempty"`
 }
 
-type SimpleStreamsManifestProduct struct {
-	Aliases         string                                         `json:"aliases"`
-	Architecture    string                                         `json:"arch"`
-	OperatingSystem string                                         `json:"os"`
-	Release         string                                         `json:"release"`
-	ReleaseCodename string                                         `json:"release_codename,omitempty"`
-	ReleaseTitle    string                                         `json:"release_title"`
-	Supported       bool                                           `json:"supported,omitempty"`
-	SupportedEOL    string                                         `json:"support_eol,omitempty"`
-	Version         string                                         `json:"version,omitempty"`
-	Versions        map[string]SimpleStreamsManifestProductVersion `json:"versions"`
+// Product represents a single product inside download.json
+type Product struct {
+	Aliases         string                    `json:"aliases"`
+	Architecture    string                    `json:"arch"`
+	OperatingSystem string                    `json:"os"`
+	Release         string                    `json:"release"`
+	ReleaseCodename string                    `json:"release_codename,omitempty"`
+	ReleaseTitle    string                    `json:"release_title"`
+	Supported       bool                      `json:"supported,omitempty"`
+	SupportedEOL    string                    `json:"support_eol,omitempty"`
+	Version         string                    `json:"version,omitempty"`
+	Versions        map[string]ProductVersion `json:"versions"`
 }
 
-type SimpleStreamsManifestProductVersion struct {
-	PublicName string                                             `json:"pubname,omitempty"`
-	Label      string                                             `json:"label,omitempty"`
-	Items      map[string]SimpleStreamsManifestProductVersionItem `json:"items"`
+// ProductVersion represents a particular version of a product
+type ProductVersion struct {
+	Items      map[string]ProductVersionItem `json:"items"`
+	Label      string                        `json:"label,omitempty"`
+	PublicName string                        `json:"pubname,omitempty"`
 }
 
-type SimpleStreamsManifestProductVersionItem struct {
-	Path                  string `json:"path"`
+// ProductVersionItem represents a file/item of a particular ProductVersion
+type ProductVersionItem struct {
+	LXDHashSha256RootXz   string `json:"combined_rootxz_sha256,omitempty"`
+	LXDHashSha256         string `json:"combined_sha256,omitempty"`
+	LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"`
 	FileType              string `json:"ftype"`
 	HashMd5               string `json:"md5,omitempty"`
+	Path                  string `json:"path"`
 	HashSha256            string `json:"sha256,omitempty"`
-	LXDHashSha256         string `json:"combined_sha256,omitempty"`
-	LXDHashSha256RootXz   string `json:"combined_rootxz_sha256,omitempty"`
-	LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"`
 	Size                  int64  `json:"size"`
 	DeltaBase             string `json:"delta_base,omitempty"`
 }
 
-func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) {
+
+// ToLXD converts the products data into a list of LXD images and associated downloadable files
+func (s *Products) ToLXD() ([]api.Image, map[string][][]string) {
 	downloads := map[string][][]string{}
 
 	images := []api.Image{}
@@ -79,10 +86,10 @@ func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) {
 				continue
 			}
 
-			var meta SimpleStreamsManifestProductVersionItem
-			var rootTar SimpleStreamsManifestProductVersionItem
-			var rootSquash SimpleStreamsManifestProductVersionItem
-			deltas := []SimpleStreamsManifestProductVersionItem{}
+			var meta ProductVersionItem
+			var rootTar ProductVersionItem
+			var rootSquash ProductVersionItem
+			deltas := []ProductVersionItem{}
 
 			for _, item := range version.Items {
 				// Identify deltas
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 81f4a68012..6182b1c2e3 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -33,7 +33,7 @@ func NewClient(url string, httpClient http.Client, useragent string) *SimpleStre
 	return &SimpleStreams{
 		http:           &httpClient,
 		url:            url,
-		cachedManifest: map[string]*SimpleStreamsManifest{},
+		cachedManifest: map[string]*Products{},
 		useragent:      useragent,
 	}
 }
@@ -44,7 +44,7 @@ type SimpleStreams struct {
 	useragent string
 
 	cachedStream   *Stream
-	cachedManifest map[string]*SimpleStreamsManifest
+	cachedManifest map[string]*Products
 	cachedImages   []api.Image
 	cachedAliases  map[string]*api.ImageAliasesEntry
 }
@@ -91,7 +91,7 @@ func (s *SimpleStreams) parseIndex() (*Stream, error) {
 	return &stream, nil
 }
 
-func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) {
+func (s *SimpleStreams) parseManifest(path string) (*Products, error) {
 	if s.cachedManifest[path] != nil {
 		return s.cachedManifest[path], nil
 	}
@@ -122,7 +122,7 @@ func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, erro
 	}
 
 	// Parse the idnex
-	ssManifest := SimpleStreamsManifest{}
+	ssManifest := Products{}
 	err = json.Unmarshal(body, &ssManifest)
 	if err != nil {
 		return nil, err

From c4d50f1249636cd73063b6399c41421f56762b4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:19:31 -0400
Subject: [PATCH 06/11] shared/simplestreams: Rename SimpleStreamsFile
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/products.go      | 1 -
 shared/simplestreams/simplestreams.go | 8 ++++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/shared/simplestreams/products.go b/shared/simplestreams/products.go
index ab89eaea48..920dc01f84 100644
--- a/shared/simplestreams/products.go
+++ b/shared/simplestreams/products.go
@@ -54,7 +54,6 @@ type ProductVersionItem struct {
 	DeltaBase             string `json:"delta_base,omitempty"`
 }
 
-
 // ToLXD converts the products data into a list of LXD images and associated downloadable files
 func (s *Products) ToLXD() ([]api.Image, map[string][][]string) {
 	downloads := map[string][][]string{}
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 6182b1c2e3..56d904c765 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -23,7 +23,7 @@ var urlDefaultOS = map[string]string{
 	"https://cloud-images.ubuntu.com": "ubuntu",
 }
 
-type SimpleStreamsFile struct {
+type DownloadableFile struct {
 	Path   string
 	Sha256 string
 	Size   int64
@@ -244,7 +244,7 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn
 	return images, aliases, nil
 }
 
-func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]SimpleStreamsFile, error) {
+func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFile, error) {
 	// Load the main index
 	ssIndex, err := s.parseIndex()
 	if err != nil {
@@ -272,7 +272,7 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]SimpleStreamsFi
 
 		for _, image := range manifestImages {
 			if strings.HasPrefix(image.Fingerprint, fingerprint) {
-				files := map[string]SimpleStreamsFile{}
+				files := map[string]DownloadableFile{}
 
 				for _, path := range downloads[image.Fingerprint] {
 					size, err := strconv.ParseInt(path[3], 10, 64)
@@ -280,7 +280,7 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]SimpleStreamsFi
 						return nil, err
 					}
 
-					files[path[2]] = SimpleStreamsFile{
+					files[path[2]] = DownloadableFile{
 						Path:   path[0],
 						Sha256: path[1],
 						Size:   size}

From 40e44f1ed21ab7d0a5c94db262ecdf338ce65464 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:25:26 -0400
Subject: [PATCH 07/11] shared/simplestreams: Rename internal functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/simplestreams.go | 48 +++++++++++++--------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 56d904c765..7c85aae880 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -33,7 +33,7 @@ func NewClient(url string, httpClient http.Client, useragent string) *SimpleStre
 	return &SimpleStreams{
 		http:           &httpClient,
 		url:            url,
-		cachedManifest: map[string]*Products{},
+		cachedProducts: map[string]*Products{},
 		useragent:      useragent,
 	}
 }
@@ -44,12 +44,12 @@ type SimpleStreams struct {
 	useragent string
 
 	cachedStream   *Stream
-	cachedManifest map[string]*Products
+	cachedProducts map[string]*Products
 	cachedImages   []api.Image
 	cachedAliases  map[string]*api.ImageAliasesEntry
 }
 
-func (s *SimpleStreams) parseIndex() (*Stream, error) {
+func (s *SimpleStreams) parseStream() (*Stream, error) {
 	if s.cachedStream != nil {
 		return s.cachedStream, nil
 	}
@@ -91,9 +91,9 @@ func (s *SimpleStreams) parseIndex() (*Stream, error) {
 	return &stream, nil
 }
 
-func (s *SimpleStreams) parseManifest(path string) (*Products, error) {
-	if s.cachedManifest[path] != nil {
-		return s.cachedManifest[path], nil
+func (s *SimpleStreams) parseProducts(path string) (*Products, error) {
+	if s.cachedProducts[path] != nil {
+		return s.cachedProducts[path], nil
 	}
 
 	url := fmt.Sprintf("%s/%s", s.url, path)
@@ -122,15 +122,15 @@ func (s *SimpleStreams) parseManifest(path string) (*Products, error) {
 	}
 
 	// Parse the idnex
-	ssManifest := Products{}
-	err = json.Unmarshal(body, &ssManifest)
+	products := Products{}
+	err = json.Unmarshal(body, &products)
 	if err != nil {
 		return nil, err
 	}
 
-	s.cachedManifest[path] = &ssManifest
+	s.cachedProducts[path] = &products
 
-	return &ssManifest, nil
+	return &products, nil
 }
 
 func (s *SimpleStreams) applyAliases(images []api.Image) ([]api.Image, map[string]*api.ImageAliasesEntry, error) {
@@ -202,14 +202,14 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn
 
 	images := []api.Image{}
 
-	// Load the main index
-	ssIndex, err := s.parseIndex()
+	// Load the stream data
+	stream, err := s.parseStream()
 	if err != nil {
 		return nil, nil, err
 	}
 
-	// Iterate through the various image manifests
-	for _, entry := range ssIndex.Index {
+	// Iterate through the various indices
+	for _, entry := range stream.Index {
 		// We only care about images
 		if entry.DataType != "image-downloads" {
 			continue
@@ -220,14 +220,14 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn
 			continue
 		}
 
-		manifest, err := s.parseManifest(entry.Path)
+		products, err := s.parseProducts(entry.Path)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		manifestImages, _ := manifest.ToLXD()
+		streamImages, _ := products.ToLXD()
 
-		for _, image := range manifestImages {
+		for _, image := range streamImages {
 			images = append(images, image)
 		}
 	}
@@ -245,14 +245,14 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn
 }
 
 func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFile, error) {
-	// Load the main index
-	ssIndex, err := s.parseIndex()
+	// Load the main stream
+	stream, err := s.parseStream()
 	if err != nil {
 		return nil, err
 	}
 
-	// Iterate through the various image manifests
-	for _, entry := range ssIndex.Index {
+	// Iterate through the various indices
+	for _, entry := range stream.Index {
 		// We only care about images
 		if entry.DataType != "image-downloads" {
 			continue
@@ -263,14 +263,14 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFil
 			continue
 		}
 
-		manifest, err := s.parseManifest(entry.Path)
+		products, err := s.parseProducts(entry.Path)
 		if err != nil {
 			return nil, err
 		}
 
-		manifestImages, downloads := manifest.ToLXD()
+		images, downloads := products.ToLXD()
 
-		for _, image := range manifestImages {
+		for _, image := range images {
 			if strings.HasPrefix(image.Fingerprint, fingerprint) {
 				files := map[string]DownloadableFile{}
 

From 3b594dffa599658947e872563873af5c3559a130 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:33:40 -0400
Subject: [PATCH 08/11] shared/simplestreams: Remove dead code
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/simplestreams.go | 109 --------------------------
 1 file changed, 109 deletions(-)

diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 7c85aae880..e8e16dfafd 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -1,21 +1,15 @@
 package simplestreams
 
 import (
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
-	"os"
-	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
 
-	"github.com/lxc/lxd/shared"
 	"github.com/lxc/lxd/shared/api"
-	"github.com/lxc/lxd/shared/ioprogress"
 	"github.com/lxc/lxd/shared/osarch"
 )
 
@@ -294,72 +288,6 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFil
 	return nil, fmt.Errorf("Couldn't find the requested image")
 }
 
-func (s *SimpleStreams) downloadFile(path string, hash string, target string, progress func(int64, int64)) error {
-	download := func(url string, hash string, target string) error {
-		out, err := os.Create(target)
-		if err != nil {
-			return err
-		}
-		defer out.Close()
-
-		req, err := http.NewRequest("GET", url, nil)
-		if err != nil {
-			return err
-		}
-
-		if s.useragent != "" {
-			req.Header.Set("User-Agent", s.useragent)
-		}
-
-		r, err := s.http.Do(req)
-		if err != nil {
-			return err
-		}
-		defer r.Body.Close()
-
-		if r.StatusCode != http.StatusOK {
-			return fmt.Errorf("Unable to fetch %s: %s", url, r.Status)
-		}
-
-		body := &ioprogress.ProgressReader{
-			ReadCloser: r.Body,
-			Tracker: &ioprogress.ProgressTracker{
-				Length:  r.ContentLength,
-				Handler: progress,
-			},
-		}
-
-		sha256 := sha256.New()
-		_, err = io.Copy(io.MultiWriter(out, sha256), body)
-		if err != nil {
-			return err
-		}
-
-		result := fmt.Sprintf("%x", sha256.Sum(nil))
-		if result != hash {
-			os.Remove(target)
-			return fmt.Errorf("Hash mismatch for %s: %s != %s", path, result, hash)
-		}
-
-		return nil
-	}
-
-	// Try http first
-	if strings.HasPrefix(s.url, "https://") {
-		err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, target)
-		if err == nil {
-			return nil
-		}
-	}
-
-	err := download(fmt.Sprintf("%s/%s", s.url, path), hash, target)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (s *SimpleStreams) ListAliases() ([]api.ImageAliasesEntry, error) {
 	_, aliasesMap, err := s.getImages()
 	if err != nil {
@@ -416,40 +344,3 @@ func (s *SimpleStreams) GetImage(fingerprint string) (*api.Image, error) {
 
 	return &matches[0], nil
 }
-
-func (s *SimpleStreams) ExportImage(image string, target string) (string, error) {
-	if !shared.IsDir(target) {
-		return "", fmt.Errorf("Split images can only be written to a directory")
-	}
-
-	files, err := s.GetFiles(image)
-	if err != nil {
-		return "", err
-	}
-
-	for _, file := range files {
-		fields := strings.Split(file.Path, "/")
-		targetFile := filepath.Join(target, fields[len(fields)-1])
-
-		err := s.downloadFile(file.Path, file.Sha256, targetFile, nil)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	return target, nil
-}
-
-func (s *SimpleStreams) Download(image string, fileType string, target string, progress func(int64, int64)) error {
-	files, err := s.GetFiles(image)
-	if err != nil {
-		return err
-	}
-
-	file, ok := files[fileType]
-	if ok {
-		return s.downloadFile(file.Path, file.Sha256, target, progress)
-	}
-
-	return fmt.Errorf("The file couldn't be found")
-}

From 50cb1720a930493faff47f427335e21ebf28dca9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:34:50 -0400
Subject: [PATCH 09/11] shared/simplestreams: Make golint clean
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/simplestreams.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index e8e16dfafd..f6f805894f 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -17,12 +17,14 @@ var urlDefaultOS = map[string]string{
 	"https://cloud-images.ubuntu.com": "ubuntu",
 }
 
+// DownloadableFile represents a file with its URL, hash and size
 type DownloadableFile struct {
 	Path   string
 	Sha256 string
 	Size   int64
 }
 
+// NewClient returns a simplestreams client for the provided stream URL
 func NewClient(url string, httpClient http.Client, useragent string) *SimpleStreams {
 	return &SimpleStreams{
 		http:           &httpClient,
@@ -32,6 +34,7 @@ func NewClient(url string, httpClient http.Client, useragent string) *SimpleStre
 	}
 }
 
+// SimpleStreams represents a simplestream client
 type SimpleStreams struct {
 	http      *http.Client
 	url       string
@@ -238,6 +241,7 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn
 	return images, aliases, nil
 }
 
+// GetFiles returns a map of files for the provided image fingerprint
 func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFile, error) {
 	// Load the main stream
 	stream, err := s.parseStream()
@@ -288,6 +292,7 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFil
 	return nil, fmt.Errorf("Couldn't find the requested image")
 }
 
+// ListAliases returns a list of image aliases for the provided image fingerprint
 func (s *SimpleStreams) ListAliases() ([]api.ImageAliasesEntry, error) {
 	_, aliasesMap, err := s.getImages()
 	if err != nil {
@@ -303,11 +308,13 @@ func (s *SimpleStreams) ListAliases() ([]api.ImageAliasesEntry, error) {
 	return aliases, nil
 }
 
+// ListImages returns a list of LXD images
 func (s *SimpleStreams) ListImages() ([]api.Image, error) {
 	images, _, err := s.getImages()
 	return images, err
 }
 
+// GetAlias returns a LXD ImageAliasesEntry for the provided alias name
 func (s *SimpleStreams) GetAlias(name string) (*api.ImageAliasesEntry, error) {
 	_, aliasesMap, err := s.getImages()
 	if err != nil {
@@ -322,6 +329,7 @@ func (s *SimpleStreams) GetAlias(name string) (*api.ImageAliasesEntry, error) {
 	return alias, nil
 }
 
+// GetImage returns a LXD image for the provided image fingerprint
 func (s *SimpleStreams) GetImage(fingerprint string) (*api.Image, error) {
 	images, _, err := s.getImages()
 	if err != nil {

From 72f42c01ce7cb714cf8f8ac4df596cbbf29c500a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 16:35:32 -0400
Subject: [PATCH 10/11] tests: Add simplestreams to golint
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 test/suites/static_analysis.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/suites/static_analysis.sh b/test/suites/static_analysis.sh
index cd2da54018..8f40270444 100644
--- a/test/suites/static_analysis.sh
+++ b/test/suites/static_analysis.sh
@@ -101,6 +101,7 @@ test_static_analysis() {
       golint -set_exit_status shared/log15/stack
       golint -set_exit_status shared/logger/
       golint -set_exit_status shared/logging/
+      golint -set_exit_status shared/simplestreams/
       golint -set_exit_status shared/subtest/
       golint -set_exit_status shared/termios/
       golint -set_exit_status shared/version/

From 4c857306e1d866312ea13959f15e3a3171d8c176 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 3 Sep 2019 17:27:59 -0400
Subject: [PATCH 11/11] shared/simplestreams: Record all images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Until now when both squashfs and tar.xz existed, only squashfs would
make it to the final index, making it impossible to query information
about the tar.xz image.

This fixes that, while still prefering squashfs in results.

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 shared/simplestreams/products.go | 277 ++++++++++++++++---------------
 shared/simplestreams/sort.go     |   6 +
 2 files changed, 148 insertions(+), 135 deletions(-)

diff --git a/shared/simplestreams/products.go b/shared/simplestreams/products.go
index 920dc01f84..b1b06c0681 100644
--- a/shared/simplestreams/products.go
+++ b/shared/simplestreams/products.go
@@ -85,170 +85,177 @@ func (s *Products) ToLXD() ([]api.Image, map[string][][]string) {
 				continue
 			}
 
-			var meta ProductVersionItem
-			var rootTar ProductVersionItem
-			var rootSquash ProductVersionItem
-			deltas := []ProductVersionItem{}
+			// Image processing function
+			addImage := func(meta *ProductVersionItem, root *ProductVersionItem) error {
+				// Look for deltas (only on squashfs)
+				deltas := []ProductVersionItem{}
+				if root != nil && root.FileType == "squashfs" {
+					for _, item := range version.Items {
+						if item.FileType == "squashfs.vcdiff" {
+							deltas = append(deltas, item)
+						}
+					}
+				}
 
-			for _, item := range version.Items {
-				// Identify deltas
-				if item.FileType == "squashfs.vcdiff" {
-					deltas = append(deltas, item)
+				// Figure out the fingerprint
+				fingerprint := ""
+				if root != nil {
+					if root.FileType == "root.tar.xz" {
+						if meta.LXDHashSha256RootXz != "" {
+							fingerprint = meta.LXDHashSha256RootXz
+						} else {
+							fingerprint = meta.LXDHashSha256
+						}
+					} else if root.FileType == "squashfs" {
+						fingerprint = meta.LXDHashSha256SquashFs
+					}
+				} else {
+					fingerprint = meta.HashSha256
 				}
 
-				// Skip the files we don't care about
-				if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "lxd_combined.tar.gz", "squashfs"}) {
-					continue
+				if fingerprint == "" {
+					return fmt.Errorf("No LXD image fingerprint found")
 				}
 
-				if item.FileType == "lxd.tar.xz" {
-					meta = item
-				} else if item.FileType == "squashfs" {
-					rootSquash = item
-				} else if item.FileType == "root.tar.xz" {
-					rootTar = item
-				} else if item.FileType == "lxd_combined.tar.gz" {
-					meta = item
-					rootTar = item
+				// Figure out the size
+				size := meta.Size
+				if root != nil {
+					size += root.Size
 				}
-			}
 
-			if meta.FileType == "" || (rootTar.FileType == "" && rootSquash.FileType == "") {
-				// Invalid image
-				continue
-			}
+				// Determine filename
+				if meta.Path == "" {
+					return fmt.Errorf("Missing path field on metadata entry")
+				}
 
-			var rootfsSize int64
-			metaPath := meta.Path
-			metaHash := meta.HashSha256
-			metaSize := meta.Size
-			rootfsPath := ""
-			rootfsHash := ""
-			fields := strings.Split(meta.Path, "/")
-			filename := fields[len(fields)-1]
-			size := meta.Size
-			fingerprint := ""
-
-			if rootSquash.FileType != "" {
-				if meta.LXDHashSha256SquashFs != "" {
-					fingerprint = meta.LXDHashSha256SquashFs
-				} else {
-					fingerprint = meta.LXDHashSha256
+				fields := strings.Split(meta.Path, "/")
+				filename := fields[len(fields)-1]
+
+				// Generate the actual image entry
+				description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
+				if version.Label != "" {
+					description = fmt.Sprintf("%s (%s)", description, version.Label)
 				}
-				size += rootSquash.Size
-				rootfsPath = rootSquash.Path
-				rootfsHash = rootSquash.HashSha256
-				rootfsSize = rootSquash.Size
-			} else {
-				if meta == rootTar {
-					fingerprint = meta.HashSha256
-					size = meta.Size
-				} else {
-					if meta.LXDHashSha256RootXz != "" {
-						fingerprint = meta.LXDHashSha256RootXz
-					} else {
-						fingerprint = meta.LXDHashSha256
-					}
-					size += rootTar.Size
+				description = fmt.Sprintf("%s (%s)", description, name)
+
+				image := api.Image{}
+				image.Architecture = architectureName
+				image.Public = true
+				image.Size = size
+				image.CreatedAt = creationDate
+				image.UploadedAt = creationDate
+				image.Filename = filename
+				image.Fingerprint = fingerprint
+				image.Properties = map[string]string{
+					"os":           product.OperatingSystem,
+					"release":      product.Release,
+					"version":      product.Version,
+					"architecture": product.Architecture,
+					"label":        version.Label,
+					"serial":       name,
+					"description":  description,
 				}
-				rootfsPath = rootTar.Path
-				rootfsHash = rootTar.HashSha256
-				rootfsSize = rootTar.Size
-			}
 
-			if size == 0 || filename == "" || fingerprint == "" {
-				// Invalid image
-				continue
-			}
+				if root != nil {
+					image.Properties["type"] = root.FileType
+				} else {
+					image.Properties["type"] = "tar.gz"
+				}
 
-			// Generate the actual image entry
-			description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
-			if version.Label != "" {
-				description = fmt.Sprintf("%s (%s)", description, version.Label)
-			}
-			description = fmt.Sprintf("%s (%s)", description, name)
-
-			image := api.Image{}
-			image.Architecture = architectureName
-			image.Public = true
-			image.Size = size
-			image.CreatedAt = creationDate
-			image.UploadedAt = creationDate
-			image.Filename = filename
-			image.Fingerprint = fingerprint
-			image.Properties = map[string]string{
-				"os":           product.OperatingSystem,
-				"release":      product.Release,
-				"version":      product.Version,
-				"architecture": product.Architecture,
-				"label":        version.Label,
-				"serial":       name,
-				"description":  description,
-			}
+				// Clear unset properties
+				for k, v := range image.Properties {
+					if v == "" {
+						delete(image.Properties, k)
+					}
+				}
 
-			// Add the provided aliases
-			if product.Aliases != "" {
-				image.Aliases = []api.ImageAlias{}
-				for _, entry := range strings.Split(product.Aliases, ",") {
-					image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry})
+				// Add the provided aliases
+				if product.Aliases != "" {
+					image.Aliases = []api.ImageAlias{}
+					for _, entry := range strings.Split(product.Aliases, ",") {
+						image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry})
+					}
 				}
-			}
 
-			// Clear unset properties
-			for k, v := range image.Properties {
-				if v == "" {
-					delete(image.Properties, k)
+				// Attempt to parse the EOL
+				image.ExpiresAt = time.Unix(0, 0).UTC()
+				if product.SupportedEOL != "" {
+					eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
+					if err == nil {
+						image.ExpiresAt = eolDate
+					}
 				}
-			}
 
-			// Attempt to parse the EOL
-			image.ExpiresAt = time.Unix(0, 0).UTC()
-			if product.SupportedEOL != "" {
-				eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
-				if err == nil {
-					image.ExpiresAt = eolDate
+				// Set the file list
+				var imgDownloads [][]string
+				if root == nil {
+					imgDownloads = [][]string{{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)}}
+				} else {
+					imgDownloads = [][]string{
+						{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)},
+						{root.Path, root.HashSha256, "root", fmt.Sprintf("%d", root.Size)}}
 				}
-			}
 
-			var imgDownloads [][]string
-			if meta == rootTar {
-				imgDownloads = [][]string{{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}}
-			} else {
-				imgDownloads = [][]string{
-					{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)},
-					{rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}}
-			}
+				// Add the deltas
+				for _, delta := range deltas {
+					srcImage, ok := product.Versions[delta.DeltaBase]
+					if !ok {
+						// Delta for a since expired image
+						continue
+					}
 
-			// Add the deltas
-			for _, delta := range deltas {
-				srcImage, ok := product.Versions[delta.DeltaBase]
-				if !ok {
-					continue
-				}
+					// Locate source image fingerprint
+					var srcFingerprint string
+					for _, item := range srcImage.Items {
+						if item.FileType != "lxd.tar.xz" {
+							continue
+						}
 
-				var srcFingerprint string
-				for _, item := range srcImage.Items {
-					if item.FileType != "lxd.tar.xz" {
+						srcFingerprint = item.LXDHashSha256SquashFs
+						break
+					}
+
+					if srcFingerprint == "" {
+						// Couldn't find the image
 						continue
 					}
 
-					srcFingerprint = item.LXDHashSha256SquashFs
-					break
+					// Add the delta
+					imgDownloads = append(imgDownloads, []string{
+						delta.Path,
+						delta.HashSha256,
+						fmt.Sprintf("root.delta-%s", srcFingerprint),
+						fmt.Sprintf("%d", delta.Size)})
 				}
 
-				if srcFingerprint == "" {
-					continue
-				}
+				// Add the image
+				downloads[fingerprint] = imgDownloads
+				images = append(images, image)
 
-				imgDownloads = append(imgDownloads, []string{
-					delta.Path,
-					delta.HashSha256,
-					fmt.Sprintf("root.delta-%s", srcFingerprint),
-					fmt.Sprintf("%d", delta.Size)})
+				return nil
 			}
 
-			downloads[fingerprint] = imgDownloads
-			images = append(images, image)
+			// Locate a valid LXD image
+			for _, item := range version.Items {
+				if item.FileType == "lxd_combined.tar.gz" {
+					err := addImage(&item, nil)
+					if err != nil {
+						continue
+					}
+				}
+
+				if item.FileType == "lxd.tar.xz" {
+					// Locate the root files
+					for _, subItem := range version.Items {
+						if shared.StringInSlice(subItem.FileType, []string{"root.tar.xz", "squashfs"}) {
+							err := addImage(&item, &subItem)
+							if err != nil {
+								continue
+							}
+						}
+					}
+				}
+			}
 		}
 	}
 
diff --git a/shared/simplestreams/sort.go b/shared/simplestreams/sort.go
index bb62abb422..78872e4e7d 100644
--- a/shared/simplestreams/sort.go
+++ b/shared/simplestreams/sort.go
@@ -16,6 +16,12 @@ func (a sortedImages) Swap(i, j int) {
 }
 
 func (a sortedImages) Less(i, j int) bool {
+	if a[i].Properties["type"] != a[j].Properties["type"] {
+		if a[i].Properties["type"] == "squashfs" {
+			return true
+		}
+	}
+
 	if a[i].Properties["os"] == a[j].Properties["os"] {
 		if a[i].Properties["release"] == a[j].Properties["release"] {
 			if !shared.TimeIsSet(a[i].CreatedAt) {


More information about the lxc-devel mailing list