[lxc-devel] [lxd/master] Refactoring
stgraber on Github
lxc-bot at linuxcontainers.org
Thu Dec 15 23:50:24 UTC 2016
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20161215/2e908cb4/attachment.bin>
-------------- next part --------------
From 524e330fb9670f71be23e2b2bc05647853ff1cc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 15 Dec 2016 18:20:08 -0500
Subject: [PATCH 1/5] daemon: Common codepath for http client
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/daemon.go | 40 ++++++++++++++--------------------------
lxd/images.go | 14 ++------------
2 files changed, 16 insertions(+), 38 deletions(-)
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 3d1c85c..4687c8c 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -107,10 +107,10 @@ type Command struct {
patch func(d *Daemon, r *http.Request) Response
}
-func (d *Daemon) httpGetSync(url string, certificate string) (*lxd.Response, error) {
+func (d *Daemon) httpClient(certificate string) (*http.Client, error) {
var err error
-
var cert *x509.Certificate
+
if certificate != "" {
certBlock, _ := pem.Decode([]byte(certificate))
if certBlock == nil {
@@ -139,6 +139,12 @@ func (d *Daemon) httpGetSync(url string, certificate string) (*lxd.Response, err
Transport: tr,
}
+ return &myhttp, nil
+}
+
+func (d *Daemon) httpGetSync(url string, certificate string) (*lxd.Response, error) {
+ var err error
+
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
@@ -146,6 +152,11 @@ func (d *Daemon) httpGetSync(url string, certificate string) (*lxd.Response, err
req.Header.Set("User-Agent", shared.UserAgent)
+ myhttp, err := d.httpClient(certificate)
+ if err != nil {
+ return nil, err
+ }
+
r, err := myhttp.Do(req)
if err != nil {
return nil, err
@@ -166,34 +177,11 @@ func (d *Daemon) httpGetSync(url string, certificate string) (*lxd.Response, err
func (d *Daemon) httpGetFile(url string, certificate string) (*http.Response, error) {
var err error
- var cert *x509.Certificate
- if certificate != "" {
- certBlock, _ := pem.Decode([]byte(certificate))
- if certBlock == nil {
- return nil, fmt.Errorf("Invalid certificate")
- }
-
- cert, err = x509.ParseCertificate(certBlock.Bytes)
- if err != nil {
- return nil, err
- }
- }
-
- tlsConfig, err := shared.GetTLSConfig("", "", "", cert)
+ myhttp, err := d.httpClient(certificate)
if err != nil {
return nil, err
}
- tr := &http.Transport{
- TLSClientConfig: tlsConfig,
- Dial: shared.RFC3493Dialer,
- Proxy: d.proxy,
- DisableKeepAlives: true,
- }
- myhttp := http.Client{
- Transport: tr,
- }
-
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
diff --git a/lxd/images.go b/lxd/images.go
index dfa2d5c..291789f 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -375,22 +375,12 @@ func imgPostURLInfo(d *Daemon, req imagePostReq, op *operation) error {
return fmt.Errorf("Missing URL")
}
- // Resolve the image URL
- tlsConfig, err := shared.GetTLSConfig("", "", "", nil)
+ myhttp, err := d.httpClient("")
if err != nil {
return err
}
- tr := &http.Transport{
- TLSClientConfig: tlsConfig,
- Dial: shared.RFC3493Dialer,
- Proxy: d.proxy,
- }
-
- myhttp := http.Client{
- Transport: tr,
- }
-
+ // Resolve the image URL
head, err := http.NewRequest("HEAD", req.Source["url"], nil)
if err != nil {
return err
From c3c611bceb1aa85d2d194c4debe286185e917e4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 15 Dec 2016 17:18:44 -0500
Subject: [PATCH 2/5] shared: Give IO progress tracker its own package
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client.go | 9 ++--
lxd/daemon_images.go | 5 +-
lxd/storage.go | 9 ++--
shared/ioprogress/reader.go | 23 +++++++++
shared/ioprogress/tracker.go | 76 ++++++++++++++++++++++++++++++
shared/ioprogress/writer.go | 23 +++++++++
shared/simplestreams.go | 6 ++-
shared/util.go | 108 -------------------------------------------
8 files changed, 139 insertions(+), 120 deletions(-)
create mode 100644 shared/ioprogress/reader.go
create mode 100644 shared/ioprogress/tracker.go
create mode 100644 shared/ioprogress/writer.go
diff --git a/client.go b/client.go
index e931216..f7eb6af 100644
--- a/client.go
+++ b/client.go
@@ -24,6 +24,7 @@ import (
"github.com/gorilla/websocket"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/ioprogress"
)
// Client can talk to a LXD daemon.
@@ -1028,9 +1029,9 @@ func (c *Client) PostImage(imageFile string, rootfsFile string, properties []str
return "", err
}
- progress := &shared.ProgressReader{
+ progress := &ioprogress.ProgressReader{
ReadCloser: body,
- Tracker: &shared.ProgressTracker{
+ Tracker: &ioprogress.ProgressTracker{
Length: size,
Handler: progressHandler,
},
@@ -1050,9 +1051,9 @@ func (c *Client) PostImage(imageFile string, rootfsFile string, properties []str
return "", err
}
- progress := &shared.ProgressReader{
+ progress := &ioprogress.ProgressReader{
ReadCloser: fImage,
- Tracker: &shared.ProgressTracker{
+ Tracker: &ioprogress.ProgressTracker{
Length: stat.Size(),
Handler: progressHandler,
},
diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go
index f841a6d..6ce6799 100644
--- a/lxd/daemon_images.go
+++ b/lxd/daemon_images.go
@@ -16,6 +16,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/ioprogress"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -359,9 +360,9 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce
ctype = "application/octet-stream"
}
- body := &shared.ProgressReader{
+ body := &ioprogress.ProgressReader{
ReadCloser: raw.Body,
- Tracker: &shared.ProgressTracker{
+ Tracker: &ioprogress.ProgressTracker{
Length: raw.ContentLength,
Handler: progress,
},
diff --git a/lxd/storage.go b/lxd/storage.go
index 2ae706d..3edf294 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -14,6 +14,7 @@ import (
"github.com/gorilla/websocket"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/ioprogress"
"github.com/lxc/lxd/shared/logging"
log "gopkg.in/inconshreveable/log15.v2"
@@ -833,9 +834,9 @@ func StorageProgressReader(op *operation, key string, description string) func(i
progressWrapperRender(op, key, description, progressInt, speedInt)
}
- readPipe := &shared.ProgressReader{
+ readPipe := &ioprogress.ProgressReader{
ReadCloser: reader,
- Tracker: &shared.ProgressTracker{
+ Tracker: &ioprogress.ProgressTracker{
Handler: progress,
},
}
@@ -854,9 +855,9 @@ func StorageProgressWriter(op *operation, key string, description string) func(i
progressWrapperRender(op, key, description, progressInt, speedInt)
}
- writePipe := &shared.ProgressWriter{
+ writePipe := &ioprogress.ProgressWriter{
WriteCloser: writer,
- Tracker: &shared.ProgressTracker{
+ Tracker: &ioprogress.ProgressTracker{
Handler: progress,
},
}
diff --git a/shared/ioprogress/reader.go b/shared/ioprogress/reader.go
new file mode 100644
index 0000000..262aa40
--- /dev/null
+++ b/shared/ioprogress/reader.go
@@ -0,0 +1,23 @@
+package ioprogress
+
+import (
+ "io"
+)
+
+type ProgressReader struct {
+ io.ReadCloser
+ Tracker *ProgressTracker
+}
+
+func (pt *ProgressReader) Read(p []byte) (int, error) {
+ // Do normal reader tasks
+ n, err := pt.ReadCloser.Read(p)
+
+ // Do the actual progress tracking
+ if pt.Tracker != nil {
+ pt.Tracker.total += int64(n)
+ pt.Tracker.Update(n)
+ }
+
+ return n, err
+}
diff --git a/shared/ioprogress/tracker.go b/shared/ioprogress/tracker.go
new file mode 100644
index 0000000..78c730b
--- /dev/null
+++ b/shared/ioprogress/tracker.go
@@ -0,0 +1,76 @@
+package ioprogress
+
+import (
+ "time"
+)
+
+type ProgressTracker struct {
+ Length int64
+ Handler func(int64, int64)
+
+ percentage float64
+ total int64
+ start *time.Time
+ last *time.Time
+}
+
+func (pt *ProgressTracker) Update(n int) {
+ // Skip the rest if no handler attached
+ if pt.Handler == nil {
+ return
+ }
+
+ // Initialize start time if needed
+ if pt.start == nil {
+ cur := time.Now()
+ pt.start = &cur
+ pt.last = pt.start
+ }
+
+ // Skip if no data to count
+ if n <= 0 {
+ return
+ }
+
+ // Update interval handling
+ var percentage float64
+ if pt.Length > 0 {
+ // If running in relative mode, check that we increased by at least 1%
+ percentage = float64(pt.total) / float64(pt.Length) * float64(100)
+ if percentage-pt.percentage < 0.9 {
+ return
+ }
+ } else {
+ // If running in absolute mode, check that at least a second elapsed
+ interval := time.Since(*pt.last).Seconds()
+ if interval < 1 {
+ return
+ }
+ }
+
+ // Determine speed
+ speedInt := int64(0)
+ duration := time.Since(*pt.start).Seconds()
+ if duration > 0 {
+ speed := float64(pt.total) / duration
+ speedInt = int64(speed)
+ }
+
+ // Determine progress
+ progressInt := int64(0)
+ if pt.Length > 0 {
+ pt.percentage = percentage
+ progressInt = int64(1 - (int(percentage) % 1) + int(percentage))
+ if progressInt > 100 {
+ progressInt = 100
+ }
+ } else {
+ progressInt = pt.total
+
+ // Update timestamp
+ cur := time.Now()
+ pt.last = &cur
+ }
+
+ pt.Handler(progressInt, speedInt)
+}
diff --git a/shared/ioprogress/writer.go b/shared/ioprogress/writer.go
new file mode 100644
index 0000000..708911b
--- /dev/null
+++ b/shared/ioprogress/writer.go
@@ -0,0 +1,23 @@
+package ioprogress
+
+import (
+ "io"
+)
+
+type ProgressWriter struct {
+ io.WriteCloser
+ Tracker *ProgressTracker
+}
+
+func (pt *ProgressWriter) Write(p []byte) (int, error) {
+ // Do normal writer tasks
+ n, err := pt.WriteCloser.Write(p)
+
+ // Do the actual progress tracking
+ if pt.Tracker != nil {
+ pt.Tracker.total += int64(n)
+ pt.Tracker.Update(n)
+ }
+
+ return n, err
+}
diff --git a/shared/simplestreams.go b/shared/simplestreams.go
index 5bc9ab1..8f1ccfa 100644
--- a/shared/simplestreams.go
+++ b/shared/simplestreams.go
@@ -13,6 +13,8 @@ import (
"sort"
"strings"
"time"
+
+ "github.com/lxc/lxd/shared/ioprogress"
)
type ssSortImage []ImageInfo
@@ -535,9 +537,9 @@ func (s *SimpleStreams) downloadFile(path string, hash string, target string, pr
return fmt.Errorf("invalid simplestreams source: got %d looking for %s", resp.StatusCode, path)
}
- body := &ProgressReader{
+ body := &ioprogress.ProgressReader{
ReadCloser: resp.Body,
- Tracker: &ProgressTracker{
+ Tracker: &ioprogress.ProgressTracker{
Length: resp.ContentLength,
Handler: progress,
},
diff --git a/shared/util.go b/shared/util.go
index 8651c9d..cb28ba3 100644
--- a/shared/util.go
+++ b/shared/util.go
@@ -19,7 +19,6 @@ import (
"regexp"
"strconv"
"strings"
- "time"
)
const SnapshotDelimiter = "/"
@@ -738,113 +737,6 @@ func RemoveDuplicatesFromString(s string, sep string) string {
return s
}
-type ProgressTracker struct {
- Length int64
- Handler func(int64, int64)
-
- percentage float64
- total int64
- start *time.Time
- last *time.Time
-}
-
-func (pt *ProgressTracker) Update(n int) {
- // Skip the rest if no handler attached
- if pt.Handler == nil {
- return
- }
-
- // Initialize start time if needed
- if pt.start == nil {
- cur := time.Now()
- pt.start = &cur
- pt.last = pt.start
- }
-
- // Skip if no data to count
- if n <= 0 {
- return
- }
-
- // Update interval handling
- var percentage float64
- if pt.Length > 0 {
- // If running in relative mode, check that we increased by at least 1%
- percentage = float64(pt.total) / float64(pt.Length) * float64(100)
- if percentage-pt.percentage < 0.9 {
- return
- }
- } else {
- // If running in absolute mode, check that at least a second elapsed
- interval := time.Since(*pt.last).Seconds()
- if interval < 1 {
- return
- }
- }
-
- // Determine speed
- speedInt := int64(0)
- duration := time.Since(*pt.start).Seconds()
- if duration > 0 {
- speed := float64(pt.total) / duration
- speedInt = int64(speed)
- }
-
- // Determine progress
- progressInt := int64(0)
- if pt.Length > 0 {
- pt.percentage = percentage
- progressInt = int64(1 - (int(percentage) % 1) + int(percentage))
- if progressInt > 100 {
- progressInt = 100
- }
- } else {
- progressInt = pt.total
-
- // Update timestamp
- cur := time.Now()
- pt.last = &cur
- }
-
- pt.Handler(progressInt, speedInt)
-}
-
-type ProgressReader struct {
- io.ReadCloser
- Tracker *ProgressTracker
-}
-
-func (pt *ProgressReader) Read(p []byte) (int, error) {
- // Do normal reader tasks
- n, err := pt.ReadCloser.Read(p)
-
- // Do the actual progress tracking
- if pt.Tracker != nil {
- pt.Tracker.total += int64(n)
- pt.Tracker.Update(n)
- }
-
- return n, err
-}
-
-type ProgressWriter struct {
- io.WriteCloser
- Tracker *ProgressTracker
-}
-
-func (pt *ProgressWriter) Write(p []byte) (int, error) {
- // Do normal writer tasks
- n, err := pt.WriteCloser.Write(p)
-
- // Do the actual progress tracking
- if pt.Tracker != nil {
- pt.Tracker.total += int64(n)
- pt.Tracker.Update(n)
- }
-
- return n, err
-}
-
func RunCommand(name string, arg ...string) error {
output, err := exec.Command(name, arg...).CombinedOutput()
if err != nil {
From 9ad2a56d0f8697dabb9859a391d5f01c85364025 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 15 Dec 2016 17:19:29 -0500
Subject: [PATCH 3/5] shared: Give simplestreams client its own package
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client.go | 5 +-
lxd/daemon_images.go | 9 +-
shared/simplestreams.go | 666 ---------------------------------
shared/simplestreams/simplestreams.go | 667 ++++++++++++++++++++++++++++++++++
4 files changed, 675 insertions(+), 672 deletions(-)
delete mode 100644 shared/simplestreams.go
create mode 100644 shared/simplestreams/simplestreams.go
diff --git a/client.go b/client.go
index f7eb6af..0ce204e 100644
--- a/client.go
+++ b/client.go
@@ -25,6 +25,7 @@ import (
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/ioprogress"
+ "github.com/lxc/lxd/shared/simplestreams"
)
// Client can talk to a LXD daemon.
@@ -39,7 +40,7 @@ type Client struct {
Http http.Client
websocketDialer websocket.Dialer
- simplestreams *shared.SimpleStreams
+ simplestreams *simplestreams.SimpleStreams
}
type ResponseType string
@@ -338,7 +339,7 @@ func NewClientFromInfo(info ConnectInfo) (*Client, error) {
}
if info.RemoteConfig.Protocol == "simplestreams" {
- ss, err := shared.SimpleStreamsClient(c.Remote.Addr, shared.ProxyFromEnvironment)
+ ss, err := simplestreams.SimpleStreamsClient(c.Remote.Addr, shared.ProxyFromEnvironment)
if err != nil {
return nil, err
}
diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go
index 6ce6799..5928233 100644
--- a/lxd/daemon_images.go
+++ b/lxd/daemon_images.go
@@ -17,6 +17,7 @@ import (
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/ioprogress"
+ "github.com/lxc/lxd/shared/simplestreams"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -26,7 +27,7 @@ type imageStreamCacheEntry struct {
Aliases shared.ImageAliases `yaml:"aliases"`
Fingerprints []string `yaml:"fingerprints"`
expiry time.Time
- ss *shared.SimpleStreams
+ ss *simplestreams.SimpleStreams
}
var imageStreamCache = map[string]*imageStreamCacheEntry{}
@@ -66,7 +67,7 @@ func imageLoadStreamCache(d *Daemon) error {
for url, entry := range imageStreamCache {
if entry.ss == nil {
- ss, err := shared.SimpleStreamsClient(url, d.proxy)
+ ss, err := simplestreams.SimpleStreamsClient(url, d.proxy)
if err != nil {
return err
}
@@ -82,7 +83,7 @@ func imageLoadStreamCache(d *Daemon) error {
// downloads the image from a remote server.
func (d *Daemon) ImageDownload(op *operation, server string, protocol string, certificate string, secret string, alias string, forContainer bool, autoUpdate bool) (string, error) {
var err error
- var ss *shared.SimpleStreams
+ var ss *simplestreams.SimpleStreams
var ctxMap log.Ctx
if protocol == "" {
@@ -98,7 +99,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce
if entry == nil || entry.expiry.Before(time.Now()) {
refresh := func() (*imageStreamCacheEntry, error) {
// Setup simplestreams client
- ss, err = shared.SimpleStreamsClient(server, d.proxy)
+ ss, err = simplestreams.SimpleStreamsClient(server, d.proxy)
if err != nil {
return nil, err
}
diff --git a/shared/simplestreams.go b/shared/simplestreams.go
deleted file mode 100644
index 8f1ccfa..0000000
--- a/shared/simplestreams.go
+++ /dev/null
@@ -1,666 +0,0 @@
-package shared
-
-import (
- "crypto/sha256"
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "net/url"
- "os"
- "path/filepath"
- "sort"
- "strings"
- "time"
-
- "github.com/lxc/lxd/shared/ioprogress"
-)
-
-type ssSortImage []ImageInfo
-
-func (a ssSortImage) Len() int {
- return len(a)
-}
-
-func (a ssSortImage) Swap(i, j int) {
- a[i], a[j] = a[j], a[i]
-}
-
-func (a ssSortImage) Less(i, j int) bool {
- if a[i].Properties["os"] == a[j].Properties["os"] {
- if a[i].Properties["release"] == a[j].Properties["release"] {
- if a[i].CreationDate.UTC().Unix() == 0 {
- return true
- }
-
- if a[j].CreationDate.UTC().Unix() == 0 {
- return false
- }
-
- return a[i].CreationDate.UTC().Unix() > a[j].CreationDate.UTC().Unix()
- }
-
- if a[i].Properties["release"] == "" {
- return false
- }
-
- if a[j].Properties["release"] == "" {
- return true
- }
-
- return a[i].Properties["release"] < a[j].Properties["release"]
- }
-
- if a[i].Properties["os"] == "" {
- return false
- }
-
- if a[j].Properties["os"] == "" {
- return true
- }
-
- return a[i].Properties["os"] < a[j].Properties["os"]
-}
-
-var ssDefaultOS = map[string]string{
- "https://cloud-images.ubuntu.com": "ubuntu",
-}
-
-type SimpleStreamsManifest struct {
- Updated string `json:"updated"`
- DataType string `json:"datatype"`
- Format string `json:"format"`
- License string `json:"license"`
- Products map[string]SimpleStreamsManifestProduct `json:"products"`
-}
-
-func (s *SimpleStreamsManifest) ToLXD() ([]ImageInfo, map[string][][]string) {
- downloads := map[string][][]string{}
-
- images := []ImageInfo{}
- nameLayout := "20060102"
- eolLayout := "2006-01-02"
-
- for _, product := range s.Products {
- // Skip unsupported architectures
- architecture, err := ArchitectureId(product.Architecture)
- if err != nil {
- continue
- }
-
- architectureName, err := ArchitectureName(architecture)
- if err != nil {
- continue
- }
-
- for name, version := range product.Versions {
- // Short of anything better, use the name as date (see format above)
- if len(name) < 8 {
- continue
- }
-
- creationDate, err := time.Parse(nameLayout, name[0:8])
- if err != nil {
- continue
- }
-
- size := int64(0)
- filename := ""
- fingerprint := ""
-
- metaPath := ""
- metaHash := ""
- rootfsPath := ""
- rootfsHash := ""
-
- found := 0
- for _, item := range version.Items {
- // Skip the files we don't care about
- if !StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "squashfs"}) {
- continue
- }
- found += 1
-
- if fingerprint == "" {
- if item.LXDHashSha256SquashFs != "" {
- fingerprint = item.LXDHashSha256SquashFs
- } else if item.LXDHashSha256RootXz != "" {
- fingerprint = item.LXDHashSha256RootXz
- } else if item.LXDHashSha256 != "" {
- fingerprint = item.LXDHashSha256
- }
- }
-
- if item.FileType == "lxd.tar.xz" {
- fields := strings.Split(item.Path, "/")
- filename = fields[len(fields)-1]
- metaPath = item.Path
- metaHash = item.HashSha256
-
- size += item.Size
- }
-
- if rootfsPath == "" || rootfsHash == "" {
- if item.FileType == "squashfs" {
- rootfsPath = item.Path
- rootfsHash = item.HashSha256
- }
-
- if item.FileType == "root.tar.xz" {
- rootfsPath = item.Path
- rootfsHash = item.HashSha256
- }
-
- size += item.Size
- }
- }
-
- if found < 2 || size == 0 || filename == "" || fingerprint == "" {
- // Invalid image
- continue
- }
-
- // Generate the actual image entry
- description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
- if version.Label != "" {
- description = fmt.Sprintf("%s (%s)", description, version.Label)
- }
- description = fmt.Sprintf("%s (%s)", description, name)
-
- image := ImageInfo{}
- image.Architecture = architectureName
- image.Public = true
- image.Size = size
- image.CreationDate = creationDate
- image.UploadDate = creationDate
- image.Filename = filename
- image.Fingerprint = fingerprint
- image.Properties = map[string]string{
- "os": product.OperatingSystem,
- "release": product.Release,
- "version": product.Version,
- "architecture": product.Architecture,
- "label": version.Label,
- "serial": name,
- "description": description,
- }
-
- // Add the provided aliases
- if product.Aliases != "" {
- image.Aliases = []ImageAlias{}
- for _, entry := range strings.Split(product.Aliases, ",") {
- image.Aliases = append(image.Aliases, ImageAlias{Name: entry})
- }
- }
-
- // Clear unset properties
- for k, v := range image.Properties {
- if v == "" {
- delete(image.Properties, k)
- }
- }
-
- // Attempt to parse the EOL
- image.ExpiryDate = time.Unix(0, 0).UTC()
- if product.SupportedEOL != "" {
- eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
- if err == nil {
- image.ExpiryDate = eolDate
- }
- }
-
- downloads[fingerprint] = [][]string{[]string{metaPath, metaHash, "meta"}, []string{rootfsPath, rootfsHash, "root"}}
- images = append(images, image)
- }
- }
-
- return images, downloads
-}
-
-type SimpleStreamsManifestProduct struct {
- Aliases string `json:"aliases"`
- Architecture string `json:"arch"`
- OperatingSystem string `json:"os"`
- Release string `json:"release"`
- ReleaseCodename string `json:"release_codename"`
- ReleaseTitle string `json:"release_title"`
- Supported bool `json:"supported"`
- SupportedEOL string `json:"support_eol"`
- Version string `json:"version"`
- Versions map[string]SimpleStreamsManifestProductVersion `json:"versions"`
-}
-
-type SimpleStreamsManifestProductVersion struct {
- PublicName string `json:"pubname"`
- Label string `json:"label"`
- Items map[string]SimpleStreamsManifestProductVersionItem `json:"items"`
-}
-
-type SimpleStreamsManifestProductVersionItem struct {
- Path string `json:"path"`
- FileType string `json:"ftype"`
- HashMd5 string `json:"md5"`
- HashSha256 string `json:"sha256"`
- LXDHashSha256 string `json:"combined_sha256"`
- LXDHashSha256RootXz string `json:"combined_rootxz_sha256"`
- LXDHashSha256SquashFs string `json:"combined_squashfs_sha256"`
- Size int64 `json:"size"`
-}
-
-type SimpleStreamsIndex struct {
- Format string `json:"format"`
- Index map[string]SimpleStreamsIndexStream `json:"index"`
- Updated string `json:"updated"`
-}
-
-type SimpleStreamsIndexStream struct {
- Updated string `json:"updated"`
- DataType string `json:"datatype"`
- Path string `json:"path"`
- Products []string `json:"products"`
-}
-
-func SimpleStreamsClient(url string, proxy func(*http.Request) (*url.URL, error)) (*SimpleStreams, error) {
- // Setup a http client
- tlsConfig, err := GetTLSConfig("", "", "", nil)
- if err != nil {
- return nil, err
- }
-
- tr := &http.Transport{
- TLSClientConfig: tlsConfig,
- Dial: RFC3493Dialer,
- Proxy: proxy,
- }
-
- myHttp := http.Client{
- Transport: tr,
- }
-
- return &SimpleStreams{
- http: &myHttp,
- url: url,
- cachedManifest: map[string]*SimpleStreamsManifest{}}, nil
-}
-
-type SimpleStreams struct {
- http *http.Client
- url string
-
- cachedIndex *SimpleStreamsIndex
- cachedManifest map[string]*SimpleStreamsManifest
- cachedImages []ImageInfo
- cachedAliases map[string]*ImageAliasesEntry
-}
-
-func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) {
- if s.cachedIndex != nil {
- return s.cachedIndex, nil
- }
-
- req, err := http.NewRequest("GET", fmt.Sprintf("%s/streams/v1/index.json", s.url), nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set("User-Agent", UserAgent)
-
- r, err := s.http.Do(req)
- if err != nil {
- return nil, err
- }
- defer r.Body.Close()
-
- body, err := ioutil.ReadAll(r.Body)
- if err != nil {
- return nil, err
- }
-
- // Parse the idnex
- ssIndex := SimpleStreamsIndex{}
- err = json.Unmarshal(body, &ssIndex)
- if err != nil {
- return nil, err
- }
-
- s.cachedIndex = &ssIndex
-
- return &ssIndex, nil
-}
-
-func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) {
- if s.cachedManifest[path] != nil {
- return s.cachedManifest[path], nil
- }
-
- req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", s.url, path), nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set("User-Agent", UserAgent)
-
- r, err := s.http.Do(req)
- if err != nil {
- return nil, err
- }
- defer r.Body.Close()
-
- body, err := ioutil.ReadAll(r.Body)
- if err != nil {
- return nil, err
- }
-
- // Parse the idnex
- ssManifest := SimpleStreamsManifest{}
- err = json.Unmarshal(body, &ssManifest)
- if err != nil {
- return nil, err
- }
-
- s.cachedManifest[path] = &ssManifest
-
- return &ssManifest, nil
-}
-
-func (s *SimpleStreams) applyAliases(images []ImageInfo) ([]ImageInfo, map[string]*ImageAliasesEntry, error) {
- aliases := map[string]*ImageAliasesEntry{}
-
- sort.Sort(ssSortImage(images))
-
- defaultOS := ""
- for k, v := range ssDefaultOS {
- if strings.HasPrefix(s.url, k) {
- defaultOS = v
- break
- }
- }
-
- addAlias := func(name string, fingerprint string) *ImageAlias {
- if defaultOS != "" {
- name = strings.TrimPrefix(name, fmt.Sprintf("%s/", defaultOS))
- }
-
- if aliases[name] != nil {
- return nil
- }
-
- alias := ImageAliasesEntry{}
- alias.Name = name
- alias.Target = fingerprint
- aliases[name] = &alias
-
- return &ImageAlias{Name: name}
- }
-
- architectureName, _ := ArchitectureGetLocal()
-
- newImages := []ImageInfo{}
- for _, image := range images {
- if image.Aliases != nil {
- // Build a new list of aliases from the provided ones
- aliases := image.Aliases
- image.Aliases = nil
-
- for _, entry := range aliases {
- // Short
- if image.Architecture == architectureName {
- alias := addAlias(fmt.Sprintf("%s", entry.Name), image.Fingerprint)
- if alias != nil {
- image.Aliases = append(image.Aliases, *alias)
- }
- }
-
- // Medium
- alias := addAlias(fmt.Sprintf("%s/%s", entry.Name, image.Properties["architecture"]), image.Fingerprint)
- if alias != nil {
- image.Aliases = append(image.Aliases, *alias)
- }
- }
- }
-
- newImages = append(newImages, image)
- }
-
- return newImages, aliases, nil
-}
-
-func (s *SimpleStreams) getImages() ([]ImageInfo, map[string]*ImageAliasesEntry, error) {
- if s.cachedImages != nil && s.cachedAliases != nil {
- return s.cachedImages, s.cachedAliases, nil
- }
-
- images := []ImageInfo{}
-
- // Load the main index
- ssIndex, err := s.parseIndex()
- if err != nil {
- return nil, nil, err
- }
-
- // Iterate through the various image manifests
- for _, entry := range ssIndex.Index {
- // We only care about images
- if entry.DataType != "image-downloads" {
- continue
- }
-
- // No point downloading an empty image list
- if len(entry.Products) == 0 {
- continue
- }
-
- manifest, err := s.parseManifest(entry.Path)
- if err != nil {
- return nil, nil, err
- }
-
- manifestImages, _ := manifest.ToLXD()
-
- for _, image := range manifestImages {
- images = append(images, image)
- }
- }
-
- // Setup the aliases
- images, aliases, err := s.applyAliases(images)
- if err != nil {
- return nil, nil, err
- }
-
- s.cachedImages = images
- s.cachedAliases = aliases
-
- return images, aliases, nil
-}
-
-func (s *SimpleStreams) getPaths(fingerprint string) ([][]string, error) {
- // Load the main index
- ssIndex, err := s.parseIndex()
- if err != nil {
- return nil, err
- }
-
- // Iterate through the various image manifests
- for _, entry := range ssIndex.Index {
- // We only care about images
- if entry.DataType != "image-downloads" {
- continue
- }
-
- // No point downloading an empty image list
- if len(entry.Products) == 0 {
- continue
- }
-
- manifest, err := s.parseManifest(entry.Path)
- if err != nil {
- return nil, err
- }
-
- manifestImages, downloads := manifest.ToLXD()
-
- for _, image := range manifestImages {
- if strings.HasPrefix(image.Fingerprint, fingerprint) {
- urls := [][]string{}
- for _, path := range downloads[image.Fingerprint] {
- urls = append(urls, []string{path[0], path[1], path[2]})
- }
- return urls, nil
- }
- }
- }
-
- return nil, fmt.Errorf("Couldn't find the requested image")
-}
-
-func (s *SimpleStreams) downloadFile(path string, hash string, target string, progress func(int64, int64)) error {
- download := func(url string, hash string, target string) error {
- out, err := os.Create(target)
- if err != nil {
- return err
- }
- defer out.Close()
-
- req, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return err
- }
- req.Header.Set("User-Agent", UserAgent)
-
- resp, err := s.http.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("invalid simplestreams source: got %d looking for %s", resp.StatusCode, path)
- }
-
- body := &ioprogress.ProgressReader{
- ReadCloser: resp.Body,
- Tracker: &ioprogress.ProgressTracker{
- Length: resp.ContentLength,
- Handler: progress,
- },
- }
-
- sha256 := sha256.New()
- _, err = io.Copy(io.MultiWriter(out, sha256), body)
- if err != nil {
- return err
- }
-
- result := fmt.Sprintf("%x", sha256.Sum(nil))
- if result != hash {
- os.Remove(target)
- return fmt.Errorf("Hash mismatch for %s: %s != %s", path, result, hash)
- }
-
- return nil
- }
-
- // Try http first
- if strings.HasPrefix(s.url, "https://") {
- err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, target)
- if err == nil {
- return nil
- }
- }
-
- err := download(fmt.Sprintf("%s/%s", s.url, path), hash, target)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *SimpleStreams) ListAliases() (ImageAliases, error) {
- _, aliasesMap, err := s.getImages()
- if err != nil {
- return nil, err
- }
-
- aliases := ImageAliases{}
-
- for _, alias := range aliasesMap {
- aliases = append(aliases, *alias)
- }
-
- return aliases, nil
-}
-
-func (s *SimpleStreams) ListImages() ([]ImageInfo, error) {
- images, _, err := s.getImages()
- return images, err
-}
-
-func (s *SimpleStreams) GetAlias(name string) string {
- _, aliasesMap, err := s.getImages()
- if err != nil {
- return ""
- }
-
- alias, ok := aliasesMap[name]
- if !ok {
- return ""
- }
-
- return alias.Target
-}
-
-func (s *SimpleStreams) GetImageInfo(fingerprint string) (*ImageInfo, error) {
- images, _, err := s.getImages()
- if err != nil {
- return nil, err
- }
-
- for _, image := range images {
- if strings.HasPrefix(image.Fingerprint, fingerprint) {
- return &image, nil
- }
- }
-
- return nil, fmt.Errorf("The requested image couldn't be found.")
-}
-
-func (s *SimpleStreams) ExportImage(image string, target string) (string, error) {
- if !IsDir(target) {
- return "", fmt.Errorf("Split images can only be written to a directory.")
- }
-
- paths, err := s.getPaths(image)
- if err != nil {
- return "", err
- }
-
- for _, path := range paths {
- fields := strings.Split(path[0], "/")
- targetFile := filepath.Join(target, fields[len(fields)-1])
-
- err := s.downloadFile(path[0], path[1], targetFile, nil)
- if err != nil {
- return "", err
- }
- }
-
- return target, nil
-}
-
-func (s *SimpleStreams) Download(image string, file string, target string, progress func(int64, int64)) error {
- paths, err := s.getPaths(image)
- if err != nil {
- return err
- }
-
- for _, path := range paths {
- if file != path[2] {
- continue
- }
-
- return s.downloadFile(path[0], path[1], target, progress)
- }
-
- return fmt.Errorf("The file couldn't be found.")
-}
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
new file mode 100644
index 0000000..b0f9ee3
--- /dev/null
+++ b/shared/simplestreams/simplestreams.go
@@ -0,0 +1,667 @@
+package simplestreams
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/ioprogress"
+)
+
+type ssSortImage []shared.ImageInfo
+
+func (a ssSortImage) Len() int {
+ return len(a)
+}
+
+func (a ssSortImage) Swap(i, j int) {
+ a[i], a[j] = a[j], a[i]
+}
+
+func (a ssSortImage) Less(i, j int) bool {
+ if a[i].Properties["os"] == a[j].Properties["os"] {
+ if a[i].Properties["release"] == a[j].Properties["release"] {
+ if a[i].CreationDate.UTC().Unix() == 0 {
+ return true
+ }
+
+ if a[j].CreationDate.UTC().Unix() == 0 {
+ return false
+ }
+
+ return a[i].CreationDate.UTC().Unix() > a[j].CreationDate.UTC().Unix()
+ }
+
+ if a[i].Properties["release"] == "" {
+ return false
+ }
+
+ if a[j].Properties["release"] == "" {
+ return true
+ }
+
+ return a[i].Properties["release"] < a[j].Properties["release"]
+ }
+
+ if a[i].Properties["os"] == "" {
+ return false
+ }
+
+ if a[j].Properties["os"] == "" {
+ return true
+ }
+
+ return a[i].Properties["os"] < a[j].Properties["os"]
+}
+
+var ssDefaultOS = map[string]string{
+ "https://cloud-images.ubuntu.com": "ubuntu",
+}
+
+type SimpleStreamsManifest struct {
+ Updated string `json:"updated"`
+ DataType string `json:"datatype"`
+ Format string `json:"format"`
+ License string `json:"license"`
+ Products map[string]SimpleStreamsManifestProduct `json:"products"`
+}
+
+func (s *SimpleStreamsManifest) ToLXD() ([]shared.ImageInfo, map[string][][]string) {
+ downloads := map[string][][]string{}
+
+ images := []shared.ImageInfo{}
+ nameLayout := "20060102"
+ eolLayout := "2006-01-02"
+
+ for _, product := range s.Products {
+ // Skip unsupported architectures
+ architecture, err := shared.ArchitectureId(product.Architecture)
+ if err != nil {
+ continue
+ }
+
+ architectureName, err := shared.ArchitectureName(architecture)
+ if err != nil {
+ continue
+ }
+
+ for name, version := range product.Versions {
+ // Short of anything better, use the name as date (see format above)
+ if len(name) < 8 {
+ continue
+ }
+
+ creationDate, err := time.Parse(nameLayout, name[0:8])
+ if err != nil {
+ continue
+ }
+
+ size := int64(0)
+ filename := ""
+ fingerprint := ""
+
+ metaPath := ""
+ metaHash := ""
+ rootfsPath := ""
+ rootfsHash := ""
+
+ found := 0
+ for _, item := range version.Items {
+ // Skip the files we don't care about
+ if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "squashfs"}) {
+ continue
+ }
+ found += 1
+
+ if fingerprint == "" {
+ if item.LXDHashSha256SquashFs != "" {
+ fingerprint = item.LXDHashSha256SquashFs
+ } else if item.LXDHashSha256RootXz != "" {
+ fingerprint = item.LXDHashSha256RootXz
+ } else if item.LXDHashSha256 != "" {
+ fingerprint = item.LXDHashSha256
+ }
+ }
+
+ if item.FileType == "lxd.tar.xz" {
+ fields := strings.Split(item.Path, "/")
+ filename = fields[len(fields)-1]
+ metaPath = item.Path
+ metaHash = item.HashSha256
+
+ size += item.Size
+ }
+
+ if rootfsPath == "" || rootfsHash == "" {
+ if item.FileType == "squashfs" {
+ rootfsPath = item.Path
+ rootfsHash = item.HashSha256
+ }
+
+ if item.FileType == "root.tar.xz" {
+ rootfsPath = item.Path
+ rootfsHash = item.HashSha256
+ }
+
+ size += item.Size
+ }
+ }
+
+ if found < 2 || size == 0 || filename == "" || fingerprint == "" {
+ // Invalid image
+ continue
+ }
+
+ // Generate the actual image entry
+ description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
+ if version.Label != "" {
+ description = fmt.Sprintf("%s (%s)", description, version.Label)
+ }
+ description = fmt.Sprintf("%s (%s)", description, name)
+
+ image := shared.ImageInfo{}
+ image.Architecture = architectureName
+ image.Public = true
+ image.Size = size
+ image.CreationDate = creationDate
+ image.UploadDate = creationDate
+ image.Filename = filename
+ image.Fingerprint = fingerprint
+ image.Properties = map[string]string{
+ "os": product.OperatingSystem,
+ "release": product.Release,
+ "version": product.Version,
+ "architecture": product.Architecture,
+ "label": version.Label,
+ "serial": name,
+ "description": description,
+ }
+
+ // Add the provided aliases
+ if product.Aliases != "" {
+ image.Aliases = []shared.ImageAlias{}
+ for _, entry := range strings.Split(product.Aliases, ",") {
+ image.Aliases = append(image.Aliases, shared.ImageAlias{Name: entry})
+ }
+ }
+
+ // Clear unset properties
+ for k, v := range image.Properties {
+ if v == "" {
+ delete(image.Properties, k)
+ }
+ }
+
+ // Attempt to parse the EOL
+ image.ExpiryDate = time.Unix(0, 0).UTC()
+ if product.SupportedEOL != "" {
+ eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
+ if err == nil {
+ image.ExpiryDate = eolDate
+ }
+ }
+
+ downloads[fingerprint] = [][]string{[]string{metaPath, metaHash, "meta"}, []string{rootfsPath, rootfsHash, "root"}}
+ images = append(images, image)
+ }
+ }
+
+ return images, downloads
+}
+
+type SimpleStreamsManifestProduct struct {
+ Aliases string `json:"aliases"`
+ Architecture string `json:"arch"`
+ OperatingSystem string `json:"os"`
+ Release string `json:"release"`
+ ReleaseCodename string `json:"release_codename"`
+ ReleaseTitle string `json:"release_title"`
+ Supported bool `json:"supported"`
+ SupportedEOL string `json:"support_eol"`
+ Version string `json:"version"`
+ Versions map[string]SimpleStreamsManifestProductVersion `json:"versions"`
+}
+
+type SimpleStreamsManifestProductVersion struct {
+ PublicName string `json:"pubname"`
+ Label string `json:"label"`
+ Items map[string]SimpleStreamsManifestProductVersionItem `json:"items"`
+}
+
+type SimpleStreamsManifestProductVersionItem struct {
+ Path string `json:"path"`
+ FileType string `json:"ftype"`
+ HashMd5 string `json:"md5"`
+ HashSha256 string `json:"sha256"`
+ LXDHashSha256 string `json:"combined_sha256"`
+ LXDHashSha256RootXz string `json:"combined_rootxz_sha256"`
+ LXDHashSha256SquashFs string `json:"combined_squashfs_sha256"`
+ Size int64 `json:"size"`
+}
+
+type SimpleStreamsIndex struct {
+ Format string `json:"format"`
+ Index map[string]SimpleStreamsIndexStream `json:"index"`
+ Updated string `json:"updated"`
+}
+
+type SimpleStreamsIndexStream struct {
+ Updated string `json:"updated"`
+ DataType string `json:"datatype"`
+ Path string `json:"path"`
+ Products []string `json:"products"`
+}
+
+func SimpleStreamsClient(url string, proxy func(*http.Request) (*url.URL, error)) (*SimpleStreams, error) {
+ // Setup a http client
+ tlsConfig, err := shared.GetTLSConfig("", "", "", nil)
+ if err != nil {
+ return nil, err
+ }
+
+ tr := &http.Transport{
+ TLSClientConfig: tlsConfig,
+ Dial: shared.RFC3493Dialer,
+ Proxy: proxy,
+ }
+
+ myHttp := http.Client{
+ Transport: tr,
+ }
+
+ return &SimpleStreams{
+ http: &myHttp,
+ url: url,
+ cachedManifest: map[string]*SimpleStreamsManifest{}}, nil
+}
+
+type SimpleStreams struct {
+ http *http.Client
+ url string
+
+ cachedIndex *SimpleStreamsIndex
+ cachedManifest map[string]*SimpleStreamsManifest
+ cachedImages []shared.ImageInfo
+ cachedAliases map[string]*shared.ImageAliasesEntry
+}
+
+func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) {
+ if s.cachedIndex != nil {
+ return s.cachedIndex, nil
+ }
+
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/streams/v1/index.json", s.url), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", shared.UserAgent)
+
+ r, err := s.http.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Body.Close()
+
+ body, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ // Parse the idnex
+ ssIndex := SimpleStreamsIndex{}
+ err = json.Unmarshal(body, &ssIndex)
+ if err != nil {
+ return nil, err
+ }
+
+ s.cachedIndex = &ssIndex
+
+ return &ssIndex, nil
+}
+
+func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) {
+ if s.cachedManifest[path] != nil {
+ return s.cachedManifest[path], nil
+ }
+
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", s.url, path), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", shared.UserAgent)
+
+ r, err := s.http.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Body.Close()
+
+ body, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ // Parse the idnex
+ ssManifest := SimpleStreamsManifest{}
+ err = json.Unmarshal(body, &ssManifest)
+ if err != nil {
+ return nil, err
+ }
+
+ s.cachedManifest[path] = &ssManifest
+
+ return &ssManifest, nil
+}
+
+func (s *SimpleStreams) applyAliases(images []shared.ImageInfo) ([]shared.ImageInfo, map[string]*shared.ImageAliasesEntry, error) {
+ aliases := map[string]*shared.ImageAliasesEntry{}
+
+ sort.Sort(ssSortImage(images))
+
+ defaultOS := ""
+ for k, v := range ssDefaultOS {
+ if strings.HasPrefix(s.url, k) {
+ defaultOS = v
+ break
+ }
+ }
+
+ addAlias := func(name string, fingerprint string) *shared.ImageAlias {
+ if defaultOS != "" {
+ name = strings.TrimPrefix(name, fmt.Sprintf("%s/", defaultOS))
+ }
+
+ if aliases[name] != nil {
+ return nil
+ }
+
+ alias := shared.ImageAliasesEntry{}
+ alias.Name = name
+ alias.Target = fingerprint
+ aliases[name] = &alias
+
+ return &shared.ImageAlias{Name: name}
+ }
+
+ architectureName, _ := shared.ArchitectureGetLocal()
+
+ newImages := []shared.ImageInfo{}
+ for _, image := range images {
+ if image.Aliases != nil {
+ // Build a new list of aliases from the provided ones
+ aliases := image.Aliases
+ image.Aliases = nil
+
+ for _, entry := range aliases {
+ // Short
+ if image.Architecture == architectureName {
+ alias := addAlias(fmt.Sprintf("%s", entry.Name), image.Fingerprint)
+ if alias != nil {
+ image.Aliases = append(image.Aliases, *alias)
+ }
+ }
+
+ // Medium
+ alias := addAlias(fmt.Sprintf("%s/%s", entry.Name, image.Properties["architecture"]), image.Fingerprint)
+ if alias != nil {
+ image.Aliases = append(image.Aliases, *alias)
+ }
+ }
+ }
+
+ newImages = append(newImages, image)
+ }
+
+ return newImages, aliases, nil
+}
+
+func (s *SimpleStreams) getImages() ([]shared.ImageInfo, map[string]*shared.ImageAliasesEntry, error) {
+ if s.cachedImages != nil && s.cachedAliases != nil {
+ return s.cachedImages, s.cachedAliases, nil
+ }
+
+ images := []shared.ImageInfo{}
+
+ // Load the main index
+ ssIndex, err := s.parseIndex()
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // Iterate through the various image manifests
+ for _, entry := range ssIndex.Index {
+ // We only care about images
+ if entry.DataType != "image-downloads" {
+ continue
+ }
+
+ // No point downloading an empty image list
+ if len(entry.Products) == 0 {
+ continue
+ }
+
+ manifest, err := s.parseManifest(entry.Path)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ manifestImages, _ := manifest.ToLXD()
+
+ for _, image := range manifestImages {
+ images = append(images, image)
+ }
+ }
+
+ // Setup the aliases
+ images, aliases, err := s.applyAliases(images)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ s.cachedImages = images
+ s.cachedAliases = aliases
+
+ return images, aliases, nil
+}
+
+func (s *SimpleStreams) getPaths(fingerprint string) ([][]string, error) {
+ // Load the main index
+ ssIndex, err := s.parseIndex()
+ if err != nil {
+ return nil, err
+ }
+
+ // Iterate through the various image manifests
+ for _, entry := range ssIndex.Index {
+ // We only care about images
+ if entry.DataType != "image-downloads" {
+ continue
+ }
+
+ // No point downloading an empty image list
+ if len(entry.Products) == 0 {
+ continue
+ }
+
+ manifest, err := s.parseManifest(entry.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ manifestImages, downloads := manifest.ToLXD()
+
+ for _, image := range manifestImages {
+ if strings.HasPrefix(image.Fingerprint, fingerprint) {
+ urls := [][]string{}
+ for _, path := range downloads[image.Fingerprint] {
+ urls = append(urls, []string{path[0], path[1], path[2]})
+ }
+ return urls, nil
+ }
+ }
+ }
+
+ return nil, fmt.Errorf("Couldn't find the requested image")
+}
+
+func (s *SimpleStreams) downloadFile(path string, hash string, target string, progress func(int64, int64)) error {
+ download := func(url string, hash string, target string) error {
+ out, err := os.Create(target)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("User-Agent", shared.UserAgent)
+
+ resp, err := s.http.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("invalid simplestreams source: got %d looking for %s", resp.StatusCode, path)
+ }
+
+ body := &ioprogress.ProgressReader{
+ ReadCloser: resp.Body,
+ Tracker: &ioprogress.ProgressTracker{
+ Length: resp.ContentLength,
+ Handler: progress,
+ },
+ }
+
+ sha256 := sha256.New()
+ _, err = io.Copy(io.MultiWriter(out, sha256), body)
+ if err != nil {
+ return err
+ }
+
+ result := fmt.Sprintf("%x", sha256.Sum(nil))
+ if result != hash {
+ os.Remove(target)
+ return fmt.Errorf("Hash mismatch for %s: %s != %s", path, result, hash)
+ }
+
+ return nil
+ }
+
+ // Try http first
+ if strings.HasPrefix(s.url, "https://") {
+ err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, target)
+ if err == nil {
+ return nil
+ }
+ }
+
+ err := download(fmt.Sprintf("%s/%s", s.url, path), hash, target)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *SimpleStreams) ListAliases() (shared.ImageAliases, error) {
+ _, aliasesMap, err := s.getImages()
+ if err != nil {
+ return nil, err
+ }
+
+ aliases := shared.ImageAliases{}
+
+ for _, alias := range aliasesMap {
+ aliases = append(aliases, *alias)
+ }
+
+ return aliases, nil
+}
+
+func (s *SimpleStreams) ListImages() ([]shared.ImageInfo, error) {
+ images, _, err := s.getImages()
+ return images, err
+}
+
+func (s *SimpleStreams) GetAlias(name string) string {
+ _, aliasesMap, err := s.getImages()
+ if err != nil {
+ return ""
+ }
+
+ alias, ok := aliasesMap[name]
+ if !ok {
+ return ""
+ }
+
+ return alias.Target
+}
+
+func (s *SimpleStreams) GetImageInfo(fingerprint string) (*shared.ImageInfo, error) {
+ images, _, err := s.getImages()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, image := range images {
+ if strings.HasPrefix(image.Fingerprint, fingerprint) {
+ return &image, nil
+ }
+ }
+
+ return nil, fmt.Errorf("The requested image couldn't be found.")
+}
+
+func (s *SimpleStreams) ExportImage(image string, target string) (string, error) {
+ if !shared.IsDir(target) {
+ return "", fmt.Errorf("Split images can only be written to a directory.")
+ }
+
+ paths, err := s.getPaths(image)
+ if err != nil {
+ return "", err
+ }
+
+ for _, path := range paths {
+ fields := strings.Split(path[0], "/")
+ targetFile := filepath.Join(target, fields[len(fields)-1])
+
+ err := s.downloadFile(path[0], path[1], targetFile, nil)
+ if err != nil {
+ return "", err
+ }
+ }
+
+ return target, nil
+}
+
+func (s *SimpleStreams) Download(image string, file string, target string, progress func(int64, int64)) error {
+ paths, err := s.getPaths(image)
+ if err != nil {
+ return err
+ }
+
+ for _, path := range paths {
+ if file != path[2] {
+ continue
+ }
+
+ return s.downloadFile(path[0], path[1], target, progress)
+ }
+
+ return fmt.Errorf("The file couldn't be found.")
+}
From a57aac3c27ef1c9d07939d5ebfd781790870d138 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 15 Dec 2016 17:54:53 -0500
Subject: [PATCH 4/5] shared: Give Architecture handling its own package
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/api_1.0.go | 3 +-
lxd/api_internal.go | 5 +-
lxd/container.go | 9 +--
lxd/container_lxc.go | 19 +++---
lxd/container_patch.go | 4 +-
lxd/container_put.go | 4 +-
lxd/containers_post.go | 8 ++-
lxd/daemon.go | 7 ++-
lxd/db_images.go | 7 ++-
lxd/images.go | 5 +-
lxd/seccomp.go | 3 +-
shared/architectures.go | 107 ----------------------------------
shared/architectures_linux.go | 24 --------
shared/architectures_others.go | 7 ---
shared/osarch/architectures.go | 107 ++++++++++++++++++++++++++++++++++
shared/osarch/architectures_linux.go | 24 ++++++++
shared/osarch/architectures_others.go | 7 +++
shared/simplestreams/simplestreams.go | 7 ++-
18 files changed, 186 insertions(+), 171 deletions(-)
delete mode 100644 shared/architectures.go
delete mode 100644 shared/architectures_linux.go
delete mode 100644 shared/architectures_others.go
create mode 100644 shared/osarch/architectures.go
create mode 100644 shared/osarch/architectures_linux.go
create mode 100644 shared/osarch/architectures_others.go
diff --git a/lxd/api_1.0.go b/lxd/api_1.0.go
index 8ffd973..3117106 100644
--- a/lxd/api_1.0.go
+++ b/lxd/api_1.0.go
@@ -11,6 +11,7 @@ import (
"gopkg.in/lxc/go-lxc.v2"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
)
var api10 = []Command{
@@ -139,7 +140,7 @@ func api10Get(d *Daemon, r *http.Request) Response {
architectures := []string{}
for _, architecture := range d.architectures {
- architectureName, err := shared.ArchitectureName(architecture)
+ architectureName, err := osarch.ArchitectureName(architecture)
if err != nil {
return InternalError(err)
}
diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index 6723535..7fa2205 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -11,6 +11,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -141,7 +142,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
}
}
- arch, err := shared.ArchitectureId(sf.Container.Architecture)
+ arch, err := osarch.ArchitectureId(sf.Container.Architecture)
if err != nil {
return SmartError(err)
}
@@ -170,7 +171,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
}
}
- arch, err := shared.ArchitectureId(snap.Architecture)
+ arch, err := osarch.ArchitectureId(snap.Architecture)
if err != nil {
return SmartError(err)
}
diff --git a/lxd/container.go b/lxd/container.go
index fdd3223..6b43e07 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -10,6 +10,7 @@ import (
"gopkg.in/lxc/go-lxc.v2"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
)
// Helper functions
@@ -48,9 +49,9 @@ func containerValidConfigKey(d *Daemon, key string, value string) error {
}
if key == "security.syscalls.blacklist_compat" {
for _, arch := range d.architectures {
- if arch == shared.ARCH_64BIT_INTEL_X86 ||
- arch == shared.ARCH_64BIT_ARMV8_LITTLE_ENDIAN ||
- arch == shared.ARCH_64BIT_POWERPC_BIG_ENDIAN {
+ if arch == osarch.ARCH_64BIT_INTEL_X86 ||
+ arch == osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN ||
+ arch == osarch.ARCH_64BIT_POWERPC_BIG_ENDIAN {
return nil
}
}
@@ -623,7 +624,7 @@ func containerCreateInternal(d *Daemon, args containerArgs) (container, error) {
}
// Validate architecture
- _, err = shared.ArchitectureName(args.Architecture)
+ _, err = osarch.ArchitectureName(args.Architecture)
if err != nil {
return nil, err
}
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index a24b5ce..95232bc 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -25,6 +25,7 @@ import (
"gopkg.in/yaml.v2"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -870,9 +871,9 @@ func (c *containerLXC) initLXC() error {
}
// Setup architecture
- personality, err := shared.ArchitecturePersonality(c.architecture)
+ personality, err := osarch.ArchitecturePersonality(c.architecture)
if err != nil {
- personality, err = shared.ArchitecturePersonality(c.daemon.architectures[0])
+ personality, err = osarch.ArchitecturePersonality(c.daemon.architectures[0])
if err != nil {
return err
}
@@ -2380,7 +2381,7 @@ func (c *containerLXC) Render() (interface{}, interface{}, error) {
}
// Ignore err as the arch string on error is correct (unknown)
- architectureName, _ := shared.ArchitectureName(c.architecture)
+ architectureName, _ := osarch.ArchitectureName(c.architecture)
// Prepare the ETag
etag := []interface{}{c.architecture, c.localConfig, c.localDevices, c.ephemeral, c.profiles}
@@ -2882,7 +2883,7 @@ func (c *containerLXC) Update(args containerArgs, userRequested bool) error {
// Validate the new architecture
if args.Architecture != 0 {
- _, err = shared.ArchitectureName(args.Architecture)
+ _, err = osarch.ArchitectureName(args.Architecture)
if err != nil {
return fmt.Errorf("Invalid architecture id: %s", err)
}
@@ -3694,13 +3695,13 @@ func (c *containerLXC) Export(w io.Writer, properties map[string]string) error {
return err
}
- arch, _ = shared.ArchitectureName(parent.Architecture())
+ arch, _ = osarch.ArchitectureName(parent.Architecture())
} else {
- arch, _ = shared.ArchitectureName(c.architecture)
+ arch, _ = osarch.ArchitectureName(c.architecture)
}
if arch == "" {
- arch, err = shared.ArchitectureName(c.daemon.architectures[0])
+ arch, err = osarch.ArchitectureName(c.daemon.architectures[0])
if err != nil {
shared.LogError("Failed exporting container", ctxMap)
return err
@@ -4118,9 +4119,9 @@ func (c *containerLXC) templateApplyNow(trigger string) error {
}
// Figure out the architecture
- arch, err := shared.ArchitectureName(c.architecture)
+ arch, err := osarch.ArchitectureName(c.architecture)
if err != nil {
- arch, err = shared.ArchitectureName(c.daemon.architectures[0])
+ arch, err = osarch.ArchitectureName(c.daemon.architectures[0])
if err != nil {
return err
}
diff --git a/lxd/container_patch.go b/lxd/container_patch.go
index 5bf7c79..f7752bb 100644
--- a/lxd/container_patch.go
+++ b/lxd/container_patch.go
@@ -8,7 +8,9 @@ import (
"net/http"
"github.com/gorilla/mux"
+
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
)
func containerPatch(d *Daemon, r *http.Request) Response {
@@ -54,7 +56,7 @@ func containerPatch(d *Daemon, r *http.Request) Response {
if err != nil {
architecture = c.Architecture()
} else {
- architecture, err = shared.ArchitectureId(req.Architecture)
+ architecture, err = osarch.ArchitectureId(req.Architecture)
if err != nil {
architecture = 0
}
diff --git a/lxd/container_put.go b/lxd/container_put.go
index b6aca32..471b022 100644
--- a/lxd/container_put.go
+++ b/lxd/container_put.go
@@ -7,7 +7,9 @@ import (
"net/http"
"github.com/gorilla/mux"
+
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -45,7 +47,7 @@ func containerPut(d *Daemon, r *http.Request) Response {
return BadRequest(err)
}
- architecture, err := shared.ArchitectureId(configRaw.Architecture)
+ architecture, err := osarch.ArchitectureId(configRaw.Architecture)
if err != nil {
architecture = 0
}
diff --git a/lxd/containers_post.go b/lxd/containers_post.go
index 21e6742..93b21a4 100644
--- a/lxd/containers_post.go
+++ b/lxd/containers_post.go
@@ -10,7 +10,9 @@ import (
"github.com/dustinkirkland/golang-petname"
"github.com/gorilla/websocket"
+
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -139,7 +141,7 @@ func createFromImage(d *Daemon, req *containerPostReq) Response {
hash = imgInfo.Fingerprint
- architecture, err := shared.ArchitectureId(imgInfo.Architecture)
+ architecture, err := osarch.ArchitectureId(imgInfo.Architecture)
if err != nil {
architecture = 0
}
@@ -171,7 +173,7 @@ func createFromImage(d *Daemon, req *containerPostReq) Response {
}
func createFromNone(d *Daemon, req *containerPostReq) Response {
- architecture, err := shared.ArchitectureId(req.Architecture)
+ architecture, err := osarch.ArchitectureId(req.Architecture)
if err != nil {
architecture = 0
}
@@ -207,7 +209,7 @@ func createFromMigration(d *Daemon, req *containerPostReq) Response {
return NotImplemented
}
- architecture, err := shared.ArchitectureId(req.Architecture)
+ architecture, err := osarch.ArchitectureId(req.Architecture)
if err != nil {
architecture = 0
}
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 4687c8c..d2dcd31 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -32,6 +32,7 @@ import (
"github.com/lxc/lxd"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logging"
+ "github.com/lxc/lxd/shared/osarch"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -712,18 +713,18 @@ func (d *Daemon) Init() error {
/* Get the list of supported architectures */
var architectures = []int{}
- architectureName, err := shared.ArchitectureGetLocal()
+ architectureName, err := osarch.ArchitectureGetLocal()
if err != nil {
return err
}
- architecture, err := shared.ArchitectureId(architectureName)
+ architecture, err := osarch.ArchitectureId(architectureName)
if err != nil {
return err
}
architectures = append(architectures, architecture)
- personalities, err := shared.ArchitecturePersonalities(architecture)
+ personalities, err := osarch.ArchitecturePersonalities(architecture)
if err != nil {
return err
}
diff --git a/lxd/db_images.go b/lxd/db_images.go
index 76ce7bb..ac515a3 100644
--- a/lxd/db_images.go
+++ b/lxd/db_images.go
@@ -8,6 +8,7 @@ import (
_ "github.com/mattn/go-sqlite3"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
)
var dbImageSourceProtocol = map[int]string{
@@ -175,7 +176,7 @@ func dbImageGet(db *sql.DB, fingerprint string, public bool, strictMatching bool
image.LastUsedDate = time.Time{}
}
- image.Architecture, _ = shared.ArchitectureName(arch)
+ image.Architecture, _ = osarch.ArchitectureName(arch)
// The upload date is enforced by NOT NULL in the schema, so it can never be nil.
image.UploadDate = *upload
@@ -312,7 +313,7 @@ func dbImageLastAccessInit(db *sql.DB, fingerprint string) error {
}
func dbImageUpdate(db *sql.DB, id int, fname string, sz int64, public bool, autoUpdate bool, architecture string, creationDate time.Time, expiryDate time.Time, properties map[string]string) error {
- arch, err := shared.ArchitectureId(architecture)
+ arch, err := osarch.ArchitectureId(architecture)
if err != nil {
arch = 0
}
@@ -369,7 +370,7 @@ func dbImageUpdate(db *sql.DB, id int, fname string, sz int64, public bool, auto
}
func dbImageInsert(db *sql.DB, fp string, fname string, sz int64, public bool, autoUpdate bool, architecture string, creationDate time.Time, expiryDate time.Time, properties map[string]string) error {
- arch, err := shared.ArchitectureId(architecture)
+ arch, err := osarch.ArchitectureId(architecture)
if err != nil {
arch = 0
}
diff --git a/lxd/images.go b/lxd/images.go
index 291789f..c6724e1 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -24,6 +24,7 @@ import (
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logging"
+ "github.com/lxc/lxd/shared/osarch"
log "gopkg.in/inconshreveable/log15.v2"
)
@@ -319,7 +320,7 @@ func imgPostContInfo(d *Daemon, r *http.Request, req imagePostReq,
return info, err
}
- info.Architecture, _ = shared.ArchitectureName(c.Architecture())
+ info.Architecture, _ = osarch.ArchitectureName(c.Architecture())
info.Properties = req.Properties
return info, nil
@@ -804,7 +805,7 @@ func getImageMetadata(fname string) (*imageMetadata, error) {
return nil, fmt.Errorf("Could not parse %s: %v", metadataName, err)
}
- _, err = shared.ArchitectureId(metadata.Architecture)
+ _, err = osarch.ArchitectureId(metadata.Architecture)
if err != nil {
return nil, err
}
diff --git a/lxd/seccomp.go b/lxd/seccomp.go
index 7e40330..1c9bb4c 100644
--- a/lxd/seccomp.go
+++ b/lxd/seccomp.go
@@ -7,6 +7,7 @@ import (
"path"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/osarch"
)
const SECCOMP_HEADER = `2
@@ -121,7 +122,7 @@ func getSeccompProfileContent(c container) (string, error) {
compat := config["security.syscalls.blacklist_compat"]
if shared.IsTrue(compat) {
- arch, err := shared.ArchitectureName(c.Architecture())
+ arch, err := osarch.ArchitectureName(c.Architecture())
if err != nil {
return "", err
}
diff --git a/shared/architectures.go b/shared/architectures.go
deleted file mode 100644
index 0f4f9ba..0000000
--- a/shared/architectures.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package shared
-
-import (
- "fmt"
-)
-
-const (
- ARCH_UNKNOWN = 0
- ARCH_32BIT_INTEL_X86 = 1
- ARCH_64BIT_INTEL_X86 = 2
- ARCH_32BIT_ARMV7_LITTLE_ENDIAN = 3
- ARCH_64BIT_ARMV8_LITTLE_ENDIAN = 4
- ARCH_32BIT_POWERPC_BIG_ENDIAN = 5
- ARCH_64BIT_POWERPC_BIG_ENDIAN = 6
- ARCH_64BIT_POWERPC_LITTLE_ENDIAN = 7
- ARCH_64BIT_S390_BIG_ENDIAN = 8
-)
-
-var architectureNames = map[int]string{
- ARCH_32BIT_INTEL_X86: "i686",
- ARCH_64BIT_INTEL_X86: "x86_64",
- ARCH_32BIT_ARMV7_LITTLE_ENDIAN: "armv7l",
- ARCH_64BIT_ARMV8_LITTLE_ENDIAN: "aarch64",
- ARCH_32BIT_POWERPC_BIG_ENDIAN: "ppc",
- ARCH_64BIT_POWERPC_BIG_ENDIAN: "ppc64",
- ARCH_64BIT_POWERPC_LITTLE_ENDIAN: "ppc64le",
- ARCH_64BIT_S390_BIG_ENDIAN: "s390x",
-}
-
-var architectureAliases = map[int][]string{
- ARCH_32BIT_INTEL_X86: []string{"i386"},
- ARCH_64BIT_INTEL_X86: []string{"amd64"},
- ARCH_32BIT_ARMV7_LITTLE_ENDIAN: []string{"armel", "armhf"},
- ARCH_64BIT_ARMV8_LITTLE_ENDIAN: []string{"arm64"},
- ARCH_32BIT_POWERPC_BIG_ENDIAN: []string{"powerpc"},
- ARCH_64BIT_POWERPC_BIG_ENDIAN: []string{"powerpc64"},
- ARCH_64BIT_POWERPC_LITTLE_ENDIAN: []string{"ppc64el"},
-}
-
-var architecturePersonalities = map[int]string{
- ARCH_32BIT_INTEL_X86: "linux32",
- ARCH_64BIT_INTEL_X86: "linux64",
- ARCH_32BIT_ARMV7_LITTLE_ENDIAN: "linux32",
- ARCH_64BIT_ARMV8_LITTLE_ENDIAN: "linux64",
- ARCH_32BIT_POWERPC_BIG_ENDIAN: "linux32",
- ARCH_64BIT_POWERPC_BIG_ENDIAN: "linux64",
- ARCH_64BIT_POWERPC_LITTLE_ENDIAN: "linux64",
- ARCH_64BIT_S390_BIG_ENDIAN: "linux64",
-}
-
-var architectureSupportedPersonalities = map[int][]int{
- ARCH_32BIT_INTEL_X86: []int{},
- ARCH_64BIT_INTEL_X86: []int{ARCH_32BIT_INTEL_X86},
- ARCH_32BIT_ARMV7_LITTLE_ENDIAN: []int{},
- ARCH_64BIT_ARMV8_LITTLE_ENDIAN: []int{ARCH_32BIT_ARMV7_LITTLE_ENDIAN},
- ARCH_32BIT_POWERPC_BIG_ENDIAN: []int{},
- ARCH_64BIT_POWERPC_BIG_ENDIAN: []int{ARCH_32BIT_POWERPC_BIG_ENDIAN},
- ARCH_64BIT_POWERPC_LITTLE_ENDIAN: []int{},
- ARCH_64BIT_S390_BIG_ENDIAN: []int{},
-}
-
-const ArchitectureDefault = "x86_64"
-
-func ArchitectureName(arch int) (string, error) {
- arch_name, exists := architectureNames[arch]
- if exists {
- return arch_name, nil
- }
-
- return "unknown", fmt.Errorf("Architecture isn't supported: %d", arch)
-}
-
-func ArchitectureId(arch string) (int, error) {
- for arch_id, arch_name := range architectureNames {
- if arch_name == arch {
- return arch_id, nil
- }
- }
-
- for arch_id, arch_aliases := range architectureAliases {
- for _, arch_name := range arch_aliases {
- if arch_name == arch {
- return arch_id, nil
- }
- }
- }
-
- return 0, fmt.Errorf("Architecture isn't supported: %s", arch)
-}
-
-func ArchitecturePersonality(arch int) (string, error) {
- arch_personality, exists := architecturePersonalities[arch]
- if exists {
- return arch_personality, nil
- }
-
- return "", fmt.Errorf("Architecture isn't supported: %d", arch)
-}
-
-func ArchitecturePersonalities(arch int) ([]int, error) {
- personalities, exists := architectureSupportedPersonalities[arch]
- if exists {
- return personalities, nil
- }
-
- return []int{}, fmt.Errorf("Architecture isn't supported: %d", arch)
-}
diff --git a/shared/architectures_linux.go b/shared/architectures_linux.go
deleted file mode 100644
index 0cc82ff..0000000
--- a/shared/architectures_linux.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// +build linux
-
-package shared
-
-import (
- "syscall"
-)
-
-func ArchitectureGetLocal() (string, error) {
- uname := syscall.Utsname{}
- if err := syscall.Uname(&uname); err != nil {
- return ArchitectureDefault, err
- }
-
- architectureName := ""
- for _, c := range uname.Machine {
- if c == 0 {
- break
- }
- architectureName += string(byte(c))
- }
-
- return architectureName, nil
-}
diff --git a/shared/architectures_others.go b/shared/architectures_others.go
deleted file mode 100644
index 5609316..0000000
--- a/shared/architectures_others.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// +build !linux
-
-package shared
-
-func ArchitectureGetLocal() (string, error) {
- return ArchitectureDefault, nil
-}
diff --git a/shared/osarch/architectures.go b/shared/osarch/architectures.go
new file mode 100644
index 0000000..728098c
--- /dev/null
+++ b/shared/osarch/architectures.go
@@ -0,0 +1,107 @@
+package osarch
+
+import (
+ "fmt"
+)
+
+const (
+ ARCH_UNKNOWN = 0
+ ARCH_32BIT_INTEL_X86 = 1
+ ARCH_64BIT_INTEL_X86 = 2
+ ARCH_32BIT_ARMV7_LITTLE_ENDIAN = 3
+ ARCH_64BIT_ARMV8_LITTLE_ENDIAN = 4
+ ARCH_32BIT_POWERPC_BIG_ENDIAN = 5
+ ARCH_64BIT_POWERPC_BIG_ENDIAN = 6
+ ARCH_64BIT_POWERPC_LITTLE_ENDIAN = 7
+ ARCH_64BIT_S390_BIG_ENDIAN = 8
+)
+
+var architectureNames = map[int]string{
+ ARCH_32BIT_INTEL_X86: "i686",
+ ARCH_64BIT_INTEL_X86: "x86_64",
+ ARCH_32BIT_ARMV7_LITTLE_ENDIAN: "armv7l",
+ ARCH_64BIT_ARMV8_LITTLE_ENDIAN: "aarch64",
+ ARCH_32BIT_POWERPC_BIG_ENDIAN: "ppc",
+ ARCH_64BIT_POWERPC_BIG_ENDIAN: "ppc64",
+ ARCH_64BIT_POWERPC_LITTLE_ENDIAN: "ppc64le",
+ ARCH_64BIT_S390_BIG_ENDIAN: "s390x",
+}
+
+var architectureAliases = map[int][]string{
+ ARCH_32BIT_INTEL_X86: []string{"i386"},
+ ARCH_64BIT_INTEL_X86: []string{"amd64"},
+ ARCH_32BIT_ARMV7_LITTLE_ENDIAN: []string{"armel", "armhf"},
+ ARCH_64BIT_ARMV8_LITTLE_ENDIAN: []string{"arm64"},
+ ARCH_32BIT_POWERPC_BIG_ENDIAN: []string{"powerpc"},
+ ARCH_64BIT_POWERPC_BIG_ENDIAN: []string{"powerpc64"},
+ ARCH_64BIT_POWERPC_LITTLE_ENDIAN: []string{"ppc64el"},
+}
+
+var architecturePersonalities = map[int]string{
+ ARCH_32BIT_INTEL_X86: "linux32",
+ ARCH_64BIT_INTEL_X86: "linux64",
+ ARCH_32BIT_ARMV7_LITTLE_ENDIAN: "linux32",
+ ARCH_64BIT_ARMV8_LITTLE_ENDIAN: "linux64",
+ ARCH_32BIT_POWERPC_BIG_ENDIAN: "linux32",
+ ARCH_64BIT_POWERPC_BIG_ENDIAN: "linux64",
+ ARCH_64BIT_POWERPC_LITTLE_ENDIAN: "linux64",
+ ARCH_64BIT_S390_BIG_ENDIAN: "linux64",
+}
+
+var architectureSupportedPersonalities = map[int][]int{
+ ARCH_32BIT_INTEL_X86: []int{},
+ ARCH_64BIT_INTEL_X86: []int{ARCH_32BIT_INTEL_X86},
+ ARCH_32BIT_ARMV7_LITTLE_ENDIAN: []int{},
+ ARCH_64BIT_ARMV8_LITTLE_ENDIAN: []int{ARCH_32BIT_ARMV7_LITTLE_ENDIAN},
+ ARCH_32BIT_POWERPC_BIG_ENDIAN: []int{},
+ ARCH_64BIT_POWERPC_BIG_ENDIAN: []int{ARCH_32BIT_POWERPC_BIG_ENDIAN},
+ ARCH_64BIT_POWERPC_LITTLE_ENDIAN: []int{},
+ ARCH_64BIT_S390_BIG_ENDIAN: []int{},
+}
+
+const ArchitectureDefault = "x86_64"
+
+func ArchitectureName(arch int) (string, error) {
+ arch_name, exists := architectureNames[arch]
+ if exists {
+ return arch_name, nil
+ }
+
+ return "unknown", fmt.Errorf("Architecture isn't supported: %d", arch)
+}
+
+func ArchitectureId(arch string) (int, error) {
+ for arch_id, arch_name := range architectureNames {
+ if arch_name == arch {
+ return arch_id, nil
+ }
+ }
+
+ for arch_id, arch_aliases := range architectureAliases {
+ for _, arch_name := range arch_aliases {
+ if arch_name == arch {
+ return arch_id, nil
+ }
+ }
+ }
+
+ return 0, fmt.Errorf("Architecture isn't supported: %s", arch)
+}
+
+func ArchitecturePersonality(arch int) (string, error) {
+ arch_personality, exists := architecturePersonalities[arch]
+ if exists {
+ return arch_personality, nil
+ }
+
+ return "", fmt.Errorf("Architecture isn't supported: %d", arch)
+}
+
+func ArchitecturePersonalities(arch int) ([]int, error) {
+ personalities, exists := architectureSupportedPersonalities[arch]
+ if exists {
+ return personalities, nil
+ }
+
+ return []int{}, fmt.Errorf("Architecture isn't supported: %d", arch)
+}
diff --git a/shared/osarch/architectures_linux.go b/shared/osarch/architectures_linux.go
new file mode 100644
index 0000000..c95b58a
--- /dev/null
+++ b/shared/osarch/architectures_linux.go
@@ -0,0 +1,24 @@
+// +build linux
+
+package osarch
+
+import (
+ "syscall"
+)
+
+func ArchitectureGetLocal() (string, error) {
+ uname := syscall.Utsname{}
+ if err := syscall.Uname(&uname); err != nil {
+ return ArchitectureDefault, err
+ }
+
+ architectureName := ""
+ for _, c := range uname.Machine {
+ if c == 0 {
+ break
+ }
+ architectureName += string(byte(c))
+ }
+
+ return architectureName, nil
+}
diff --git a/shared/osarch/architectures_others.go b/shared/osarch/architectures_others.go
new file mode 100644
index 0000000..22bd1ea
--- /dev/null
+++ b/shared/osarch/architectures_others.go
@@ -0,0 +1,7 @@
+// +build !linux
+
+package osarch
+
+func ArchitectureGetLocal() (string, error) {
+ return ArchitectureDefault, nil
+}
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index b0f9ee3..8d13c75 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -16,6 +16,7 @@ import (
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/ioprogress"
+ "github.com/lxc/lxd/shared/osarch"
)
type ssSortImage []shared.ImageInfo
@@ -85,12 +86,12 @@ func (s *SimpleStreamsManifest) ToLXD() ([]shared.ImageInfo, map[string][][]stri
for _, product := range s.Products {
// Skip unsupported architectures
- architecture, err := shared.ArchitectureId(product.Architecture)
+ architecture, err := osarch.ArchitectureId(product.Architecture)
if err != nil {
continue
}
- architectureName, err := shared.ArchitectureName(architecture)
+ architectureName, err := osarch.ArchitectureName(architecture)
if err != nil {
continue
}
@@ -393,7 +394,7 @@ func (s *SimpleStreams) applyAliases(images []shared.ImageInfo) ([]shared.ImageI
return &shared.ImageAlias{Name: name}
}
- architectureName, _ := shared.ArchitectureGetLocal()
+ architectureName, _ := osarch.ArchitectureGetLocal()
newImages := []shared.ImageInfo{}
for _, image := range images {
From 46e11b62b7748bcab557d66e387a0cf677dac926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 15 Dec 2016 18:23:48 -0500
Subject: [PATCH 5/5] simplestreams: Don't depend on custom http handler
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client.go | 15 ++++++++++++++-
lxd/daemon_images.go | 14 ++++++++++++--
shared/simplestreams/simplestreams.go | 21 ++-------------------
3 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/client.go b/client.go
index 0ce204e..fbfa9ed 100644
--- a/client.go
+++ b/client.go
@@ -339,7 +339,20 @@ func NewClientFromInfo(info ConnectInfo) (*Client, error) {
}
if info.RemoteConfig.Protocol == "simplestreams" {
- ss, err := simplestreams.SimpleStreamsClient(c.Remote.Addr, shared.ProxyFromEnvironment)
+ tlsconfig, err := shared.GetTLSConfig("", "", "", nil)
+ if err != nil {
+ return nil, err
+ }
+
+ tr := &http.Transport{
+ TLSClientConfig: tlsconfig,
+ Dial: shared.RFC3493Dialer,
+ Proxy: shared.ProxyFromEnvironment,
+ DisableKeepAlives: true,
+ }
+ c.Http.Transport = tr
+
+ ss, err := simplestreams.NewClient(c.Remote.Addr, c.Http)
if err != nil {
return nil, err
}
diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go
index 5928233..c2dd4f6 100644
--- a/lxd/daemon_images.go
+++ b/lxd/daemon_images.go
@@ -67,7 +67,12 @@ func imageLoadStreamCache(d *Daemon) error {
for url, entry := range imageStreamCache {
if entry.ss == nil {
- ss, err := simplestreams.SimpleStreamsClient(url, d.proxy)
+ myhttp, err := d.httpClient("")
+ if err != nil {
+ return err
+ }
+
+ ss, err := simplestreams.NewClient(url, *myhttp)
if err != nil {
return err
}
@@ -99,7 +104,12 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce
if entry == nil || entry.expiry.Before(time.Now()) {
refresh := func() (*imageStreamCacheEntry, error) {
// Setup simplestreams client
- ss, err = simplestreams.SimpleStreamsClient(server, d.proxy)
+ myhttp, err := d.httpClient(certificate)
+ if err != nil {
+ return nil, err
+ }
+
+ ss, err = simplestreams.NewClient(server, *myhttp)
if err != nil {
return nil, err
}
diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go
index 8d13c75..fd45e4a 100644
--- a/shared/simplestreams/simplestreams.go
+++ b/shared/simplestreams/simplestreams.go
@@ -7,7 +7,6 @@ import (
"io"
"io/ioutil"
"net/http"
- "net/url"
"os"
"path/filepath"
"sort"
@@ -263,25 +262,9 @@ type SimpleStreamsIndexStream struct {
Products []string `json:"products"`
}
-func SimpleStreamsClient(url string, proxy func(*http.Request) (*url.URL, error)) (*SimpleStreams, error) {
- // Setup a http client
- tlsConfig, err := shared.GetTLSConfig("", "", "", nil)
- if err != nil {
- return nil, err
- }
-
- tr := &http.Transport{
- TLSClientConfig: tlsConfig,
- Dial: shared.RFC3493Dialer,
- Proxy: proxy,
- }
-
- myHttp := http.Client{
- Transport: tr,
- }
-
+func NewClient(url string, httpClient http.Client) (*SimpleStreams, error) {
return &SimpleStreams{
- http: &myHttp,
+ http: &httpClient,
url: url,
cachedManifest: map[string]*SimpleStreamsManifest{}}, nil
}
More information about the lxc-devel
mailing list