[lxc-devel] [lxd/master] Implement initial simplestreams support
stgraber on Github
lxc-bot at linuxcontainers.org
Fri Feb 26 22:43:02 UTC 2016
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 588 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20160226/3b3fbe0d/attachment.bin>
-------------- next part --------------
From 7a9f057e1f5d53320513af4c152cfb242372f1d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 16 Feb 2016 17:18:57 -0500
Subject: [PATCH 1/3] Implement simplestreams support in the client
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client.go | 182 +++++++++------
config.go | 35 ++-
lxc/image.go | 6 +-
lxc/remote.go | 34 ++-
po/lxd.pot | 99 ++++----
shared/architectures.go | 18 ++
shared/simplestreams.go | 595 ++++++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 842 insertions(+), 127 deletions(-)
create mode 100644 shared/simplestreams.go
diff --git a/client.go b/client.go
index 768f27b..c5b1468 100644
--- a/client.go
+++ b/client.go
@@ -37,6 +37,7 @@ type Client struct {
Http http.Client
websocketDialer websocket.Dialer
+ simplestreams *shared.SimpleStreams
}
type ResponseType string
@@ -201,6 +202,17 @@ func NewClient(config *Config, remote string) (*Client, error) {
return nil, err
}
c.Config = *config
+ c.Remote = &r
+
+ if c.Remote.Protocol == "simplestreams" {
+ ss, err := shared.SimpleStreamsClient(c.Remote.Addr)
+ if err != nil {
+ return nil, err
+ }
+
+ c.simplestreams = ss
+ }
+
return c, nil
}
@@ -554,42 +566,46 @@ func (c *Client) ListContainers() ([]shared.ContainerInfo, error) {
}
func (c *Client) CopyImage(image string, dest *Client, copy_aliases bool, aliases []string, public bool, progressHandler func(progress string)) error {
- fingerprint := c.GetAlias(image)
- if fingerprint == "" {
- fingerprint = image
+ source := shared.Jmap{
+ "type": "image",
+ "mode": "pull",
+ "server": c.BaseURL,
+ "protocol": c.Remote.Protocol,
+ "certificate": c.Certificate,
+ "fingerprint": image}
+
+ target := c.GetAlias(image)
+ if target != "" {
+ image = target
}
- info, err := c.GetImageInfo(fingerprint)
+ info, err := c.GetImageInfo(image)
if err != nil {
return err
}
- source := shared.Jmap{
- "type": "image",
- "mode": "pull",
- "server": c.BaseURL,
- "certificate": c.Certificate,
- "fingerprint": fingerprint}
+ if c.Remote.Protocol != "simplestreams" {
+ if !info.Public {
+ var secret string
- if !info.Public {
- var secret string
+ resp, err := c.post("images/"+image+"/secret", nil, Async)
+ if err != nil {
+ return err
+ }
- resp, err := c.post("images/"+fingerprint+"/secret", nil, Async)
- if err != nil {
- return err
- }
+ op, err := resp.MetadataAsOperation()
+ if err != nil {
+ return err
+ }
- op, err := resp.MetadataAsOperation()
- if err != nil {
- return err
- }
+ secret, err = op.Metadata.GetString("secret")
+ if err != nil {
+ return err
+ }
- secret, err = op.Metadata.GetString("secret")
- if err != nil {
- return err
+ source["secret"] = secret
}
-
- source["secret"] = secret
+ source["fingerprint"] = image
}
addresses, err := c.Addresses()
@@ -680,11 +696,15 @@ func (c *Client) CopyImage(image string, dest *Client, copy_aliases bool, aliase
return err
}
-func (c *Client) ExportImage(image string, target string) (*Response, string, error) {
+func (c *Client) ExportImage(image string, target string) (string, error) {
+ if c.Remote.Protocol == "simplestreams" && c.simplestreams != nil {
+ return c.simplestreams.ExportImage(image, target)
+ }
+
uri := c.url(shared.APIVersion, "images", image, "export")
raw, err := c.getRaw(uri)
if err != nil {
- return nil, "", err
+ return "", err
}
ctype, ctypeParams, err := mime.ParseMediaType(raw.Header.Get("Content-Type"))
@@ -695,7 +715,7 @@ func (c *Client) ExportImage(image string, target string) (*Response, string, er
// Deal with split images
if ctype == "multipart/form-data" {
if !shared.IsDir(target) {
- return nil, "", fmt.Errorf("Split images can only be written to a directory.")
+ return "", fmt.Errorf("Split images can only be written to a directory.")
}
// Parse the POST data
@@ -704,48 +724,48 @@ func (c *Client) ExportImage(image string, target string) (*Response, string, er
// Get the metadata tarball
part, err := mr.NextPart()
if err != nil {
- return nil, "", err
+ return "", err
}
if part.FormName() != "metadata" {
- return nil, "", fmt.Errorf("Invalid multipart image")
+ return "", fmt.Errorf("Invalid multipart image")
}
- imageTarf, err := os.OpenFile(part.FileName(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+ imageTarf, err := os.OpenFile(filepath.Join(target, part.FileName()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
- return nil, "", err
+ return "", err
}
_, err = io.Copy(imageTarf, part)
imageTarf.Close()
if err != nil {
- return nil, "", err
+ return "", err
}
// Get the rootfs tarball
part, err = mr.NextPart()
if err != nil {
- return nil, "", err
+ return "", err
}
if part.FormName() != "rootfs" {
- return nil, "", fmt.Errorf("Invalid multipart image")
+ return "", fmt.Errorf("Invalid multipart image")
}
- rootfsTarf, err := os.OpenFile(part.FileName(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+ rootfsTarf, err := os.OpenFile(filepath.Join(part.FileName()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
- return nil, "", err
+ return "", err
}
_, err = io.Copy(rootfsTarf, part)
rootfsTarf.Close()
if err != nil {
- return nil, "", err
+ return "", err
}
- return nil, target, nil
+ return target, nil
}
// Deal with unified images
@@ -768,7 +788,7 @@ func (c *Client) ExportImage(image string, target string) (*Response, string, er
defer f.Close()
if err != nil {
- return nil, "", err
+ return "", err
}
wr = f
@@ -780,14 +800,12 @@ func (c *Client) ExportImage(image string, target string) (*Response, string, er
defer f.Close()
if err != nil {
- return nil, "", err
+ return "", err
}
wr = f
}
-
} else {
-
// write as simple file
destpath = target
f, err := os.Create(destpath)
@@ -795,19 +813,18 @@ func (c *Client) ExportImage(image string, target string) (*Response, string, er
wr = f
if err != nil {
- return nil, "", err
+ return "", err
}
-
}
_, err = io.Copy(wr, raw.Body)
if err != nil {
- return nil, "", err
+ return "", err
}
// it streams to stdout or file, so no response returned
- return nil, destpath, nil
+ return destpath, nil
}
func (c *Client) PostImageURL(imageFile string, public bool, aliases []string) (string, error) {
@@ -976,6 +993,10 @@ func (c *Client) PostImage(imageFile string, rootfsFile string, properties []str
}
func (c *Client) GetImageInfo(image string) (*shared.ImageInfo, error) {
+ if c.Remote.Protocol == "simplestreams" && c.simplestreams != nil {
+ return c.simplestreams.GetImageInfo(image)
+ }
+
resp, err := c.get(fmt.Sprintf("images/%s", image))
if err != nil {
return nil, err
@@ -999,6 +1020,10 @@ func (c *Client) PutImageInfo(name string, p shared.BriefImageInfo) error {
}
func (c *Client) ListImages() ([]shared.ImageInfo, error) {
+ if c.Remote.Protocol == "simplestreams" && c.simplestreams != nil {
+ return c.simplestreams.ListImages()
+ }
+
resp, err := c.get("images?recursion=1")
if err != nil {
return nil, err
@@ -1030,6 +1055,10 @@ func (c *Client) DeleteAlias(alias string) error {
}
func (c *Client) ListAliases() (shared.ImageAliases, error) {
+ if c.Remote.Protocol == "simplestreams" && c.simplestreams != nil {
+ return c.simplestreams.ListAliases()
+ }
+
resp, err := c.get("images/aliases?recursion=1")
if err != nil {
return nil, err
@@ -1089,6 +1118,10 @@ func (c *Client) IsAlias(alias string) (bool, error) {
}
func (c *Client) GetAlias(alias string) string {
+ if c.Remote.Protocol == "simplestreams" && c.simplestreams != nil {
+ return c.simplestreams.GetAlias(alias)
+ }
+
resp, err := c.get(fmt.Sprintf("images/aliases/%s", alias))
if err != nil {
return ""
@@ -1131,44 +1164,47 @@ func (c *Client) Init(name string, imgremote string, image string, profiles *[]s
return nil, err
}
- fingerprint := tmpremote.GetAlias(image)
- if fingerprint == "" {
- fingerprint = image
- }
-
- imageinfo, err := tmpremote.GetImageInfo(fingerprint)
- if err != nil {
- return nil, err
- }
-
- if len(architectures) != 0 && !shared.StringInSlice(imageinfo.Architecture, architectures) {
- return nil, fmt.Errorf("The image architecture is incompatible with the target server")
- }
-
- if !imageinfo.Public {
- var secret string
-
- resp, err := tmpremote.post("images/"+fingerprint+"/secret", nil, Async)
- if err != nil {
- return nil, err
+ if tmpremote.Remote.Protocol != "simplestreams" {
+ target := tmpremote.GetAlias(image)
+ if target != "" {
+ image = target
}
- op, err := resp.MetadataAsOperation()
+ imageinfo, err := tmpremote.GetImageInfo(image)
if err != nil {
return nil, err
}
- secret, err = op.Metadata.GetString("secret")
- if err != nil {
- return nil, err
+ if len(architectures) != 0 && !shared.StringInSlice(imageinfo.Architecture, architectures) {
+ return nil, fmt.Errorf("The image architecture is incompatible with the target server")
}
- source["secret"] = secret
+ if !imageinfo.Public {
+ var secret string
+
+ resp, err := tmpremote.post("images/"+image+"/secret", nil, Async)
+ if err != nil {
+ return nil, err
+ }
+
+ op, err := resp.MetadataAsOperation()
+ if err != nil {
+ return nil, err
+ }
+
+ secret, err = op.Metadata.GetString("secret")
+ if err != nil {
+ return nil, err
+ }
+
+ source["secret"] = secret
+ }
}
source["server"] = tmpremote.BaseURL
+ source["protocol"] = tmpremote.Remote.Protocol
source["certificate"] = tmpremote.Certificate
- source["fingerprint"] = fingerprint
+ source["fingerprint"] = image
} else {
fingerprint := c.GetAlias(image)
if fingerprint == "" {
diff --git a/config.go b/config.go
index eb4343a..ba06b97 100644
--- a/config.go
+++ b/config.go
@@ -40,17 +40,36 @@ type Config struct {
// RemoteConfig holds details for communication with a remote daemon.
type RemoteConfig struct {
- Addr string `yaml:"addr"`
- Public bool `yaml:"public"`
+ Addr string `yaml:"addr"`
+ Public bool `yaml:"public"`
+ Protocol string `yaml:"protocol,omitempty"`
+ Static bool `yaml:"-"`
}
var LocalRemote = RemoteConfig{
Addr: "unix://",
+ Static: true,
Public: false}
-var defaultRemote = map[string]RemoteConfig{"local": LocalRemote}
+
+var UbuntuRemote = RemoteConfig{
+ Addr: "https://cloud-images.ubuntu.com/releases",
+ Static: true,
+ Public: true,
+ Protocol: "simplestreams"}
+
+var UbuntuDailyRemote = RemoteConfig{
+ Addr: "https://cloud-images.ubuntu.com/daily",
+ Static: true,
+ Public: true,
+ Protocol: "simplestreams"}
+
+var StaticRemotes = map[string]RemoteConfig{
+ "local": LocalRemote,
+ "ubuntu": UbuntuRemote,
+ "ubuntu-daily": UbuntuDailyRemote}
var DefaultConfig = Config{
- Remotes: defaultRemote,
+ Remotes: StaticRemotes,
DefaultRemote: "local",
Aliases: map[string]string{},
}
@@ -79,11 +98,19 @@ func LoadConfig(path string) (*Config, error) {
}
c.ConfigDir = filepath.Dir(path)
+ for k, v := range StaticRemotes {
+ c.Remotes[k] = v
+ }
+
return &c, nil
}
// SaveConfig writes the provided configuration to the config file.
func SaveConfig(c *Config, fname string) error {
+ for k, _ := range StaticRemotes {
+ delete(c.Remotes, k)
+ }
+
// Ignore errors on these two calls. Create will report any problems.
os.Remove(fname + ".new")
os.Mkdir(filepath.Dir(fname), 0700)
diff --git a/lxc/image.go b/lxc/image.go
index 8187333..d875b46 100644
--- a/lxc/image.go
+++ b/lxc/image.go
@@ -240,13 +240,12 @@ func (c *imageCmd) run(config *lxd.Config, args []string) error {
if err != nil {
return err
}
- image := c.dereferenceAlias(d, inName)
progressHandler := func(progress string) {
fmt.Printf(i18n.G("Copying the image: %s")+"\r", progress)
}
- err = d.CopyImage(image, dest, c.copyAliases, c.addAliases, c.publicImage, progressHandler)
+ err = d.CopyImage(inName, dest, c.copyAliases, c.addAliases, c.publicImage, progressHandler)
if err == nil {
fmt.Println(i18n.G("Image copied successfully!"))
}
@@ -450,7 +449,8 @@ func (c *imageCmd) run(config *lxd.Config, args []string) error {
if len(args) > 2 {
target = args[2]
}
- _, outfile, err := d.ExportImage(image, target)
+
+ outfile, err := d.ExportImage(image, target)
if err != nil {
return err
}
diff --git a/lxc/remote.go b/lxc/remote.go
index f20f1bc..0e1cde7 100644
--- a/lxc/remote.go
+++ b/lxc/remote.go
@@ -264,10 +264,15 @@ func (c *remoteCmd) run(config *lxd.Config, args []string) error {
return errArgs
}
- if _, ok := config.Remotes[args[1]]; !ok {
+ rc, ok := config.Remotes[args[1]]
+ if !ok {
return fmt.Errorf(i18n.G("remote %s doesn't exist"), args[1])
}
+ if rc.Static {
+ return fmt.Errorf(i18n.G("remote %s is static and cannot be modified"), args[1])
+ }
+
if config.DefaultRemote == args[1] {
return fmt.Errorf(i18n.G("can't remove the default remote"))
}
@@ -284,11 +289,20 @@ func (c *remoteCmd) run(config *lxd.Config, args []string) error {
strPublic = i18n.G("YES")
}
+ strStatic := i18n.G("NO")
+ if rc.Static {
+ strStatic = i18n.G("YES")
+ }
+
+ if rc.Protocol == "" {
+ rc.Protocol = "lxd"
+ }
+
strName := name
if name == config.DefaultRemote {
strName = fmt.Sprintf("%s (%s)", name, i18n.G("default"))
}
- data = append(data, []string{strName, rc.Addr, strPublic})
+ data = append(data, []string{strName, rc.Addr, rc.Protocol, strPublic, strStatic})
}
table := tablewriter.NewWriter(os.Stdout)
@@ -297,7 +311,9 @@ func (c *remoteCmd) run(config *lxd.Config, args []string) error {
table.SetHeader([]string{
i18n.G("NAME"),
i18n.G("URL"),
- i18n.G("PUBLIC")})
+ i18n.G("PROTOCOL"),
+ i18n.G("PUBLIC"),
+ i18n.G("STATIC")})
sort.Sort(byName(data))
table.AppendBulk(data)
table.Render()
@@ -314,6 +330,10 @@ func (c *remoteCmd) run(config *lxd.Config, args []string) error {
return fmt.Errorf(i18n.G("remote %s doesn't exist"), args[1])
}
+ if rc.Static {
+ return fmt.Errorf(i18n.G("remote %s is static and cannot be modified"), args[1])
+ }
+
if _, ok := config.Remotes[args[2]]; ok {
return fmt.Errorf(i18n.G("remote %s already exists"), args[2])
}
@@ -339,10 +359,16 @@ func (c *remoteCmd) run(config *lxd.Config, args []string) error {
if len(args) != 3 {
return errArgs
}
- _, ok := config.Remotes[args[1]]
+
+ rc, ok := config.Remotes[args[1]]
if !ok {
return fmt.Errorf(i18n.G("remote %s doesn't exist"), args[1])
}
+
+ if rc.Static {
+ return fmt.Errorf(i18n.G("remote %s is static and cannot be modified"), args[1])
+ }
+
config.Remotes[args[1]] = lxd.RemoteConfig{Addr: args[2]}
case "set-default":
diff --git a/po/lxd.pot b/po/lxd.pot
index 52fbebf..50f539a 100644
--- a/po/lxd.pot
+++ b/po/lxd.pot
@@ -7,7 +7,7 @@
msgid ""
msgstr "Project-Id-Version: lxd\n"
"Report-Msgid-Bugs-To: lxc-devel at lists.linuxcontainers.org\n"
- "POT-Creation-Date: 2016-02-25 16:37-0500\n"
+ "POT-Creation-Date: 2016-02-25 23:25-0500\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL at ADDRESS>\n"
"Language-Team: LANGUAGE <LL at li.org>\n"
@@ -16,7 +16,7 @@ msgstr "Project-Id-Version: lxd\n"
"Content-Type: text/plain; charset=CHARSET\n"
"Content-Transfer-Encoding: 8bit\n"
-#: lxc/config.go:36
+#: lxc/config.go:37
msgid "### This is a yaml representation of the configuration.\n"
"### Any line starting with a '# will be ignored.\n"
"###\n"
@@ -65,7 +65,7 @@ msgid "### This is a yaml representation of the profile.\n"
"### Note that the name is shown but cannot be changed"
msgstr ""
-#: lxc/image.go:534
+#: lxc/image.go:535
#, c-format
msgid "%s (%d more)"
msgstr ""
@@ -78,11 +78,11 @@ msgstr ""
msgid "(none)"
msgstr ""
-#: lxc/image.go:553 lxc/image.go:575
+#: lxc/image.go:555 lxc/image.go:579
msgid "ALIAS"
msgstr ""
-#: lxc/image.go:557
+#: lxc/image.go:559
msgid "ARCH"
msgstr ""
@@ -116,7 +116,7 @@ msgstr ""
msgid "Available commands:"
msgstr ""
-#: lxc/config.go:265
+#: lxc/config.go:268
msgid "COMMON NAME"
msgstr ""
@@ -124,12 +124,12 @@ msgstr ""
msgid "CREATED AT"
msgstr ""
-#: lxc/config.go:112
+#: lxc/config.go:113
#, c-format
msgid "Can't read from stdin: %s"
msgstr ""
-#: lxc/config.go:125 lxc/config.go:158 lxc/config.go:180
+#: lxc/config.go:126 lxc/config.go:159 lxc/config.go:181
#, c-format
msgid "Can't unset key '%s', it's not currently set."
msgstr ""
@@ -162,7 +162,7 @@ msgstr ""
msgid "Config key/value to apply to the new container"
msgstr ""
-#: lxc/config.go:491 lxc/config.go:556 lxc/image.go:631 lxc/profile.go:187
+#: lxc/config.go:492 lxc/config.go:557 lxc/image.go:633 lxc/profile.go:187
#, c-format
msgid "Config parsing error: %s"
msgstr ""
@@ -234,7 +234,7 @@ msgstr ""
msgid "Creating the container"
msgstr ""
-#: lxc/image.go:556 lxc/image.go:577
+#: lxc/image.go:558 lxc/image.go:581
msgid "DESCRIPTION"
msgstr ""
@@ -246,12 +246,12 @@ msgid "Delete containers or container snapshots.\n"
"Destroy containers or snapshots with any attached data (configuration, snapshots, ...)."
msgstr ""
-#: lxc/config.go:604
+#: lxc/config.go:605
#, c-format
msgid "Device %s added to %s"
msgstr ""
-#: lxc/config.go:632
+#: lxc/config.go:633
#, c-format
msgid "Device %s removed from %s"
msgstr ""
@@ -260,7 +260,7 @@ msgstr ""
msgid "EPHEMERAL"
msgstr ""
-#: lxc/config.go:267
+#: lxc/config.go:270
msgid "EXPIRY DATE"
msgstr ""
@@ -301,7 +301,7 @@ msgstr ""
msgid "Expires: never"
msgstr ""
-#: lxc/config.go:264 lxc/image.go:554 lxc/image.go:576
+#: lxc/config.go:267 lxc/image.go:556 lxc/image.go:580
msgid "FINGERPRINT"
msgstr ""
@@ -320,7 +320,7 @@ msgid "Fingers the LXD instance to check if it is up and working.\n"
"lxc finger <remote>"
msgstr ""
-#: lxc/main.go:156
+#: lxc/main.go:146
msgid "For example: 'lxd-images import ubuntu --alias ubuntu'."
msgstr ""
@@ -336,7 +336,7 @@ msgstr ""
msgid "Force using the local unix socket."
msgstr ""
-#: lxc/main.go:148
+#: lxc/main.go:138
msgid "Generating a client certificate. This may take a minute..."
msgstr ""
@@ -348,11 +348,11 @@ msgstr ""
msgid "IPV6"
msgstr ""
-#: lxc/config.go:266
+#: lxc/config.go:269
msgid "ISSUE DATE"
msgstr ""
-#: lxc/main.go:155
+#: lxc/main.go:145
msgid "If this is your first run, you will need to import images using the 'lxd-images' script."
msgstr ""
@@ -499,7 +499,7 @@ msgid "Manage configuration profiles.\n"
" using the specified profile."
msgstr ""
-#: lxc/config.go:57
+#: lxc/config.go:58
msgid "Manage configuration.\n"
"\n"
"lxc config device add <[remote:]container> <name> <type> [key=value]... Add a device to a container.\n"
@@ -646,11 +646,11 @@ msgid "Move containers within or in between lxd instances.\n"
" Rename a local container.\n"
msgstr ""
-#: lxc/list.go:336 lxc/remote.go:296
+#: lxc/list.go:336 lxc/remote.go:312
msgid "NAME"
msgstr ""
-#: lxc/remote.go:282
+#: lxc/remote.go:287 lxc/remote.go:292
msgid "NO"
msgstr ""
@@ -663,11 +663,11 @@ msgstr ""
msgid "New alias to define at target"
msgstr ""
-#: lxc/config.go:278
+#: lxc/config.go:279
msgid "No certificate provided to add"
msgstr ""
-#: lxc/config.go:301
+#: lxc/config.go:302
msgid "No fingerprint specified."
msgstr ""
@@ -675,11 +675,11 @@ msgstr ""
msgid "Only https:// is supported for remote image import."
msgstr ""
-#: lxc/help.go:63 lxc/main.go:132
+#: lxc/help.go:63 lxc/main.go:122
msgid "Options:"
msgstr ""
-#: lxc/image.go:459
+#: lxc/image.go:460
#, c-format
msgid "Output is in %s"
msgstr ""
@@ -700,7 +700,11 @@ msgstr ""
msgid "PROFILES"
msgstr ""
-#: lxc/image.go:555 lxc/remote.go:298
+#: lxc/remote.go:314
+msgid "PROTOCOL"
+msgstr ""
+
+#: lxc/image.go:557 lxc/remote.go:315
msgid "PUBLIC"
msgstr ""
@@ -731,7 +735,7 @@ msgstr ""
msgid "Press enter to open the editor again"
msgstr ""
-#: lxc/config.go:492 lxc/config.go:557 lxc/image.go:632
+#: lxc/config.go:493 lxc/config.go:558 lxc/image.go:634
msgid "Press enter to start the editor again"
msgstr ""
@@ -819,7 +823,7 @@ msgstr ""
msgid "Retrieving image: %s"
msgstr ""
-#: lxc/image.go:558
+#: lxc/image.go:560
msgid "SIZE"
msgstr ""
@@ -831,6 +835,10 @@ msgstr ""
msgid "STATE"
msgstr ""
+#: lxc/remote.go:316
+msgid "STATIC"
+msgstr ""
+
#: lxc/remote.go:164
msgid "Server certificate NACKed by user"
msgstr ""
@@ -936,11 +944,11 @@ msgstr ""
msgid "Type: persistent"
msgstr ""
-#: lxc/image.go:559
+#: lxc/image.go:561
msgid "UPLOAD DATE"
msgstr ""
-#: lxc/remote.go:297
+#: lxc/remote.go:313
msgid "URL"
msgstr ""
@@ -949,7 +957,7 @@ msgstr ""
msgid "Uploaded: %s"
msgstr ""
-#: lxc/main.go:132
+#: lxc/main.go:122
#, c-format
msgid "Usage: %s"
msgstr ""
@@ -970,11 +978,11 @@ msgstr ""
msgid "Whether or not to snapshot the container's running state"
msgstr ""
-#: lxc/config.go:32
+#: lxc/config.go:33
msgid "Whether to show the expanded configuration"
msgstr ""
-#: lxc/remote.go:284
+#: lxc/remote.go:289 lxc/remote.go:294
msgid "YES"
msgstr ""
@@ -994,11 +1002,11 @@ msgstr ""
msgid "can't copy to the same container name"
msgstr ""
-#: lxc/remote.go:272
+#: lxc/remote.go:277
msgid "can't remove the default remote"
msgstr ""
-#: lxc/remote.go:289
+#: lxc/remote.go:303
msgid "default"
msgstr ""
@@ -1006,12 +1014,12 @@ msgstr ""
msgid "didn't get any affected image, container or snapshot from server"
msgstr ""
-#: lxc/main.go:25 lxc/main.go:167
+#: lxc/main.go:25 lxc/main.go:157
#, c-format
msgid "error: %v"
msgstr ""
-#: lxc/help.go:40 lxc/main.go:127
+#: lxc/help.go:40 lxc/main.go:117
#, c-format
msgid "error: unknown command: %s"
msgstr ""
@@ -1020,7 +1028,7 @@ msgstr ""
msgid "got bad version"
msgstr ""
-#: lxc/image.go:291 lxc/image.go:537
+#: lxc/image.go:291 lxc/image.go:538
msgid "no"
msgstr ""
@@ -1032,17 +1040,17 @@ msgstr ""
msgid "ok (y/n)?"
msgstr ""
-#: lxc/main.go:274 lxc/main.go:278
+#: lxc/main.go:264 lxc/main.go:268
#, c-format
msgid "processing aliases failed %s\n"
msgstr ""
-#: lxc/remote.go:316
+#: lxc/remote.go:338
#, c-format
msgid "remote %s already exists"
msgstr ""
-#: lxc/remote.go:268 lxc/remote.go:312 lxc/remote.go:342 lxc/remote.go:353
+#: lxc/remote.go:269 lxc/remote.go:330 lxc/remote.go:365 lxc/remote.go:381
#, c-format
msgid "remote %s doesn't exist"
msgstr ""
@@ -1052,6 +1060,11 @@ msgstr ""
msgid "remote %s exists as <%s>"
msgstr ""
+#: lxc/remote.go:273 lxc/remote.go:334 lxc/remote.go:369
+#, c-format
+msgid "remote %s is static and cannot be modified"
+msgstr ""
+
#: lxc/info.go:139
msgid "stateful"
msgstr ""
@@ -1069,11 +1082,11 @@ msgstr ""
msgid "unreachable return reached"
msgstr ""
-#: lxc/main.go:207
+#: lxc/main.go:197
msgid "wrong number of subcommand arguments"
msgstr ""
-#: lxc/delete.go:45 lxc/image.go:294 lxc/image.go:541
+#: lxc/delete.go:45 lxc/image.go:294 lxc/image.go:542
msgid "yes"
msgstr ""
diff --git a/shared/architectures.go b/shared/architectures.go
index 64dbc3f..f8784a8 100644
--- a/shared/architectures.go
+++ b/shared/architectures.go
@@ -27,6 +27,16 @@ var architectureNames = map[int]string{
ARCH_64BIT_S390_BIG_ENDIAN: "s390x",
}
+var architectureAliases = map[int][]string{
+ ARCH_32BIT_INTEL_X86: []string{"i386"},
+ ARCH_64BIT_INTEL_X86: []string{"amd64"},
+ ARCH_32BIT_ARMV7_LITTLE_ENDIAN: []string{"armel", "armhf"},
+ ARCH_64BIT_ARMV8_LITTLE_ENDIAN: []string{"arm64"},
+ ARCH_32BIT_POWERPC_BIG_ENDIAN: []string{"powerpc"},
+ ARCH_64BIT_POWERPC_BIG_ENDIAN: []string{"powerpc64"},
+ ARCH_64BIT_POWERPC_LITTLE_ENDIAN: []string{"ppc64el"},
+}
+
var architecturePersonalities = map[int]string{
ARCH_32BIT_INTEL_X86: "linux32",
ARCH_64BIT_INTEL_X86: "linux64",
@@ -65,6 +75,14 @@ func ArchitectureId(arch string) (int, error) {
}
}
+ for arch_id, arch_aliases := range architectureAliases {
+ for _, arch_name := range arch_aliases {
+ if arch_name == arch {
+ return arch_id, nil
+ }
+ }
+ }
+
return 0, fmt.Errorf("Architecture isn't supported: %s", arch)
}
diff --git a/shared/simplestreams.go b/shared/simplestreams.go
new file mode 100644
index 0000000..e8b76a6
--- /dev/null
+++ b/shared/simplestreams.go
@@ -0,0 +1,595 @@
+package shared
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "syscall"
+ "time"
+)
+
+type ssSortImage []ImageInfo
+
+func (a ssSortImage) Len() int {
+ return len(a)
+}
+
+func (a ssSortImage) Swap(i, j int) {
+ a[i], a[j] = a[j], a[i]
+}
+
+func (a ssSortImage) Less(i, j int) bool {
+ if a[i].Properties["os"] == a[j].Properties["os"] {
+ if a[i].Properties["release"] == a[j].Properties["release"] {
+ if a[i].CreationDate.UTC().Unix() == 0 {
+ return true
+ }
+
+ if a[j].CreationDate.UTC().Unix() == 0 {
+ return false
+ }
+
+ return a[i].CreationDate.UTC().Unix() > a[j].CreationDate.UTC().Unix()
+ }
+
+ if a[i].Properties["release"] == "" {
+ return false
+ }
+
+ if a[j].Properties["release"] == "" {
+ return true
+ }
+
+ return a[i].Properties["release"] < a[j].Properties["release"]
+ }
+
+ if a[i].Properties["os"] == "" {
+ return false
+ }
+
+ if a[j].Properties["os"] == "" {
+ return true
+ }
+
+ return a[i].Properties["os"] < a[j].Properties["os"]
+}
+
+var ssDefaultOS = map[string]string{
+ "https://cloud-images.ubuntu.com": "ubuntu",
+}
+
+type SimpleStreamsManifest struct {
+ Updated string `json:"updated"`
+ DataType string `json:"datatype"`
+ Format string `json:"format"`
+ License string `json:"license"`
+ Products map[string]SimpleStreamsManifestProduct `json:"products"`
+}
+
+func (s *SimpleStreamsManifest) ToLXD() ([]ImageInfo, map[string][][]string) {
+ downloads := map[string][][]string{}
+
+ images := []ImageInfo{}
+ nameLayout := "20060102"
+ eolLayout := "2006-01-02"
+
+ for _, product := range s.Products {
+ // Skip unsupported architectures
+ architecture, err := ArchitectureId(product.Architecture)
+ if err != nil {
+ continue
+ }
+
+ architectureName, err := ArchitectureName(architecture)
+ if err != nil {
+ continue
+ }
+
+ for name, version := range product.Versions {
+ // Short of anything better, use the name as date
+ creationDate, err := time.Parse(nameLayout, name)
+ if err != nil {
+ continue
+ }
+
+ size := int64(0)
+ filename := ""
+ fingerprint := ""
+
+ metaPath := ""
+ metaHash := ""
+ rootfsPath := ""
+ rootfsHash := ""
+
+ found := 0
+ for _, item := range version.Items {
+ // Skip the files we don't care about
+ if !StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz"}) {
+ continue
+ }
+ found += 1
+
+ size += item.Size
+ if item.LXDHashSha256 != "" {
+ fingerprint = item.LXDHashSha256
+ }
+
+ if item.FileType == "lxd.tar.xz" {
+ fields := strings.Split(item.Path, "/")
+ filename = fields[len(fields)-1]
+ metaPath = item.Path
+ metaHash = item.HashSha256
+ }
+
+ if item.FileType == "root.tar.xz" {
+ rootfsPath = item.Path
+ rootfsHash = item.HashSha256
+ }
+ }
+
+ if found != 2 || size == 0 || filename == "" || fingerprint == "" {
+ // Invalid image
+ continue
+ }
+
+ // Generate the actual image entry
+ image := ImageInfo{}
+ image.Architecture = architectureName
+ image.Public = true
+ image.Size = size
+ image.CreationDate = creationDate
+ image.UploadDate = creationDate
+ image.Filename = filename
+ image.Fingerprint = fingerprint
+ image.Properties = map[string]string{
+ "os": product.OperatingSystem,
+ "release": product.Release,
+ "architecture": product.Architecture,
+ "label": version.Label,
+ "serial": name,
+ "description": fmt.Sprintf("%s %s %s (%s) (%s)", product.OperatingSystem, product.ReleaseTitle, product.Architecture, version.Label, name),
+ }
+
+ // Attempt to parse the EOL
+ if product.SupportedEOL != "" {
+ eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
+ if err == nil {
+ image.ExpiryDate = eolDate
+ }
+ }
+
+ downloads[fingerprint] = [][]string{[]string{metaPath, metaHash}, []string{rootfsPath, rootfsHash}}
+ images = append(images, image)
+ }
+ }
+
+ return images, downloads
+}
+
+type SimpleStreamsManifestProduct struct {
+ Architecture string `json:"arch"`
+ OperatingSystem string `json:"os"`
+ Release string `json:"release"`
+ ReleaseCodename string `json:"release_codename"`
+ ReleaseTitle string `json:"release_title"`
+ Supported bool `json:"supported"`
+ SupportedEOL string `json:"support_eol"`
+ Version string `json:"version"`
+ Versions map[string]SimpleStreamsManifestProductVersion `json:"versions"`
+}
+
+type SimpleStreamsManifestProductVersion struct {
+ PublicName string `json:"pubname"`
+ Label string `json:"label"`
+ Items map[string]SimpleStreamsManifestProductVersionItem `json:"items"`
+}
+
+type SimpleStreamsManifestProductVersionItem struct {
+ Path string `json:"path"`
+ FileType string `json:"ftype"`
+ HashMd5 string `json:"md5"`
+ HashSha256 string `json:"sha256"`
+ LXDHashSha256 string `json:"combined_sha256"`
+ Size int64 `json:"size"`
+}
+
+type SimpleStreamsIndex struct {
+ Format string `json:"format"`
+ Index map[string]SimpleStreamsIndexStream `json:"index"`
+ Updated string `json:"updated"`
+}
+
+type SimpleStreamsIndexStream struct {
+ Updated string `json:"updated"`
+ DataType string `json:"datatype"`
+ Path string `json:"path"`
+ Products []string `json:"products"`
+}
+
+func SimpleStreamsClient(url string) (*SimpleStreams, error) {
+ // Setup a http client
+ tlsConfig, err := GetTLSConfig("", "", nil)
+ if err != nil {
+ return nil, err
+ }
+
+ tr := &http.Transport{
+ TLSClientConfig: tlsConfig,
+ Dial: RFC3493Dialer,
+ Proxy: http.ProxyFromEnvironment,
+ }
+
+ myHttp := http.Client{
+ Transport: tr,
+ }
+
+ return &SimpleStreams{
+ http: &myHttp,
+ url: url,
+ cachedManifest: map[string]*SimpleStreamsManifest{}}, nil
+}
+
+type SimpleStreams struct {
+ http *http.Client
+ url string
+
+ cachedIndex *SimpleStreamsIndex
+ cachedManifest map[string]*SimpleStreamsManifest
+ cachedImages []ImageInfo
+ cachedAliases map[string]*ImageAliasesEntry
+}
+
+func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) {
+ if s.cachedIndex != nil {
+ return s.cachedIndex, nil
+ }
+
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/streams/v1/index.json", s.url), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", UserAgent)
+
+ r, err := s.http.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Body.Close()
+
+ body, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ // Parse the idnex
+ ssIndex := SimpleStreamsIndex{}
+ err = json.Unmarshal(body, &ssIndex)
+ if err != nil {
+ return nil, err
+ }
+
+ s.cachedIndex = &ssIndex
+
+ return &ssIndex, nil
+}
+
+func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) {
+ if s.cachedManifest[path] != nil {
+ return s.cachedManifest[path], nil
+ }
+
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", s.url, path), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", UserAgent)
+
+ r, err := s.http.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Body.Close()
+
+ body, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ // Parse the idnex
+ ssManifest := SimpleStreamsManifest{}
+ err = json.Unmarshal(body, &ssManifest)
+ if err != nil {
+ return nil, err
+ }
+
+ s.cachedManifest[path] = &ssManifest
+
+ return &ssManifest, nil
+}
+
+func (s *SimpleStreams) applyAliases(images []ImageInfo) ([]ImageInfo, map[string]*ImageAliasesEntry, error) {
+ aliases := map[string]*ImageAliasesEntry{}
+
+ sort.Sort(ssSortImage(images))
+
+ defaultOS := ""
+ for k, v := range ssDefaultOS {
+ if strings.HasPrefix(s.url, k) {
+ defaultOS = v
+ break
+ }
+ }
+
+ addAlias := func(name string, fingerprint string) *ImageAlias {
+ if defaultOS != "" {
+ name = strings.TrimPrefix(name, fmt.Sprintf("%s/", defaultOS))
+ }
+
+ if aliases[name] != nil {
+ return nil
+ }
+
+ alias := ImageAliasesEntry{}
+ alias.Name = name
+ alias.Target = fingerprint
+ aliases[name] = &alias
+
+ return &ImageAlias{Name: name}
+ }
+
+ uname := syscall.Utsname{}
+ if err := syscall.Uname(&uname); err != nil {
+ return nil, nil, err
+ }
+
+ architectureName := ""
+ for _, c := range uname.Machine {
+ if c == 0 {
+ break
+ }
+ architectureName += string(byte(c))
+ }
+
+ newImages := []ImageInfo{}
+ for _, image := range images {
+ // Short
+ if image.Architecture == architectureName {
+ alias := addAlias(fmt.Sprintf("%s/%s", image.Properties["os"], image.Properties["release"]), image.Fingerprint)
+ if alias != nil {
+ image.Aliases = append(image.Aliases, *alias)
+ }
+
+ alias = addAlias(fmt.Sprintf("%s/%s/%s", image.Properties["os"], image.Properties["release"], image.Properties["serial"]), image.Fingerprint)
+ if alias != nil {
+ image.Aliases = append(image.Aliases, *alias)
+ }
+ }
+
+ // Medium
+ alias := addAlias(fmt.Sprintf("%s/%s/%s", image.Properties["os"], image.Properties["release"], image.Properties["architecture"]), image.Fingerprint)
+ if alias != nil {
+ image.Aliases = append(image.Aliases, *alias)
+ }
+
+ // Medium
+ alias = addAlias(fmt.Sprintf("%s/%s/%s/%s", image.Properties["os"], image.Properties["release"], image.Properties["architecture"], image.Properties["serial"]), image.Fingerprint)
+ if alias != nil {
+ image.Aliases = append(image.Aliases, *alias)
+ }
+
+ newImages = append(newImages, image)
+ }
+
+ return newImages, aliases, nil
+}
+
+func (s *SimpleStreams) getImages() ([]ImageInfo, map[string]*ImageAliasesEntry, error) {
+ if s.cachedImages != nil && s.cachedAliases != nil {
+ return s.cachedImages, s.cachedAliases, nil
+ }
+
+ images := []ImageInfo{}
+
+ // Load the main index
+ ssIndex, err := s.parseIndex()
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // Iterate through the various image manifests
+ for _, entry := range ssIndex.Index {
+ // We only care about images
+ if entry.DataType != "image-downloads" {
+ continue
+ }
+
+ // No point downloading an empty image list
+ if len(entry.Products) == 0 {
+ continue
+ }
+
+ manifest, err := s.parseManifest(entry.Path)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ manifestImages, _ := manifest.ToLXD()
+
+ for _, image := range manifestImages {
+ images = append(images, image)
+ }
+ }
+
+ // Setup the aliases
+ images, aliases, err := s.applyAliases(images)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ s.cachedImages = images
+ s.cachedAliases = aliases
+
+ return images, aliases, nil
+}
+
+func (s *SimpleStreams) getPaths(fingerprint string) ([][]string, error) {
+ // Load the main index
+ ssIndex, err := s.parseIndex()
+ if err != nil {
+ return nil, err
+ }
+
+ // Iterate through the various image manifests
+ for _, entry := range ssIndex.Index {
+ // We only care about images
+ if entry.DataType != "image-downloads" {
+ continue
+ }
+
+ // No point downloading an empty image list
+ if len(entry.Products) == 0 {
+ continue
+ }
+
+ manifest, err := s.parseManifest(entry.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ manifestImages, downloads := manifest.ToLXD()
+
+ for _, image := range manifestImages {
+ if strings.HasPrefix(image.Fingerprint, fingerprint) {
+ urls := [][]string{}
+ for _, path := range downloads[image.Fingerprint] {
+ urls = append(urls, []string{path[0], path[1]})
+ }
+ return urls, nil
+ }
+ }
+ }
+
+ return nil, fmt.Errorf("Couldn't find the requested image")
+}
+
+func (s *SimpleStreams) ListAliases() (ImageAliases, error) {
+ _, aliasesMap, err := s.getImages()
+ if err != nil {
+ return nil, err
+ }
+
+ aliases := ImageAliases{}
+
+ for _, alias := range aliasesMap {
+ aliases = append(aliases, *alias)
+ }
+
+ return aliases, nil
+}
+
+func (s *SimpleStreams) ListImages() ([]ImageInfo, error) {
+ images, _, err := s.getImages()
+ return images, err
+}
+
+func (s *SimpleStreams) GetAlias(name string) string {
+ _, aliasesMap, err := s.getImages()
+ if err != nil {
+ return ""
+ }
+
+ alias, ok := aliasesMap[name]
+ if !ok {
+ return ""
+ }
+
+ return alias.Target
+}
+
+func (s *SimpleStreams) GetImageInfo(fingerprint string) (*ImageInfo, error) {
+ images, _, err := s.getImages()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, image := range images {
+ if strings.HasPrefix(image.Fingerprint, fingerprint) {
+ return &image, nil
+ }
+ }
+
+ return nil, fmt.Errorf("The requested image couldn't be found.")
+}
+
+func (s *SimpleStreams) downloadFile(path string, hash string, target string) error {
+ if !IsDir(target) {
+ return fmt.Errorf("Split images can only be written to a directory.")
+ }
+
+ download := func(url string, hash string, target string) error {
+ out, err := os.Create(target)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ resp, err := s.http.Get(url)
+ if err != nil {
+ }
+ defer resp.Body.Close()
+
+ sha256 := sha256.New()
+ _, err = io.Copy(io.MultiWriter(out, sha256), resp.Body)
+ if err != nil {
+ return err
+ }
+
+ if fmt.Sprintf("%x", sha256.Sum(nil)) != hash {
+ os.Remove(target)
+ return fmt.Errorf("Hash mismatch")
+ }
+
+ return nil
+ }
+
+ fields := strings.Split(path, "/")
+ targetFile := filepath.Join(target, fields[len(fields)-1])
+
+ // Try http first
+ if strings.HasPrefix(s.url, "https://") {
+ err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, targetFile)
+ if err == nil {
+ return nil
+ }
+ }
+
+ err := download(fmt.Sprintf("%s/%s", s.url, path), hash, targetFile)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *SimpleStreams) ExportImage(image string, target string) (string, error) {
+ paths, err := s.getPaths(image)
+ if err != nil {
+ return "", err
+ }
+
+ for _, path := range paths {
+ err := s.downloadFile(path[0], path[1], target)
+ if err != nil {
+ return "", err
+ }
+ }
+
+ return target, nil
+}
From c8779e067d231146554dac33c987bc670962df8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 26 Feb 2016 14:44:08 -0500
Subject: [PATCH 2/3] Add the images remote to default config
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
But not to static, so it can be removed.
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
config.go | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/config.go b/config.go
index ba06b97..8c935c5 100644
--- a/config.go
+++ b/config.go
@@ -51,6 +51,10 @@ var LocalRemote = RemoteConfig{
Static: true,
Public: false}
+var ImagesRemote = RemoteConfig{
+ Addr: "https://images.linuxcontainers.org",
+ Public: true}
+
var UbuntuRemote = RemoteConfig{
Addr: "https://cloud-images.ubuntu.com/releases",
Static: true,
@@ -68,8 +72,14 @@ var StaticRemotes = map[string]RemoteConfig{
"ubuntu": UbuntuRemote,
"ubuntu-daily": UbuntuDailyRemote}
+var DefaultRemotes = map[string]RemoteConfig{
+ "images": ImagesRemote,
+ "local": LocalRemote,
+ "ubuntu": UbuntuRemote,
+ "ubuntu-daily": UbuntuDailyRemote}
+
var DefaultConfig = Config{
- Remotes: StaticRemotes,
+ Remotes: DefaultRemotes,
DefaultRemote: "local",
Aliases: map[string]string{},
}
From 5e5641a40b75f5ba9db871c4aa23627251f57c02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 26 Feb 2016 00:46:02 -0500
Subject: [PATCH 3/3] Implement simplestreams support in the daemon
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/containers_post.go | 3 +-
lxd/daemon_images.go | 155 +++++++++++++++++++++++++++---------------------
lxd/images.go | 6 +-
shared/simplestreams.go | 113 ++++++++++++++++++++---------------
shared/util.go | 35 +++++++++++
specs/rest-api.md | 2 +
6 files changed, 196 insertions(+), 118 deletions(-)
diff --git a/lxd/containers_post.go b/lxd/containers_post.go
index 6b90ecc..b0f12d1 100644
--- a/lxd/containers_post.go
+++ b/lxd/containers_post.go
@@ -24,6 +24,7 @@ type containerImageSource struct {
Fingerprint string `json:"fingerprint"`
Server string `json:"server"`
Secret string `json:"secret"`
+ Protocol string `json:"protocol"`
/*
* for "migration" and "copy" types, as an optimization users can
@@ -80,7 +81,7 @@ func createFromImage(d *Daemon, req *containerPostReq) Response {
run := func(op *operation) error {
if req.Source.Server != "" {
- err := d.ImageDownload(op, req.Source.Server, req.Source.Certificate, req.Source.Secret, hash, true, false)
+ hash, err = d.ImageDownload(op, req.Source.Server, req.Source.Protocol, req.Source.Certificate, req.Source.Secret, hash, true)
if err != nil {
return err
}
diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go
index 2ae6a31..9773143 100644
--- a/lxd/daemon_images.go
+++ b/lxd/daemon_images.go
@@ -15,51 +15,28 @@ import (
log "gopkg.in/inconshreveable/log15.v2"
)
-type Progress struct {
- io.Reader
- total int64
- length int64
- percentage float64
- op *operation
-}
+// ImageDownload checks if we have that Image Fingerprint else
+// downloads the image from a remote server.
+func (d *Daemon) ImageDownload(op *operation, server string, protocol string, certificate string, secret string, fp string, forContainer bool) (string, error) {
+ var err error
+ var ss *shared.SimpleStreams
-func (pt *Progress) Read(p []byte) (int, error) {
- n, err := pt.Reader.Read(p)
- if n > 0 {
- pt.total += int64(n)
- percentage := float64(pt.total) / float64(pt.length) * float64(100)
-
- if percentage-pt.percentage > 0.9 && pt.op != nil {
- meta := pt.op.metadata
- if meta == nil {
- meta = make(map[string]interface{})
- }
-
- progressInt := 1 - (int(percentage) % 1) + int(percentage)
- if progressInt > 100 {
- progressInt = 100
- }
- progress := fmt.Sprintf("%d%%", progressInt)
-
- if meta["download_progress"] != progress {
- meta["download_progress"] = progress
- pt.op.UpdateMetadata(meta)
- }
-
- pt.percentage = percentage
+ if protocol == "simplestreams" {
+ ss, err = shared.SimpleStreamsClient(server)
+ if err != nil {
+ return "", err
}
- }
- return n, err
-}
+ fp = ss.GetAlias(fp)
+ if fp == "" {
+ return "", fmt.Errorf("The requested image couldn't be found.")
+ }
+ }
-// ImageDownload checks if we have that Image Fingerprint else
-// downloads the image from a remote server.
-func (d *Daemon) ImageDownload(op *operation, server string, certificate string, secret string, fp string, forContainer bool, directDownload bool) error {
if _, _, err := dbImageGet(d.db, fp, false, false); err == nil {
shared.Log.Debug("Image already exists in the db", log.Ctx{"image": fp})
// already have it
- return nil
+ return fp, nil
}
shared.Log.Info(
@@ -86,14 +63,14 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Previous download didn't succeed",
log.Ctx{"image": fp})
- return fmt.Errorf("Previous download didn't succeed")
+ return "", fmt.Errorf("Previous download didn't succeed")
}
shared.Log.Info(
"Previous download succeeded",
log.Ctx{"image": fp})
- return nil
+ return fp, nil
}
d.imagesDownloadingLock.RUnlock()
@@ -122,7 +99,31 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
var info shared.ImageInfo
info.Fingerprint = fp
- if !directDownload {
+ destDir := shared.VarPath("images")
+ destName := filepath.Join(destDir, fp)
+ if shared.PathExists(destName) {
+ d.Storage.ImageDelete(fp)
+ }
+
+ progress := func(progressInt int) {
+ if op == nil {
+ return
+ }
+
+ meta := op.metadata
+ if meta == nil {
+ meta = make(map[string]interface{})
+ }
+
+ progress := fmt.Sprintf("%d%%", progressInt)
+
+ if meta["download_progress"] != progress {
+ meta["download_progress"] = progress
+ op.UpdateMetadata(meta)
+ }
+ }
+
+ if protocol == "" || protocol == "lxc" {
/* grab the metadata from /1.0/images/%s */
var url string
if secret != "" {
@@ -139,11 +140,11 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Failed to download image metadata",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
if err := json.Unmarshal(resp.Metadata, &info); err != nil {
- return err
+ return "", err
}
/* now grab the actual file from /1.0/images/%s/export */
@@ -157,6 +158,34 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"%s/%s/images/%s/export",
server, shared.APIVersion, fp)
}
+ } else if protocol == "simplestreams" {
+ err := ss.Download(fp, "meta", destName, nil)
+ if err != nil {
+ return "", err
+ }
+
+ err = ss.Download(fp, "root", destName+".rootfs", progress)
+ if err != nil {
+ return "", err
+ }
+
+ info, err := ss.GetImageInfo(fp)
+ if err != nil {
+ return "", err
+ }
+
+ info.Public = false
+
+ _, err = imageBuildFromInfo(d, *info)
+ if err != nil {
+ return "", err
+ }
+
+ if forContainer {
+ return fp, dbImageLastAccessInit(d.db, fp)
+ }
+
+ return fp, nil
}
raw, err := d.httpGetFile(exporturl, certificate)
@@ -164,22 +193,16 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
shared.Log.Error(
"Failed to download image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
info.Size = raw.ContentLength
- destDir := shared.VarPath("images")
- destName := filepath.Join(destDir, fp)
- if shared.PathExists(destName) {
- d.Storage.ImageDelete(fp)
- }
-
ctype, ctypeParams, err := mime.ParseMediaType(raw.Header.Get("Content-Type"))
if err != nil {
ctype = "application/octet-stream"
}
- body := &Progress{Reader: raw.Body, length: raw.ContentLength, op: op}
+ body := &shared.TransferProgress{Reader: raw.Body, Length: raw.ContentLength, Handler: progress}
if ctype == "multipart/form-data" {
// Parse the POST data
@@ -192,7 +215,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Invalid multipart image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
if part.FormName() != "metadata" {
@@ -200,7 +223,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Invalid multipart image",
log.Ctx{"image": fp, "err": err})
- return fmt.Errorf("Invalid multipart image")
+ return "", fmt.Errorf("Invalid multipart image")
}
destName = filepath.Join(destDir, info.Fingerprint)
@@ -210,7 +233,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Failed to save image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
_, err = io.Copy(f, part)
@@ -221,7 +244,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Failed to save image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
// Get the rootfs tarball
@@ -231,14 +254,14 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Invalid multipart image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
if part.FormName() != "rootfs" {
shared.Log.Error(
"Invalid multipart image",
log.Ctx{"image": fp})
- return fmt.Errorf("Invalid multipart image")
+ return "", fmt.Errorf("Invalid multipart image")
}
destName = filepath.Join(destDir, info.Fingerprint+".rootfs")
@@ -247,7 +270,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
shared.Log.Error(
"Failed to save image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
_, err = io.Copy(f, part)
@@ -257,7 +280,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
shared.Log.Error(
"Failed to save image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
} else {
destName = filepath.Join(destDir, info.Fingerprint)
@@ -268,7 +291,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Failed to save image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
_, err = io.Copy(f, body)
@@ -278,14 +301,14 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
shared.Log.Error(
"Failed to save image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
}
- if directDownload {
+ if protocol == "direct" {
imageMeta, err := getImageMetadata(destName)
if err != nil {
- return err
+ return "", err
}
info.Architecture = imageMeta.Architecture
@@ -303,7 +326,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
"Failed to create image",
log.Ctx{"image": fp, "err": err})
- return err
+ return "", err
}
shared.Log.Info(
@@ -311,8 +334,8 @@ func (d *Daemon) ImageDownload(op *operation, server string, certificate string,
log.Ctx{"image": fp})
if forContainer {
- return dbImageLastAccessInit(d.db, fp)
+ return fp, dbImageLastAccessInit(d.db, fp)
}
- return nil
+ return fp, nil
}
diff --git a/lxd/images.go b/lxd/images.go
index 0f63f31..3cd3dd0 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -292,8 +292,7 @@ func imgPostRemoteInfo(d *Daemon, req imagePostReq, op *operation) error {
return fmt.Errorf("must specify one of alias or fingerprint for init from image")
}
- err = d.ImageDownload(op, req.Source["server"], req.Source["certificate"], req.Source["secret"], hash, false, false)
-
+ hash, err = d.ImageDownload(op, req.Source["server"], req.Source["protocol"], req.Source["certificate"], req.Source["secret"], hash, false)
if err != nil {
return err
}
@@ -377,8 +376,7 @@ func imgPostURLInfo(d *Daemon, req imagePostReq, op *operation) error {
}
// Import the image
- err = d.ImageDownload(op, url, "", "", hash, false, true)
-
+ hash, err = d.ImageDownload(op, url, "direct", "", "", hash, false)
if err != nil {
return err
}
diff --git a/shared/simplestreams.go b/shared/simplestreams.go
index e8b76a6..f2076f6 100644
--- a/shared/simplestreams.go
+++ b/shared/simplestreams.go
@@ -165,7 +165,7 @@ func (s *SimpleStreamsManifest) ToLXD() ([]ImageInfo, map[string][][]string) {
}
}
- downloads[fingerprint] = [][]string{[]string{metaPath, metaHash}, []string{rootfsPath, rootfsHash}}
+ downloads[fingerprint] = [][]string{[]string{metaPath, metaHash, "meta"}, []string{rootfsPath, rootfsHash, "root"}}
images = append(images, image)
}
}
@@ -469,7 +469,7 @@ func (s *SimpleStreams) getPaths(fingerprint string) ([][]string, error) {
if strings.HasPrefix(image.Fingerprint, fingerprint) {
urls := [][]string{}
for _, path := range downloads[image.Fingerprint] {
- urls = append(urls, []string{path[0], path[1]})
+ urls = append(urls, []string{path[0], path[1], path[2]})
}
return urls, nil
}
@@ -479,6 +479,51 @@ func (s *SimpleStreams) getPaths(fingerprint string) ([][]string, error) {
return nil, fmt.Errorf("Couldn't find the requested image")
}
+func (s *SimpleStreams) downloadFile(path string, hash string, target string, progress func(int)) error {
+ download := func(url string, hash string, target string) error {
+ out, err := os.Create(target)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ resp, err := s.http.Get(url)
+ if err != nil {
+ }
+ defer resp.Body.Close()
+
+ body := &TransferProgress{Reader: resp.Body, Length: resp.ContentLength, Handler: progress}
+
+ sha256 := sha256.New()
+ _, err = io.Copy(io.MultiWriter(out, sha256), body)
+ if err != nil {
+ return err
+ }
+
+ if fmt.Sprintf("%x", sha256.Sum(nil)) != hash {
+ os.Remove(target)
+ return fmt.Errorf("Hash mismatch")
+ }
+
+ return nil
+ }
+
+ // Try http first
+ if strings.HasPrefix(s.url, "https://") {
+ err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, target)
+ if err == nil {
+ return nil
+ }
+ }
+
+ err := download(fmt.Sprintf("%s/%s", s.url, path), hash, target)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
func (s *SimpleStreams) ListAliases() (ImageAliases, error) {
_, aliasesMap, err := s.getImages()
if err != nil {
@@ -528,68 +573,42 @@ func (s *SimpleStreams) GetImageInfo(fingerprint string) (*ImageInfo, error) {
return nil, fmt.Errorf("The requested image couldn't be found.")
}
-func (s *SimpleStreams) downloadFile(path string, hash string, target string) error {
+func (s *SimpleStreams) ExportImage(image string, target string) (string, error) {
if !IsDir(target) {
- return fmt.Errorf("Split images can only be written to a directory.")
+ return "", fmt.Errorf("Split images can only be written to a directory.")
}
- download := func(url string, hash string, target string) error {
- out, err := os.Create(target)
- if err != nil {
- return err
- }
- defer out.Close()
-
- resp, err := s.http.Get(url)
- if err != nil {
- }
- defer resp.Body.Close()
-
- sha256 := sha256.New()
- _, err = io.Copy(io.MultiWriter(out, sha256), resp.Body)
- if err != nil {
- return err
- }
-
- if fmt.Sprintf("%x", sha256.Sum(nil)) != hash {
- os.Remove(target)
- return fmt.Errorf("Hash mismatch")
- }
-
- return nil
+ paths, err := s.getPaths(image)
+ if err != nil {
+ return "", err
}
- fields := strings.Split(path, "/")
- targetFile := filepath.Join(target, fields[len(fields)-1])
+ for _, path := range paths {
+ fields := strings.Split(path[0], "/")
+ targetFile := filepath.Join(target, fields[len(fields)-1])
- // Try http first
- if strings.HasPrefix(s.url, "https://") {
- err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, targetFile)
- if err == nil {
- return nil
+ err := s.downloadFile(path[0], path[1], targetFile, nil)
+ if err != nil {
+ return "", err
}
}
- err := download(fmt.Sprintf("%s/%s", s.url, path), hash, targetFile)
- if err != nil {
- return err
- }
-
- return nil
+ return target, nil
}
-func (s *SimpleStreams) ExportImage(image string, target string) (string, error) {
+func (s *SimpleStreams) Download(image string, file string, target string, progress func(int)) error {
paths, err := s.getPaths(image)
if err != nil {
- return "", err
+ return err
}
for _, path := range paths {
- err := s.downloadFile(path[0], path[1], target)
- if err != nil {
- return "", err
+ if file != path[2] {
+ continue
}
+
+ return s.downloadFile(path[0], path[1], target, progress)
}
- return target, nil
+ return fmt.Errorf("The file couldn't be found.")
}
diff --git a/shared/util.go b/shared/util.go
index e398dfc..e851cfb 100644
--- a/shared/util.go
+++ b/shared/util.go
@@ -656,3 +656,38 @@ func ParseBitSizeString(input string) (int64, error) {
return valueInt * multiplicator, nil
}
+
+type TransferProgress struct {
+ io.Reader
+ percentage float64
+ total int64
+
+ Length int64
+ Handler func(int)
+}
+
+func (pt *TransferProgress) Read(p []byte) (int, error) {
+ n, err := pt.Reader.Read(p)
+
+ if pt.Handler == nil {
+ return n, err
+ }
+
+ if n > 0 {
+ pt.total += int64(n)
+ percentage := float64(pt.total) / float64(pt.Length) * float64(100)
+
+ if percentage-pt.percentage > 0.9 {
+ pt.percentage = percentage
+
+ progressInt := 1 - (int(percentage) % 1) + int(percentage)
+ if progressInt > 100 {
+ progressInt = 100
+ }
+
+ pt.Handler(progressInt)
+ }
+ }
+
+ return n, err
+}
diff --git a/specs/rest-api.md b/specs/rest-api.md
index 1c82ea7..1dcfbf2 100644
--- a/specs/rest-api.md
+++ b/specs/rest-api.md
@@ -410,6 +410,7 @@ Input (using a public remote image):
"source": {"type": "image", # Can be: "image", "migration", "copy" or "none"
"mode": "pull", # One of "local" (default) or "pull"
"server": "https://10.0.2.3:8443", # Remote server (pull mode only)
+ "protocol": "lxd", # Protocol (one of lxd or simplestreams, defaults to lxd)
"certificate": "PEM certificate", # Optional PEM certificate. If not mentioned, system CA is used.
"alias": "ubuntu/devel"}, # Name of the alias
}
@@ -1044,6 +1045,7 @@ In the source image case, the following dict must be used:
"type": "image",
"mode": "pull", # Only pull is supported for now
"server": "https://10.0.2.3:8443", # Remote server (pull mode only)
+ "protocol": "lxd", # Protocol (one of lxd or simplestreams, defaults to lxd)
"secret": "my-secret-string", # Secret (pull mode only, private images only)
"certificate": "PEM certificate", # Optional PEM certificate. If not mentioned, system CA is used.
"fingerprint": "SHA256", # Fingerprint of the image (must be set if alias isn't)
More information about the lxc-devel
mailing list