[lxc-devel] [lxd/master] Slurp file
tych0 on Github
lxc-bot at linuxcontainers.org
Mon Nov 28 18:52:53 UTC 2016
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20161128/8f394e8b/attachment.bin>
-------------- next part --------------
From cf02ecaead9b1ce288ec3a8c7861c51258e96020 Mon Sep 17 00:00:00 2001
From: Tycho Andersen <tycho.andersen at canonical.com>
Date: Mon, 28 Nov 2016 09:04:23 -0700
Subject: [PATCH 1/3] Change ContainerStart to take the name and path to start
Instead of taking the container and computing the name and path, let's just
take the name and path themselves. We'll use this in the next patch to
start the storage for a container that we haven't created in memory or in
the database yet, when we're reading its slurp file.
Signed-off-by: Tycho Andersen <tycho.andersen at canonical.com>
---
lxd/container_lxc.go | 4 ++--
lxd/storage.go | 16 ++++++++--------
lxd/storage_btrfs.go | 4 ++--
lxd/storage_dir.go | 4 ++--
lxd/storage_lvm.go | 20 ++++++++++----------
lxd/storage_test.go | 4 ++--
lxd/storage_zfs.go | 6 +++---
7 files changed, 29 insertions(+), 29 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 82a91d3..8419cb6 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -4663,7 +4663,7 @@ func (c *containerLXC) StorageStart() error {
return c.storage.ContainerSnapshotStart(c)
}
- return c.storage.ContainerStart(c)
+ return c.storage.ContainerStart(c.Name(), c.Path())
}
func (c *containerLXC) StorageStop() error {
@@ -4671,7 +4671,7 @@ func (c *containerLXC) StorageStop() error {
return c.storage.ContainerSnapshotStop(c)
}
- return c.storage.ContainerStop(c)
+ return c.storage.ContainerStop(c.Name(), c.Path())
}
// Mount handling
diff --git a/lxd/storage.go b/lxd/storage.go
index a41ca1b..2ae706d 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -149,8 +149,8 @@ type storage interface {
ContainerCanRestore(container container, sourceContainer container) error
ContainerDelete(container container) error
ContainerCopy(container container, sourceContainer container) error
- ContainerStart(container container) error
- ContainerStop(container container) error
+ ContainerStart(name string, path string) error
+ ContainerStop(name string, path string) error
ContainerRename(container container, newName string) error
ContainerRestore(container container, sourceContainer container) error
ContainerSetQuota(container container, size int64) error
@@ -431,14 +431,14 @@ func (lw *storageLogWrapper) ContainerCopy(
return lw.w.ContainerCopy(container, sourceContainer)
}
-func (lw *storageLogWrapper) ContainerStart(container container) error {
- lw.log.Debug("ContainerStart", log.Ctx{"container": container.Name()})
- return lw.w.ContainerStart(container)
+func (lw *storageLogWrapper) ContainerStart(name string, path string) error {
+ lw.log.Debug("ContainerStart", log.Ctx{"container": name})
+ return lw.w.ContainerStart(name, path)
}
-func (lw *storageLogWrapper) ContainerStop(container container) error {
- lw.log.Debug("ContainerStop", log.Ctx{"container": container.Name()})
- return lw.w.ContainerStop(container)
+func (lw *storageLogWrapper) ContainerStop(name string, path string) error {
+ lw.log.Debug("ContainerStop", log.Ctx{"container": name})
+ return lw.w.ContainerStop(name, path)
}
func (lw *storageLogWrapper) ContainerRename(
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 4a8d63f..cf70b21 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -172,11 +172,11 @@ func (s *storageBtrfs) ContainerCopy(container container, sourceContainer contai
return container.TemplateApply("copy")
}
-func (s *storageBtrfs) ContainerStart(container container) error {
+func (s *storageBtrfs) ContainerStart(name string, path string) error {
return nil
}
-func (s *storageBtrfs) ContainerStop(container container) error {
+func (s *storageBtrfs) ContainerStop(name string, path string) error {
return nil
}
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index 18f4985..5adcd45 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -123,11 +123,11 @@ func (s *storageDir) ContainerCopy(
return container.TemplateApply("copy")
}
-func (s *storageDir) ContainerStart(container container) error {
+func (s *storageDir) ContainerStart(name string, path string) error {
return nil
}
-func (s *storageDir) ContainerStop(container container) error {
+func (s *storageDir) ContainerStop(name string, path string) error {
return nil
}
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index 3e9404d..a3bc02f 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -404,7 +404,7 @@ func (s *storageLvm) ContainerCopy(container container, sourceContainer containe
return err
}
- if err := s.ContainerStart(container); err != nil {
+ if err := s.ContainerStart(container.Name(), container.Path()); err != nil {
s.log.Error("Error starting/mounting container", log.Ctx{"err": err, "container": container.Name()})
s.ContainerDelete(container)
return err
@@ -419,36 +419,36 @@ func (s *storageLvm) ContainerCopy(container container, sourceContainer containe
return fmt.Errorf("rsync failed: %s", string(output))
}
- if err := s.ContainerStop(container); err != nil {
+ if err := s.ContainerStop(container.Name(), container.Path()); err != nil {
return err
}
}
return container.TemplateApply("copy")
}
-func (s *storageLvm) ContainerStart(container container) error {
- lvName := containerNameToLVName(container.Name())
+func (s *storageLvm) ContainerStart(name string, path string) error {
+ lvName := containerNameToLVName(name)
lvpath := fmt.Sprintf("/dev/%s/%s", s.vgName, lvName)
fstype := daemonConfig["storage.lvm_fstype"].Get()
mountOptions := daemonConfig["storage.lvm_mount_options"].Get()
- err := tryMount(lvpath, container.Path(), fstype, 0, mountOptions)
+ err := tryMount(lvpath, path, fstype, 0, mountOptions)
if err != nil {
return fmt.Errorf(
"Error mounting snapshot LV path='%s': %v",
- container.Path(),
+ path,
err)
}
return nil
}
-func (s *storageLvm) ContainerStop(container container) error {
- err := tryUnmount(container.Path(), 0)
+func (s *storageLvm) ContainerStop(name string, path string) error {
+ err := tryUnmount(path, 0)
if err != nil {
return fmt.Errorf(
"failed to unmount container path '%s'.\nError: %v",
- container.Path(),
+ path,
err)
}
@@ -690,7 +690,7 @@ func (s *storageLvm) ContainerSnapshotStart(container container) error {
}
func (s *storageLvm) ContainerSnapshotStop(container container) error {
- err := s.ContainerStop(container)
+ err := s.ContainerStop(container.Name(), container.Path())
if err != nil {
return err
}
diff --git a/lxd/storage_test.go b/lxd/storage_test.go
index bd723d4..63006c9 100644
--- a/lxd/storage_test.go
+++ b/lxd/storage_test.go
@@ -61,11 +61,11 @@ func (s *storageMock) ContainerCopy(
return nil
}
-func (s *storageMock) ContainerStart(container container) error {
+func (s *storageMock) ContainerStart(name string, path string) error {
return nil
}
-func (s *storageMock) ContainerStop(container container) error {
+func (s *storageMock) ContainerStop(name string, path string) error {
return nil
}
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 333ac12..6e12f31 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -92,8 +92,8 @@ func (s *storageZfs) Init(config map[string]interface{}) (storage, error) {
}
// Things we don't need to care about
-func (s *storageZfs) ContainerStart(container container) error {
- fs := fmt.Sprintf("containers/%s", container.Name())
+func (s *storageZfs) ContainerStart(name string, path string) error {
+ fs := fmt.Sprintf("containers/%s", name)
// Just in case the container filesystem got unmounted
if !shared.IsMountPoint(shared.VarPath(fs)) {
@@ -103,7 +103,7 @@ func (s *storageZfs) ContainerStart(container container) error {
return nil
}
-func (s *storageZfs) ContainerStop(container container) error {
+func (s *storageZfs) ContainerStop(name string, path string) error {
return nil
}
From 017b304ead613f74a85c1fb2564d591b5d661f96 Mon Sep 17 00:00:00 2001
From: Tycho Andersen <tycho.andersen at canonical.com>
Date: Mon, 28 Nov 2016 18:40:45 +0000
Subject: [PATCH 2/3] rework EEXISTS detection on create
In particular: let's use the database and not stat the filesystem, so that
if we are doing a `lxd import`, it doesn't fail.
Signed-off-by: Tycho Andersen <tycho.andersen at canonical.com>
---
lxd/container.go | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index 95e9c12..3d00e46 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -634,22 +634,22 @@ func containerCreateInternal(d *Daemon, args containerArgs) (container, error) {
}
}
- path := containerPath(args.Name, args.Ctype == cTypeSnapshot)
- if shared.PathExists(path) {
- if shared.IsSnapshot(args.Name) {
- return nil, fmt.Errorf("Snapshot '%s' already exists", args.Name)
+ // Create the container entry
+ id, err := dbContainerCreate(d.db, args)
+ if err != nil {
+ if err == DbErrAlreadyDefined {
+ thing := "Container"
+ if shared.IsSnapshot(args.Name) {
+ thing = "Snapshot"
+ }
+ return nil, fmt.Errorf("%s '%s' already exists", thing, args.Name)
}
- return nil, fmt.Errorf("The container already exists")
+ return nil, err
}
// Wipe any existing log for this container name
os.RemoveAll(shared.LogPath(args.Name))
- // Create the container entry
- id, err := dbContainerCreate(d.db, args)
- if err != nil {
- return nil, err
- }
args.Id = id
// Read the timestamp from the database
From 189641641f1effe9d53449d3e0a14f08b91bff69 Mon Sep 17 00:00:00 2001
From: Tycho Andersen <tycho.andersen at canonical.com>
Date: Tue, 8 Nov 2016 16:56:40 +0200
Subject: [PATCH 3/3] implement `lxd import`
The basic idea here is to maintain a file in the container's directory that
indicates what configuration and snapshots it has, so that if someone just
backs up the filesystem the container is on (e.g. the zfs pool), they can
re-import their containers with a simple `lxd import` command.
Closes #2286
Signed-off-by: Tycho Andersen <tycho.andersen at canonical.com>
---
lxd/api_internal.go | 102 +++++++++++++++++++++++++++++++++++++++++++++++
lxd/container.go | 12 +++++-
lxd/container_lxc.go | 75 ++++++++++++++++++++++++++++++++++
lxd/main.go | 28 +++++++++++++
shared/log.go | 2 +-
test/suites/migration.sh | 9 +++++
6 files changed, 226 insertions(+), 2 deletions(-)
diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index 51c11e3..11ca5aa 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -2,10 +2,13 @@ package main
import (
"fmt"
+ "io/ioutil"
"net/http"
"strconv"
+ "strings"
"github.com/gorilla/mux"
+ "gopkg.in/yaml.v2"
"github.com/lxc/lxd/shared"
@@ -17,6 +20,7 @@ var apiInternal = []Command{
internalShutdownCmd,
internalContainerOnStartCmd,
internalContainerOnStopCmd,
+ internalContainersCmd,
}
func internalReady(d *Daemon, r *http.Request) Response {
@@ -95,3 +99,101 @@ var internalShutdownCmd = Command{name: "shutdown", put: internalShutdown}
var internalReadyCmd = Command{name: "ready", put: internalReady, get: internalWaitReady}
var internalContainerOnStartCmd = Command{name: "containers/{id}/onstart", get: internalContainerOnStart}
var internalContainerOnStopCmd = Command{name: "containers/{id}/onstop", get: internalContainerOnStop}
+
+func slurpSlurpFile(path string) (*slurpFile, error) {
+ data, err := ioutil.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+
+ sf := slurpFile{}
+
+ if err := yaml.Unmarshal(data, &sf); err != nil {
+ return nil, err
+ }
+
+ return &sf, nil
+}
+
+func internalImport(d *Daemon, r *http.Request) Response {
+ name := r.FormValue("target")
+ if name == "" {
+ return BadRequest(fmt.Errorf("target is required"))
+ }
+
+ path := containerPath(name, false)
+ err := d.Storage.ContainerStart(name, path)
+ if err != nil {
+ return SmartError(err)
+ }
+
+ defer d.Storage.ContainerStop(name, path)
+
+ sf, err := slurpSlurpFile(shared.VarPath("containers", name, "backup.yaml"))
+ if err != nil {
+ return SmartError(err)
+ }
+
+ baseImage := sf.Container.Config["volatile.base_image"]
+ for k := range sf.Container.Config {
+ if strings.HasPrefix(k, "volatile") {
+ delete(sf.Container.Config, k)
+ }
+ }
+
+ arch, err := shared.ArchitectureId(sf.Container.Architecture)
+ if err != nil {
+ return SmartError(err)
+ }
+ _, err = containerCreateInternal(d, containerArgs{
+ Architecture: arch,
+ BaseImage: baseImage,
+ Config: sf.Container.Config,
+ CreationDate: sf.Container.CreationDate,
+ LastUsedDate: sf.Container.LastUsedDate,
+ Ctype: cTypeRegular,
+ Devices: sf.Container.Devices,
+ Ephemeral: sf.Container.Ephemeral,
+ Name: sf.Container.Name,
+ Profiles: sf.Container.Profiles,
+ Stateful: sf.Container.Stateful,
+ })
+ if err != nil {
+ return SmartError(err)
+ }
+
+ for _, snap := range sf.Snapshots {
+ baseImage := snap.Config["volatile.base_image"]
+ for k := range snap.Config {
+ if strings.HasPrefix(k, "volatile") {
+ delete(snap.Config, k)
+ }
+ }
+
+ arch, err := shared.ArchitectureId(snap.Architecture)
+ if err != nil {
+ return SmartError(err)
+ }
+
+ _, err = containerCreateInternal(d, containerArgs{
+ Architecture: arch,
+ BaseImage: baseImage,
+ Config: snap.Config,
+ CreationDate: snap.CreationDate,
+ LastUsedDate: snap.LastUsedDate,
+ Ctype: cTypeSnapshot,
+ Devices: snap.Devices,
+ Ephemeral: snap.Ephemeral,
+ Name: snap.Name,
+ Profiles: snap.Profiles,
+ Stateful: snap.Stateful,
+ })
+ if err != nil {
+ return SmartError(err)
+ }
+ }
+
+ return EmptySyncResponse
+}
+
+var internalContainersCmd = Command{name: "containers", post: internalImport}
diff --git a/lxd/container.go b/lxd/container.go
index 3d00e46..8c8170f 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -660,7 +660,12 @@ func containerCreateInternal(d *Daemon, args containerArgs) (container, error) {
args.CreationDate = dbArgs.CreationDate
args.LastUsedDate = dbArgs.LastUsedDate
- return containerLXCCreate(d, args)
+ c, err := containerLXCCreate(d, args)
+ if err != nil {
+ return nil, err
+ }
+
+ return c, nil
}
func containerConfigureInternal(c container) error {
@@ -683,6 +688,11 @@ func containerConfigureInternal(c container) error {
break
}
+ err := writeSlurpFile(c)
+ if err != nil {
+ return err
+ }
+
return nil
}
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 8419cb6..9602965 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -2511,6 +2511,14 @@ func (c *containerLXC) Restore(sourceContainer container) error {
return err
}
+ // The old slurp file may be out of date (e.g. it doesn't have all the
+ // current snapshots of the container listed); let's write a new one to
+ // be safe.
+ err = writeSlurpFile(c)
+ if err != nil {
+ return err
+ }
+
// If the container wasn't running but was stateful, should we restore
// it as running?
if shared.PathExists(c.StatePath()) {
@@ -2738,6 +2746,65 @@ func (c *containerLXC) ConfigKeySet(key string, value string) error {
return c.Update(args, false)
}
+type slurpFile struct {
+ Container *shared.ContainerInfo `yaml:"container"`
+ Snapshots []*shared.SnapshotInfo `yaml:"snapshots"`
+}
+
+func writeSlurpFile(c container) error {
+ /* we only write slurp files out for actual containers */
+ if c.IsSnapshot() {
+ return nil
+ }
+
+ ci, _, err := c.Render()
+ if err != nil {
+ return err
+ }
+
+ snapshots, err := c.Snapshots()
+ if err != nil {
+ return err
+ }
+
+ var sis []*shared.SnapshotInfo
+
+ for _, s := range snapshots {
+ si, _, err := s.Render()
+ if err != nil {
+ return err
+ }
+
+ sis = append(sis, si.(*shared.SnapshotInfo))
+ }
+
+ data, err := yaml.Marshal(&slurpFile{
+ Container: ci.(*shared.ContainerInfo),
+ Snapshots: sis,
+ })
+ if err != nil {
+ return err
+ }
+
+ f, err := os.Create(shared.VarPath("containers", c.Name(), "backup.yaml"))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ err = f.Chmod(0400)
+ if err != nil {
+ return err
+ }
+
+ err = shared.WriteAll(f, data)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
func (c *containerLXC) Update(args containerArgs, userRequested bool) error {
// Set sane defaults for unset keys
if args.Architecture == 0 {
@@ -3486,6 +3553,14 @@ func (c *containerLXC) Update(args containerArgs, userRequested bool) error {
return err
}
+ /* we can call Update in some cases when the directory doesn't exist
+ * yet before container creation; this is okay, because at the end of
+ * container creation we write the slurp file, so let's not worry about
+ * ENOENT. */
+ if err := writeSlurpFile(c); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+
// Update network leases
needsUpdate := false
for _, m := range updateDevices {
diff --git a/lxd/main.go b/lxd/main.go
index cba11bb..02204b5 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -88,6 +88,8 @@ func run() error {
fmt.Printf(" Perform a clean shutdown of LXD and all running containers\n")
fmt.Printf(" waitready [--timeout=15]\n")
fmt.Printf(" Wait until LXD is ready to handle requests\n")
+ fmt.Printf(" import <container name>\n")
+ fmt.Printf(" Import a pre-existing container from storage\n")
fmt.Printf("\n\nCommon options:\n")
fmt.Printf(" --debug\n")
@@ -223,6 +225,8 @@ func run() error {
return cmdShutdown()
case "waitready":
return cmdWaitReady()
+ case "import":
+ return cmdImport(os.Args[1:])
// Internal commands
case "forkgetnet":
@@ -1218,3 +1222,27 @@ func cmdMigrateDumpSuccess(args []string) error {
return c.WaitForSuccess(args[1])
}
+
+func cmdImport(args []string) error {
+ name := args[1]
+
+ c, err := lxd.NewClient(&lxd.DefaultConfig, "local")
+ if err != nil {
+ return err
+ }
+
+ url := fmt.Sprintf("%s/internal/containers?target=%s", c.BaseURL, name)
+
+ req, err := http.NewRequest("POST", url, nil)
+ if err != nil {
+ return err
+ }
+
+ raw, err := c.Http.Do(req)
+ _, err = lxd.HoistResponse(raw, lxd.Sync)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/shared/log.go b/shared/log.go
index 792ccf6..f696feb 100644
--- a/shared/log.go
+++ b/shared/log.go
@@ -93,5 +93,5 @@ func LogCritf(format string, args ...interface{}) {
func PrintStack() {
buf := make([]byte, 1<<16)
runtime.Stack(buf, true)
- LogDebugf("%s", buf)
+ LogErrorf("%s", buf)
}
diff --git a/test/suites/migration.sh b/test/suites/migration.sh
index 675049f..a3bf48f 100644
--- a/test/suites/migration.sh
+++ b/test/suites/migration.sh
@@ -21,6 +21,15 @@ test_migration() {
lxc_remote move l1:nonlive l2:
lxc_remote config show l2:nonlive/snap0 | grep user.tester | grep foo
+ # test `lxd import`
+ if [ "${LXD_BACKEND}" = "zfs" ]; then
+ lxc_remote init testimage backup
+ lxc_remote snapshot backup
+ sqlite3 "${LXD_DIR}/lxd.db" "DELETE FROM containers WHERE name='backup'"
+ lxd import backup
+ lxc_remote info backup | grep snap0
+ fi
+
# FIXME: make this backend agnostic
if [ "${LXD_BACKEND}" != "lvm" ]; then
[ -d "${LXD2_DIR}/containers/nonlive/rootfs" ]
More information about the lxc-devel
mailing list