[lxc-devel] [lxd/master] Backup: Adds OptimizedStorage and Type fields to index.yaml
tomponline on Github
lxc-bot at linuxcontainers.org
Mon Mar 23 13:31:05 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200323/03844d33/attachment.bin>
-------------- next part --------------
From 3e70f31d8a7bfa03f862b487837ef8a3a0db7d28 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 20 Mar 2020 16:26:03 +0000
Subject: [PATCH 1/5] lxd/backup: Removes Privileged field from backup.Info
struct
It is unused on restoration, and not relevant for VMs as well.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup.go | 9 ++++-----
lxd/backup/backup.go | 1 -
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index 306af17109..e91cb4d27c 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -177,11 +177,10 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
// backupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball.
func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, instanceOnly bool, indexFile string, tarWriter *instancewriter.InstanceTarWriter) error {
indexInfo := backup.Info{
- Name: sourceInst.Name(),
- Privileged: sourceInst.IsPrivileged(),
- Pool: pool.Name(),
- Snapshots: []string{},
- Backend: pool.Driver().Info().Name,
+ Name: sourceInst.Name(),
+ Pool: pool.Name(),
+ Snapshots: []string{},
+ Backend: pool.Driver().Info().Name,
}
if !instanceOnly {
diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go
index f0cec28c62..4de3f0d740 100644
--- a/lxd/backup/backup.go
+++ b/lxd/backup/backup.go
@@ -29,7 +29,6 @@ type Info struct {
Project string `json:"project" yaml:"project"`
Name string `json:"name" yaml:"name"`
Backend string `json:"backend" yaml:"backend"`
- Privileged bool `json:"privileged" yaml:"privileged"`
Pool string `json:"pool" yaml:"pool"`
Snapshots []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
OptimizedStorage bool `json:"-" yaml:"-"`
From 983a2b9b8e1f5750c4b7f238af4e6e97f4a4f0ed Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 23 Mar 2020 12:21:40 +0000
Subject: [PATCH 2/5] lxd/backup: Updates GetInfo to handle new fields in
index.yaml
Handles `type` field in, and defaults to container type if not specified.
Handles `optimized` field, defaults to false if not specified (but still supports old style optimized hint using presence of container.bin file).
Adds optimisation to avoid reading whole tarball once index.yaml is read and if `optimized` is provided in the yaml file.
Adds optimisation to avoid reading whole tarball if the container.bin optimized hint file is found.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup/backup.go | 49 +++++++++++++++++++++++++++++++++++---------
1 file changed, 39 insertions(+), 10 deletions(-)
diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go
index 4de3f0d740..805383a6a9 100644
--- a/lxd/backup/backup.go
+++ b/lxd/backup/backup.go
@@ -2,6 +2,7 @@ package backup
import (
"archive/tar"
+ "context"
"fmt"
"io"
"os"
@@ -26,21 +27,25 @@ type Instance interface {
// Info represents exported backup information.
type Info struct {
- Project string `json:"project" yaml:"project"`
- Name string `json:"name" yaml:"name"`
- Backend string `json:"backend" yaml:"backend"`
- Pool string `json:"pool" yaml:"pool"`
- Snapshots []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
- OptimizedStorage bool `json:"-" yaml:"-"`
+ Project string `json:"project" yaml:"project"`
+ Name string `json:"name" yaml:"name"`
+ Backend string `json:"backend" yaml:"backend"`
+ Pool string `json:"pool" yaml:"pool"`
+ Snapshots []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
+ OptimizedStorage *bool `json:"optimized,omitempty" yaml:"optimized,omitempty"` // Optional field to handle older optimized backups that don't have this field.
+ Type api.InstanceType `json:"type" yaml:"type"`
}
// GetInfo extracts backup information from a given ReadSeeker.
func GetInfo(r io.ReadSeeker) (*Info, error) {
var tr *tar.Reader
result := Info{}
- optimizedStorage := false
hasIndexFile := false
+ // Define some bools used to create points for OptimizedStorage field.
+ optimizedStorageTrue := true
+ optimizedStorageFalse := false
+
// Extract
r.Seek(0, 0)
_, _, unpacker, err := shared.DetectCompressionFile(r)
@@ -53,8 +58,10 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
return nil, fmt.Errorf("Unsupported backup compression")
}
+ ctx, cancelFunc := context.WithCancel(context.Background())
+
if len(unpacker) > 0 {
- cmd := exec.Command(unpacker[0], unpacker[1:]...)
+ cmd := exec.CommandContext(ctx, unpacker[0], unpacker[1:]...)
cmd.Stdin = r
stdout, err := cmd.StdoutPipe()
@@ -90,10 +97,33 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
}
hasIndexFile = true
+
+ // Default to container if index doesn't specify instance type.
+ if result.Type == api.InstanceTypeAny {
+ result.Type = api.InstanceTypeContainer
+ }
+
+ if result.OptimizedStorage != nil {
+ // No need to continue looking for optimized storage hint using the presence of the
+ // container.bin file below, as the index.yaml file tells us directly.
+ cancelFunc()
+ break
+ } else {
+ // Default to non-optimized if not specified and continue reading to see if
+ // optimized container.bin file present.
+ result.OptimizedStorage = &optimizedStorageFalse
+ }
}
+ // If the tarball contains a binary dump of the container, then this is an optimized backup.
if hdr.Name == "backup/container.bin" {
- optimizedStorage = true
+ result.OptimizedStorage = &optimizedStorageTrue
+
+ // Stop read loop if index.yaml already parsed.
+ if hasIndexFile {
+ cancelFunc()
+ break
+ }
}
}
@@ -101,7 +131,6 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
return nil, fmt.Errorf("Backup is missing index.yaml")
}
- result.OptimizedStorage = optimizedStorage
return &result, nil
}
From c08e8dc41d675f5d0f5f30789efb1111bbaa2bc0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 23 Mar 2020 12:24:17 +0000
Subject: [PATCH 3/5] lxd/instances/post: bInfo.OptimizedStorage pointer usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instances_post.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/instances_post.go b/lxd/instances_post.go
index a4fcc8f036..87e6be7235 100644
--- a/lxd/instances_post.go
+++ b/lxd/instances_post.go
@@ -604,7 +604,7 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re
// The storage pool doesn't exist. If backup is in binary format (so we cannot alter
// the backup.yaml) or the pool has been specified directly from the user restoring
// the backup then we cannot proceed so return an error.
- if bInfo.OptimizedStorage || pool != "" {
+ if *bInfo.OptimizedStorage || pool != "" {
return response.InternalError(errors.Wrap(err, "Storage pool not found"))
}
From 708edfbe58ebf4c5397ad47a95601203eff23160 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 23 Mar 2020 12:24:46 +0000
Subject: [PATCH 4/5] lxd/storage/backend/lxd: b.driver.CreateVolumeFromBackup
OptimizedStorage pointer usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/backend_lxd.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/storage/backend_lxd.go b/lxd/storage/backend_lxd.go
index 8b2403d9c0..07b8391e13 100644
--- a/lxd/storage/backend_lxd.go
+++ b/lxd/storage/backend_lxd.go
@@ -492,7 +492,7 @@ func (b *lxdBackend) CreateInstanceFromBackup(srcBackup backup.Info, srcData io.
defer revert.Fail()
// Unpack the backup into the new storage volume(s).
- volPostHook, revertHook, err := b.driver.CreateVolumeFromBackup(vol, srcBackup.Snapshots, srcData, srcBackup.OptimizedStorage, op)
+ volPostHook, revertHook, err := b.driver.CreateVolumeFromBackup(vol, srcBackup.Snapshots, srcData, *srcBackup.OptimizedStorage, op)
if err != nil {
return nil, nil, err
}
From 4fef9d2f4749e80d0b8ecd2c02a1a6d454a42004 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 23 Mar 2020 12:19:44 +0000
Subject: [PATCH 5/5] lxd/backup: Updates backupWriteIndex index.yaml fields
Changes instanceOnly arg to snapshots.
Removes `privileged` field.
Adds `optimized` field to indicate if optimized backup format is used.
Adds `type` field to indicate the instance type being backed up.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/backup.go | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/lxd/backup.go b/lxd/backup.go
index e91cb4d27c..f1f93ea033 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -25,6 +25,7 @@ import (
storagePools "github.com/lxc/lxd/lxd/storage"
"github.com/lxc/lxd/lxd/task"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/idmap"
"github.com/lxc/lxd/shared/instancewriter"
log "github.com/lxc/lxd/shared/log15"
@@ -143,7 +144,7 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
// Write index file.
indexFile := filepath.Join(tmpDirPath, "index.yaml")
logger.Debug("Adding backup index file", log.Ctx{"path": indexFile})
- err = backupWriteIndex(sourceInst, pool, b.InstanceOnly(), indexFile, tarWriter)
+ err = backupWriteIndex(sourceInst, pool, b.OptimizedStorage(), !b.InstanceOnly(), indexFile, tarWriter)
if err != nil {
return errors.Wrapf(err, "Error writing backup index file")
}
@@ -175,15 +176,18 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instanc
}
// backupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball.
-func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, instanceOnly bool, indexFile string, tarWriter *instancewriter.InstanceTarWriter) error {
+func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, optimized bool, snapshots bool, indexFile string, tarWriter *instancewriter.InstanceTarWriter) error {
+
indexInfo := backup.Info{
- Name: sourceInst.Name(),
- Pool: pool.Name(),
- Snapshots: []string{},
- Backend: pool.Driver().Info().Name,
+ Name: sourceInst.Name(),
+ Pool: pool.Name(),
+ Snapshots: []string{},
+ Backend: pool.Driver().Info().Name,
+ Type: api.InstanceType(sourceInst.Type().String()),
+ OptimizedStorage: &optimized,
}
- if !instanceOnly {
+ if snapshots {
snaps, err := sourceInst.Snapshots()
if err != nil {
return err
More information about the lxc-devel
mailing list