[lxc-devel] [lxd/master] Backup: Change Type field type in index.yaml to own backup.Type

tomponline on Github lxc-bot at linuxcontainers.org
Fri Sep 25 08:20:06 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 426 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200925/8a57335f/attachment-0001.bin>
-------------- next part --------------
From 818c804c5324ba02b5fa62fbed9e07ece0ea105c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:13:53 +0100
Subject: [PATCH 01/14] lxd/backup/backup/utils: Adds TarReader function

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/backup/backup_utils.go | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 lxd/backup/backup_utils.go

diff --git a/lxd/backup/backup_utils.go b/lxd/backup/backup_utils.go
new file mode 100644
index 0000000000..08dc35918c
--- /dev/null
+++ b/lxd/backup/backup_utils.go
@@ -0,0 +1,30 @@
+package backup
+
+import (
+	"archive/tar"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/lxc/lxd/shared"
+)
+
+// TarReader rewinds backup file handle r and returns new tar reader and process cleanup function.
+func TarReader(r io.ReadSeeker) (*tar.Reader, context.CancelFunc, error) {
+	r.Seek(0, 0)
+	_, _, unpacker, err := shared.DetectCompressionFile(r)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if unpacker == nil {
+		return nil, nil, fmt.Errorf("Unsupported backup compression")
+	}
+
+	tr, cancelFunc, err := shared.CompressedTarReader(context.Background(), r, unpacker)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return tr, cancelFunc, nil
+}

From 14b1bba74fb936221f1e53ac75c9baecde30e006 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:28:57 +0100
Subject: [PATCH 02/14] lxd/backup/backup/info: Changes Type field from
 api.InstanceType to Type

To accomodate custom volumes.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/backup/backup_info.go | 45 ++++++++++++++++++++++++++++++---------
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/lxd/backup/backup_info.go b/lxd/backup/backup_info.go
index 1c8e47083e..8e2c2a7da2 100644
--- a/lxd/backup/backup_info.go
+++ b/lxd/backup/backup_info.go
@@ -1,27 +1,52 @@
 package backup
 
 import (
-	"context"
 	"fmt"
 	"io"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 
-	"github.com/lxc/lxd/shared"
 	"github.com/lxc/lxd/shared/api"
 )
 
+// Type indicates the type of backup.
+type Type string
+
+// TypeUnknown defines the backup type value for unknown backups.
+const TypeUnknown = Type("")
+
+// TypeContainer defines the backup type value for a container.
+const TypeContainer = Type("container")
+
+// TypeVM defines the backup type value for a virtual-machine.
+const TypeVM = Type("virtual-machine")
+
+// TypeCustom defines the backup type value for a custom volume.
+const TypeCustom = Type("custom")
+
+// InstanceTypeToBackupType converts instance type to backup type.
+func InstanceTypeToBackupType(instanceType api.InstanceType) Type {
+	switch instanceType {
+	case api.InstanceTypeContainer:
+		return TypeContainer
+	case api.InstanceTypeVM:
+		return TypeVM
+	}
+
+	return TypeUnknown
+}
+
 // Info represents exported backup information.
 type Info struct {
-	Project          string           `json:"-" yaml:"-"` // Project is set during import based on current project.
-	Name             string           `json:"name" yaml:"name"`
-	Backend          string           `json:"backend" yaml:"backend"`
-	Pool             string           `json:"pool" yaml:"pool"`
-	Snapshots        []string         `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
-	OptimizedStorage *bool            `json:"optimized,omitempty" yaml:"optimized,omitempty"`               // Optional field to handle older optimized backups that don't have this field.
-	OptimizedHeader  *bool            `json:"optimized_header,omitempty" yaml:"optimized_header,omitempty"` // Optional field to handle older optimized backups that don't have this field.
-	Type             api.InstanceType `json:"type" yaml:"type"`
+	Project          string   `json:"-" yaml:"-"` // Project is set during import based on current project.
+	Name             string   `json:"name" yaml:"name"`
+	Backend          string   `json:"backend" yaml:"backend"`
+	Pool             string   `json:"pool" yaml:"pool"`
+	Snapshots        []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
+	OptimizedStorage *bool    `json:"optimized,omitempty" yaml:"optimized,omitempty"`               // Optional field to handle older optimized backups that don't have this field.
+	OptimizedHeader  *bool    `json:"optimized_header,omitempty" yaml:"optimized_header,omitempty"` // Optional field to handle older optimized backups that don't have this field.
+	Type             Type     `json:"type,omitempty" yaml:"type,omitempty"`                         // Type of backup.
 }
 
 // GetInfo extracts backup information from a given ReadSeeker.

From f0b89937bc55116c91c03e8200d895b25aecfa24 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:30:12 +0100
Subject: [PATCH 03/14] lxd/backup/backup/info: Updates GetInfo to use
 TarReader

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/backup/backup_info.go | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/lxd/backup/backup_info.go b/lxd/backup/backup_info.go
index 8e2c2a7da2..96bf73ccfd 100644
--- a/lxd/backup/backup_info.go
+++ b/lxd/backup/backup_info.go
@@ -58,18 +58,8 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
 	optimizedStorageFalse := false
 	optimizedHeaderFalse := false
 
-	// Extract
-	r.Seek(0, 0)
-	_, _, unpacker, err := shared.DetectCompressionFile(r)
-	if err != nil {
-		return nil, err
-	}
-
-	if unpacker == nil {
-		return nil, fmt.Errorf("Unsupported backup compression")
-	}
-
-	tr, cancelFunc, err := shared.CompressedTarReader(context.Background(), r, unpacker)
+	// Extract.
+	tr, cancelFunc, err := TarReader(r)
 	if err != nil {
 		return nil, err
 	}
@@ -105,7 +95,6 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
 			if result.OptimizedStorage != nil {
 				// No need to continue looking for optimized storage hint using the presence of the
 				// container.bin file below, as the index.yaml file tells us directly.
-				cancelFunc()
 				break
 			} else {
 				// Default to non-optimized if not specified and continue reading to see if
@@ -121,12 +110,13 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
 
 			// Stop read loop if index.yaml already parsed.
 			if hasIndexFile {
-				cancelFunc()
 				break
 			}
 		}
 	}
 
+	cancelFunc() // Done reading archive.
+
 	if !hasIndexFile {
 		return nil, fmt.Errorf("Backup is missing index.yaml")
 	}

From 701a4a01e0c0413c059c694ba8519f20cd34d1e3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:26:13 +0100
Subject: [PATCH 04/14] lxc/backup: Updates backupWriteIndex to use backup.Type

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/backup.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lxd/backup.go b/lxd/backup.go
index c7c352e7c2..5941c3ceef 100644
--- a/lxd/backup.go
+++ b/lxd/backup.go
@@ -187,12 +187,17 @@ func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, opti
 		poolDriverOptimizedHeader = pool.Driver().Info().OptimizedBackupHeader
 	}
 
+	backupType := backup.InstanceTypeToBackupType(api.InstanceType(sourceInst.Type().String()))
+	if backupType == backup.TypeUnknown {
+		return fmt.Errorf("Unrecognised instance type for backup type conversion")
+	}
+
 	indexInfo := backup.Info{
 		Name:             sourceInst.Name(),
 		Pool:             pool.Name(),
 		Snapshots:        []string{},
 		Backend:          pool.Driver().Info().Name,
-		Type:             api.InstanceType(sourceInst.Type().String()),
+		Type:             backupType,
 		OptimizedStorage: &optimized,
 		OptimizedHeader:  &poolDriverOptimizedHeader,
 	}

From f23dd66bbf651e45702584f6854be974bd305e7c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:30:29 +0100
Subject: [PATCH 05/14] lxd/backup/backup/info: GetInfo consistent comment
 endings

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/backup/backup_info.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/backup/backup_info.go b/lxd/backup/backup_info.go
index 96bf73ccfd..4e999bafbf 100644
--- a/lxd/backup/backup_info.go
+++ b/lxd/backup/backup_info.go
@@ -68,7 +68,7 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
-			break // End of archive
+			break // End of archive.
 		}
 		if err != nil {
 			return nil, errors.Wrapf(err, "Error reading backup file info")

From ae8d5bea42bd36d8e707004ef0b2b01f9265dd2e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:30:48 +0100
Subject: [PATCH 06/14] lxd/backup/backup/info: Updates GetInfo to support
 backup.Type

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/backup/backup_info.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lxd/backup/backup_info.go b/lxd/backup/backup_info.go
index 4e999bafbf..b219e1a7ec 100644
--- a/lxd/backup/backup_info.go
+++ b/lxd/backup/backup_info.go
@@ -83,8 +83,8 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
 			hasIndexFile = true
 
 			// Default to container if index doesn't specify instance type.
-			if result.Type == api.InstanceTypeAny {
-				result.Type = api.InstanceTypeContainer
+			if result.Type == TypeUnknown {
+				result.Type = TypeContainer
 			}
 
 			// Default to no optimized header if not specified.

From 3d12a11860378ba00a4dab8b2b4e504926dcb391 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:31:32 +0100
Subject: [PATCH 07/14] lxd/db/backups: InstanceBackup comment

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/db/backups.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/db/backups.go b/lxd/db/backups.go
index d22892d4df..80c6aad6f4 100644
--- a/lxd/db/backups.go
+++ b/lxd/db/backups.go
@@ -12,7 +12,7 @@ import (
 	log "github.com/lxc/lxd/shared/log15"
 )
 
-// InstanceBackup is a value object holding all db-related details about a backup.
+// InstanceBackup is a value object holding all db-related details about an instance backup.
 type InstanceBackup struct {
 	ID                   int
 	InstanceID           int

From 7a3023461ef064e422000f80204ab6b5269922dc Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:32:16 +0100
Subject: [PATCH 08/14] lxd/db/backups: projectName argument renaming

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/db/backups.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lxd/db/backups.go b/lxd/db/backups.go
index 80c6aad6f4..ff4dcc27e4 100644
--- a/lxd/db/backups.go
+++ b/lxd/db/backups.go
@@ -39,7 +39,7 @@ func (c *Cluster) getInstanceBackupID(name string) (int, error) {
 }
 
 // GetInstanceBackup returns the backup with the given name.
-func (c *Cluster) GetInstanceBackup(project, name string) (InstanceBackup, error) {
+func (c *Cluster) GetInstanceBackup(projectName string, name string) (InstanceBackup, error) {
 	args := InstanceBackup{}
 	args.Name = name
 
@@ -54,7 +54,7 @@ SELECT instances_backups.id, instances_backups.instance_id,
     JOIN projects ON projects.id=instances.project_id
     WHERE projects.name=? AND instances_backups.name=?
 `
-	arg1 := []interface{}{project, name}
+	arg1 := []interface{}{projectName, name}
 	arg2 := []interface{}{&args.ID, &args.InstanceID, &args.CreationDate,
 		&args.ExpiryDate, &instanceOnlyInt, &optimizedStorageInt}
 	err := dbQueryRowScan(c, q, arg1, arg2)
@@ -79,14 +79,14 @@ SELECT instances_backups.id, instances_backups.instance_id,
 
 // GetInstanceBackups returns the names of all backups of the instance with the
 // given name.
-func (c *Cluster) GetInstanceBackups(project, name string) ([]string, error) {
+func (c *Cluster) GetInstanceBackups(projectName string, name string) ([]string, error) {
 	var result []string
 
 	q := `SELECT instances_backups.name FROM instances_backups
 JOIN instances ON instances_backups.instance_id=instances.id
 JOIN projects ON projects.id=instances.project_id
 WHERE projects.name=? AND instances.name=?`
-	inargs := []interface{}{project, name}
+	inargs := []interface{}{projectName, name}
 	outfmt := []interface{}{name}
 	dbResults, err := queryScan(c, q, inargs, outfmt)
 	if err != nil {

From 6b578edec9d947ca1b7a08644fd579b3e2471e7f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:35:30 +0100
Subject: [PATCH 09/14] lxd/db/storage/volumes: Set Snapshot: true in
 StorageVolumeArgs returned from GetLocalStoragePoolVolumeSnapshotsWithType

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/db/storage_volumes.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lxd/db/storage_volumes.go b/lxd/db/storage_volumes.go
index f89d6ab1db..d6680de8a3 100644
--- a/lxd/db/storage_volumes.go
+++ b/lxd/db/storage_volumes.go
@@ -257,6 +257,7 @@ SELECT storage_volumes_snapshots.name, storage_volumes_snapshots.description FRO
 		row := StorageVolumeArgs{
 			Name:        volumeName + shared.SnapshotDelimiter + r[0].(string),
 			Description: r[1].(string),
+			Snapshot:    true,
 		}
 		result = append(result, row)
 	}

From fbea25160495a65b96522b0b5d1515d5cd18d53b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:36:06 +0100
Subject: [PATCH 10/14] lxd/instance: Spacing

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/instance.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lxd/instance.go b/lxd/instance.go
index bdc1db518c..2d812f6b97 100644
--- a/lxd/instance.go
+++ b/lxd/instance.go
@@ -119,6 +119,7 @@ func instanceCreateFromImage(d *Daemon, args db.InstanceArgs, hash string, op *o
 	if err != nil {
 		return nil, errors.Wrapf(err, "Locate image %q in the cluster", hash)
 	}
+
 	if nodeAddress != "" {
 		// The image is available from another node, let's try to import it.
 		err = instanceImageTransfer(d, args.Project, img.Fingerprint, nodeAddress)

From 9887fcbaa05670bf20d90a44aa7a16c64cdb1a33 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:38:06 +0100
Subject: [PATCH 11/14] lxd/storage/drivers/driver/btrfs/utils: Switches to
 backup.TarReader

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/storage/drivers/driver_btrfs_utils.go | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index 0f66774042..fb5d2d0a5b 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -1,7 +1,6 @@
 package drivers
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -16,6 +15,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gopkg.in/yaml.v2"
 
+	"github.com/lxc/lxd/lxd/backup"
 	"github.com/lxc/lxd/shared"
 	"github.com/lxc/lxd/shared/ioprogress"
 	"github.com/lxc/lxd/shared/logger"
@@ -408,18 +408,8 @@ func (d *btrfs) restorationHeader(vol Volume, snapshots []string) (*BTRFSMetaDat
 func (d *btrfs) loadOptimizedBackupHeader(r io.ReadSeeker) (*BTRFSMetaDataHeader, error) {
 	header := BTRFSMetaDataHeader{}
 
-	// Extract
-	r.Seek(0, 0)
-	_, _, unpacker, err := shared.DetectCompressionFile(r)
-	if err != nil {
-		return nil, err
-	}
-
-	if unpacker == nil {
-		return nil, fmt.Errorf("Unsupported backup compression")
-	}
-
-	tr, cancelFunc, err := shared.CompressedTarReader(context.Background(), r, unpacker)
+	// Extract.
+	tr, cancelFunc, err := backup.TarReader(r)
 	if err != nil {
 		return nil, err
 	}

From cbf741416f869d7470320ce11c3212f8be92f4a4 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:38:26 +0100
Subject: [PATCH 12/14] lxd/storage/drivers/driver/btrfs: Consistent comment
 ending

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/storage/drivers/driver_btrfs_utils.go   | 2 +-
 lxd/storage/drivers/driver_btrfs_volumes.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index fb5d2d0a5b..e1a359e849 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -418,7 +418,7 @@ func (d *btrfs) loadOptimizedBackupHeader(r io.ReadSeeker) (*BTRFSMetaDataHeader
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
-			break // End of archive
+			break // End of archive.
 		}
 		if err != nil {
 			return nil, errors.Wrapf(err, "Error reading backup file for optimized backup header file")
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index ebe7dc5334..338845062b 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -195,7 +195,7 @@ func (d *btrfs) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcDat
 		for {
 			hdr, err := tr.Next()
 			if err == io.EOF {
-				break // End of archive
+				break // End of archive.
 			}
 			if err != nil {
 				return "", err

From e27030f9c8a197238c367bea52f9e1a9f6eb69cf Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:40:47 +0100
Subject: [PATCH 13/14] lxd/storage/drivers/driver/zfs/volumes: consistent
 comment ending

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/storage/drivers/driver_zfs_volumes.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/storage/drivers/driver_zfs_volumes.go b/lxd/storage/drivers/driver_zfs_volumes.go
index 5cae1d92d1..a926bb30a9 100644
--- a/lxd/storage/drivers/driver_zfs_volumes.go
+++ b/lxd/storage/drivers/driver_zfs_volumes.go
@@ -299,7 +299,7 @@ func (d *zfs) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData
 		for {
 			hdr, err := tr.Next()
 			if err == io.EOF {
-				break // End of archive
+				break // End of archive.
 			}
 			if err != nil {
 				return err

From 15c7b82b380fa2975e7df685811e566d0f90bac7 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 24 Sep 2020 17:43:46 +0100
Subject: [PATCH 14/14] lxd/storage/drivers/generic/vfs: Consistent comment
 ending

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/storage/drivers/generic_vfs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/storage/drivers/generic_vfs.go b/lxd/storage/drivers/generic_vfs.go
index 8c2c7e12c9..457b872ae1 100644
--- a/lxd/storage/drivers/generic_vfs.go
+++ b/lxd/storage/drivers/generic_vfs.go
@@ -640,7 +640,7 @@ func genericVFSBackupUnpack(d Driver, vol Volume, snapshots []string, srcData io
 			for {
 				hdr, err := tr.Next()
 				if err == io.EOF {
-					break // End of archive
+					break // End of archive.
 				}
 				if err != nil {
 					return err


More information about the lxc-devel mailing list