[lxc-devel] [lxd/master] Storage: BTRFS subvolume migration support
tomponline on Github
lxc-bot at linuxcontainers.org
Wed May 6 16:28:38 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 374 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200506/32512696/attachment-0001.bin>
-------------- next part --------------
From 62ae28941db7a0674692581b420f58fc5580df3d Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 4 May 2020 10:23:08 +0100
Subject: [PATCH 01/15] lxd/storage/driver/driver/btrfs/utils: Updates
snapshotSubvolume to handle sub volumes
Marks snapshotted sub volumes and the parent volume as readonly when in recursive mode and readonly argument passed.
Returns list of snapshot volumes.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_utils.go | 42 +++++++++++++++--------
1 file changed, 28 insertions(+), 14 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index 5b30f043c4..126832c026 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -86,10 +86,12 @@ func (d *btrfs) getSubvolumes(path string) ([]string, error) {
return result, nil
}
+// snapshotSubvolume creates a snapshot of the specified path at the dest supplied. If recursion is true and
+// sub volumes are found below the path then they are created at the relative location in dest.
func (d *btrfs) snapshotSubvolume(path string, dest string, readonly bool, recursion bool) error {
// Single subvolume deletion.
- snapshot := func(path string, dest string) error {
- if readonly && !d.state.OS.RunningInUserNS {
+ snapshot := func(path string, dest string, ro bool) error {
+ if ro && !d.state.OS.RunningInUserNS {
_, err := shared.RunCommand("btrfs", "subvolume", "snapshot", "-r", path, dest)
if err != nil {
return err
@@ -109,28 +111,40 @@ func (d *btrfs) snapshotSubvolume(path string, dest string, readonly bool, recur
// Now snapshot all subvolumes of the root.
if recursion {
// Get the subvolumes list.
- subsubvols, err := d.getSubvolumes(path)
+ subSubVols, err := d.getSubvolumes(path)
if err != nil {
return err
}
- sort.Sort(sort.StringSlice(subsubvols))
-
- if len(subsubvols) > 0 && readonly {
- // Creating subvolumes requires the parent to be writable.
- readonly = false
+ sort.Sort(sort.StringSlice(subSubVols))
+
+ if len(subSubVols) > 0 && readonly {
+ // First snapshot the root volume. Creating subvolumes requires the parent to be writable.
+ err = snapshot(path, dest, false)
+ } else {
+ // First snapshot the root volume.
+ err = snapshot(path, dest, readonly)
}
- // First snapshot the root.
- err = snapshot(path, dest)
+ // Check errors from snapshot.
if err != nil {
return err
}
- for _, subsubvol := range subsubvols {
+ for _, subSubVol := range subSubVols {
+ subSubVolSnapPath := filepath.Join(dest, subSubVol)
+
// Clear the target for the subvol to use.
- os.Remove(filepath.Join(dest, subsubvol))
+ os.Remove(subSubVolSnapPath)
+
+ err := snapshot(filepath.Join(path, subSubVol), subSubVolSnapPath, readonly)
+ if err != nil {
+ return err
+ }
+ }
- err := snapshot(filepath.Join(path, subsubvol), filepath.Join(dest, subsubvol))
+ // Now set the root snapshot we created earlier as writable read only.
+ if len(subSubVols) > 0 && readonly {
+ _, err = shared.RunCommand("btrfs", "property", "set", "-ts", dest, "ro", "true")
if err != nil {
return err
}
@@ -140,7 +154,7 @@ func (d *btrfs) snapshotSubvolume(path string, dest string, readonly bool, recur
}
// Handle standalone volume.
- err := snapshot(path, dest)
+ err := snapshot(path, dest, readonly)
if err != nil {
return err
}
From e36d81e9949f31d9817a58182c3a623cb07be4e3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 4 May 2020 10:27:01 +0100
Subject: [PATCH 02/15] lxd/storag/drivers/driver/btrfs/volumes: Updates
MigrateVolume to send subvolumes
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_volumes.go | 68 +++++++++++++++++----
1 file changed, 56 insertions(+), 12 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index 9048a01f4c..4d72b61b4e 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -2,6 +2,7 @@ package drivers
import (
"context"
+ "encoding/json"
"fmt"
"io"
"io/ioutil"
@@ -606,6 +607,40 @@ func (d *btrfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *m
return nil
}
+ // Generate migration header, containing subvolume info.
+ migrationHeader, err := d.migrationHeader(vol, volSrcArgs.Snapshots)
+ if err != nil {
+ return err
+ }
+
+ // If we haven't negotiated subvolume support, check if we have any subvolumes in source and fail,
+ // otherwise we would end up not materialising all of the source's files on the target.
+ if !shared.StringInSlice(migration.BTRFSFeatureMigrationHeader, volSrcArgs.MigrationType.Features) || !shared.StringInSlice(migration.BTRFSFeatureSubvolumes, volSrcArgs.MigrationType.Features) {
+ for _, subVol := range migrationHeader.Subvolumes {
+ if subVol.Path != string(filepath.Separator) {
+ return fmt.Errorf("Subvolumes detected in source but target does not support receiving subvolumes")
+ }
+ }
+ }
+
+ // Send metadata migration header frame with subvolume info if we have negotiated that feature.
+ if shared.StringInSlice(migration.BTRFSFeatureMigrationHeader, volSrcArgs.MigrationType.Features) {
+ headerJSON, err := json.Marshal(migrationHeader)
+ if err != nil {
+ return err
+ }
+
+ _, err = conn.Write(headerJSON)
+ if err != nil {
+ return err
+ }
+
+ err = conn.Close() //End the frame.
+ if err != nil {
+ return err
+ }
+ }
+
// Transfer the snapshots first.
for i, snapName := range volSrcArgs.Snapshots {
snapshot, _ := vol.NewSnapshot(snapName)
@@ -645,12 +680,12 @@ func (d *btrfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *m
}
// Make read-only snapshot of the subvolume as writable subvolumes cannot be sent.
- migrationSendSnapshot := filepath.Join(tmpVolumesMountPoint, ".migration-send")
- err = d.snapshotSubvolume(vol.MountPath(), migrationSendSnapshot, true, false)
+ migrationSendSnapshotPrefix := filepath.Join(tmpVolumesMountPoint, ".migration-send")
+ err = d.snapshotSubvolume(vol.MountPath(), migrationSendSnapshotPrefix, true, true)
if err != nil {
return err
}
- defer d.deleteSubvolume(migrationSendSnapshot, true)
+ defer d.deleteSubvolume(migrationSendSnapshotPrefix, true)
// Setup progress tracking.
var wrapper *ioprogress.ProgressTracker
@@ -658,16 +693,25 @@ func (d *btrfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *m
wrapper = migration.ProgressTracker(op, "fs_progress", vol.name)
}
- // Compare to latest snapshot.
- btrfsParent := ""
- if len(volSrcArgs.Snapshots) > 0 {
- btrfsParent = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSrcArgs.Snapshots[len(volSrcArgs.Snapshots)-1]))
- }
+ // Send main volume (and any subvolumes) to target.
+ for _, subVolume := range migrationHeader.Subvolumes {
+ if subVolume.Snapshot != "" {
+ continue // Only sending main volume now (not snapshots).
+ }
- // Send the volume itself.
- err = d.sendSubvolume(migrationSendSnapshot, btrfsParent, conn, wrapper)
- if err != nil {
- return err
+ parent := ""
+
+ if subVolume.Path == string(filepath.Separator) && len(volSrcArgs.Snapshots) > 0 {
+ // Compare to latest snapshot.
+ parent = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSrcArgs.Snapshots[len(volSrcArgs.Snapshots)-1]))
+ }
+
+ source := filepath.Join(migrationSendSnapshotPrefix, subVolume.Path)
+ d.logger.Debug("Sending subvolume snapshot", log.Ctx{"name": vol.name, "source": source, "parent": parent, "path": subVolume.Path})
+ err = d.sendSubvolume(source, parent, conn, wrapper)
+ if err != nil {
+ return err
+ }
}
return nil
From cd4c96d970d61a42f73e28fef1028beb27d13b64 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 4 May 2020 12:21:13 +0100
Subject: [PATCH 03/15] lxd/storage/drivers/driver/btrfs/volumes: Fail backup
when cleanup fails in BackupVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_volumes.go | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index 4d72b61b4e..34beb638e4 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -874,6 +874,12 @@ func (d *btrfs) BackupVolume(vol Volume, tarWriter *instancewriter.InstanceTarWr
return err
}
+ // Ensure snapshot sub volumes are removed.
+ err = d.deleteSubvolume(targetVolume, true)
+ if err != nil {
+ return err
+ }
+
return nil
}
From 43f2ab1e5a6b0b8bbe4d9f746a934d2b1e319ba5 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 4 May 2020 12:24:07 +0100
Subject: [PATCH 04/15] lxd/storage/drivers/driver/btrfs/volumes: Better naming
of variables in unpackVolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_volumes.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index 34beb638e4..0d5da1c297 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -135,8 +135,8 @@ func (d *btrfs) CreateVolumeFromBackup(vol Volume, snapshots []string, srcData i
revert.Add(revertHook)
// Define function to unpack a volume from a backup tarball file.
- unpackVolume := func(r io.ReadSeeker, unpacker []string, srcFile string, mountPath string) error {
- d.Logger().Debug("Unpacking optimized volume", log.Ctx{"source": srcFile, "target": mountPath})
+ unpackVolume := func(r io.ReadSeeker, unpacker []string, srcFile string, targetPath string) error {
+ d.Logger().Debug("Unpacking optimized volume", log.Ctx{"source": srcFile, "target": targetPath})
tr, cancelFunc, err := shared.CompressedTarReader(context.Background(), r, unpacker)
if err != nil {
return err
@@ -154,7 +154,7 @@ func (d *btrfs) CreateVolumeFromBackup(vol Volume, snapshots []string, srcData i
if hdr.Name == srcFile {
// Extract the backup.
- err = shared.RunCommandWithFds(tr, nil, "btrfs", "receive", "-e", mountPath)
+ err = shared.RunCommandWithFds(tr, nil, "btrfs", "receive", "-e", targetPath)
if err != nil {
return err
}
From 550ac94a193a5eaa5dc16b1343de238c3cadfcb0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 4 May 2020 12:24:34 +0100
Subject: [PATCH 05/15] lxd/storage/driversr/driver/btrfs/utils: Allow ro
subvolumes to be deleted in deleteSubvolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_utils.go | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index 126832c026..bd48d3fc5a 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -193,6 +193,11 @@ func (d *btrfs) deleteSubvolume(path string, recursion bool) error {
}
sort.Sort(sort.Reverse(sort.StringSlice(subsubvols)))
+ if len(subsubvols) > 0 {
+ // Attempt to make the root subvolume writable so any subvolumes can be removed.
+ shared.RunCommand("btrfs", "property", "set", path, "ro", "false")
+ }
+
for _, subsubvol := range subsubvols {
err := destroy(filepath.Join(path, subsubvol))
if err != nil {
From a315873333767f69c70a394dc8ecc1e50993b69c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 11:44:02 +0100
Subject: [PATCH 06/15] lxd/migration/migrate/proto: Adds BTRFS Features to
offer header
Adds following features:
migration_header - indicates a migration header will be sent/recv in data channel first.
header_subvolumes - indicates migration can send/recv subvolumes.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/migration/migrate.pb.go | 209 +++++++++++++++++++++++-------------
lxd/migration/migrate.proto | 6 ++
2 files changed, 140 insertions(+), 75 deletions(-)
diff --git a/lxd/migration/migrate.pb.go b/lxd/migration/migrate.pb.go
index cf1b5fbe4c..0e824a7be0 100644
--- a/lxd/migration/migrate.pb.go
+++ b/lxd/migration/migrate.pb.go
@@ -482,6 +482,53 @@ func (m *ZfsFeatures) GetCompress() bool {
return false
}
+type BtrfsFeatures struct {
+ MigrationHeader *bool `protobuf:"varint,1,opt,name=migration_header,json=migrationHeader" json:"migration_header,omitempty"`
+ HeaderSubvolumes *bool `protobuf:"varint,2,opt,name=header_subvolumes,json=headerSubvolumes" json:"header_subvolumes,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BtrfsFeatures) Reset() { *m = BtrfsFeatures{} }
+func (m *BtrfsFeatures) String() string { return proto.CompactTextString(m) }
+func (*BtrfsFeatures) ProtoMessage() {}
+func (*BtrfsFeatures) Descriptor() ([]byte, []int) {
+ return fileDescriptor_fe8772548dc4b615, []int{6}
+}
+
+func (m *BtrfsFeatures) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BtrfsFeatures.Unmarshal(m, b)
+}
+func (m *BtrfsFeatures) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BtrfsFeatures.Marshal(b, m, deterministic)
+}
+func (m *BtrfsFeatures) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BtrfsFeatures.Merge(m, src)
+}
+func (m *BtrfsFeatures) XXX_Size() int {
+ return xxx_messageInfo_BtrfsFeatures.Size(m)
+}
+func (m *BtrfsFeatures) XXX_DiscardUnknown() {
+ xxx_messageInfo_BtrfsFeatures.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BtrfsFeatures proto.InternalMessageInfo
+
+func (m *BtrfsFeatures) GetMigrationHeader() bool {
+ if m != nil && m.MigrationHeader != nil {
+ return *m.MigrationHeader
+ }
+ return false
+}
+
+func (m *BtrfsFeatures) GetHeaderSubvolumes() bool {
+ if m != nil && m.HeaderSubvolumes != nil {
+ return *m.HeaderSubvolumes
+ }
+ return false
+}
+
type MigrationHeader struct {
Fs *MigrationFSType `protobuf:"varint,1,req,name=fs,enum=migration.MigrationFSType" json:"fs,omitempty"`
Criu *CRIUType `protobuf:"varint,2,opt,name=criu,enum=migration.CRIUType" json:"criu,omitempty"`
@@ -493,6 +540,7 @@ type MigrationHeader struct {
Refresh *bool `protobuf:"varint,9,opt,name=refresh" json:"refresh,omitempty"`
ZfsFeatures *ZfsFeatures `protobuf:"bytes,10,opt,name=zfsFeatures" json:"zfsFeatures,omitempty"`
VolumeSize *int64 `protobuf:"varint,11,opt,name=volumeSize" json:"volumeSize,omitempty"`
+ BtrfsFeatures *BtrfsFeatures `protobuf:"bytes,12,opt,name=btrfsFeatures" json:"btrfsFeatures,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@@ -502,7 +550,7 @@ func (m *MigrationHeader) Reset() { *m = MigrationHeader{} }
func (m *MigrationHeader) String() string { return proto.CompactTextString(m) }
func (*MigrationHeader) ProtoMessage() {}
func (*MigrationHeader) Descriptor() ([]byte, []int) {
- return fileDescriptor_fe8772548dc4b615, []int{6}
+ return fileDescriptor_fe8772548dc4b615, []int{7}
}
func (m *MigrationHeader) XXX_Unmarshal(b []byte) error {
@@ -593,6 +641,13 @@ func (m *MigrationHeader) GetVolumeSize() int64 {
return 0
}
+func (m *MigrationHeader) GetBtrfsFeatures() *BtrfsFeatures {
+ if m != nil {
+ return m.BtrfsFeatures
+ }
+ return nil
+}
+
type MigrationControl struct {
Success *bool `protobuf:"varint,1,req,name=success" json:"success,omitempty"`
// optional failure message if sending a failure
@@ -606,7 +661,7 @@ func (m *MigrationControl) Reset() { *m = MigrationControl{} }
func (m *MigrationControl) String() string { return proto.CompactTextString(m) }
func (*MigrationControl) ProtoMessage() {}
func (*MigrationControl) Descriptor() ([]byte, []int) {
- return fileDescriptor_fe8772548dc4b615, []int{7}
+ return fileDescriptor_fe8772548dc4b615, []int{8}
}
func (m *MigrationControl) XXX_Unmarshal(b []byte) error {
@@ -652,7 +707,7 @@ func (m *MigrationSync) Reset() { *m = MigrationSync{} }
func (m *MigrationSync) String() string { return proto.CompactTextString(m) }
func (*MigrationSync) ProtoMessage() {}
func (*MigrationSync) Descriptor() ([]byte, []int) {
- return fileDescriptor_fe8772548dc4b615, []int{8}
+ return fileDescriptor_fe8772548dc4b615, []int{9}
}
func (m *MigrationSync) XXX_Unmarshal(b []byte) error {
@@ -702,7 +757,7 @@ func (m *DumpStatsEntry) Reset() { *m = DumpStatsEntry{} }
func (m *DumpStatsEntry) String() string { return proto.CompactTextString(m) }
func (*DumpStatsEntry) ProtoMessage() {}
func (*DumpStatsEntry) Descriptor() ([]byte, []int) {
- return fileDescriptor_fe8772548dc4b615, []int{9}
+ return fileDescriptor_fe8772548dc4b615, []int{10}
}
func (m *DumpStatsEntry) XXX_Unmarshal(b []byte) error {
@@ -815,7 +870,7 @@ func (m *RestoreStatsEntry) Reset() { *m = RestoreStatsEntry{} }
func (m *RestoreStatsEntry) String() string { return proto.CompactTextString(m) }
func (*RestoreStatsEntry) ProtoMessage() {}
func (*RestoreStatsEntry) Descriptor() ([]byte, []int) {
- return fileDescriptor_fe8772548dc4b615, []int{10}
+ return fileDescriptor_fe8772548dc4b615, []int{11}
}
func (m *RestoreStatsEntry) XXX_Unmarshal(b []byte) error {
@@ -883,7 +938,7 @@ func (m *StatsEntry) Reset() { *m = StatsEntry{} }
func (m *StatsEntry) String() string { return proto.CompactTextString(m) }
func (*StatsEntry) ProtoMessage() {}
func (*StatsEntry) Descriptor() ([]byte, []int) {
- return fileDescriptor_fe8772548dc4b615, []int{11}
+ return fileDescriptor_fe8772548dc4b615, []int{12}
}
func (m *StatsEntry) XXX_Unmarshal(b []byte) error {
@@ -927,6 +982,7 @@ func init() {
proto.RegisterType((*Snapshot)(nil), "migration.Snapshot")
proto.RegisterType((*RsyncFeatures)(nil), "migration.rsyncFeatures")
proto.RegisterType((*ZfsFeatures)(nil), "migration.zfsFeatures")
+ proto.RegisterType((*BtrfsFeatures)(nil), "migration.btrfsFeatures")
proto.RegisterType((*MigrationHeader)(nil), "migration.MigrationHeader")
proto.RegisterType((*MigrationControl)(nil), "migration.MigrationControl")
proto.RegisterType((*MigrationSync)(nil), "migration.MigrationSync")
@@ -938,73 +994,76 @@ func init() {
func init() { proto.RegisterFile("lxd/migration/migrate.proto", fileDescriptor_fe8772548dc4b615) }
var fileDescriptor_fe8772548dc4b615 = []byte{
- // 1080 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xcd, 0x6e, 0xdb, 0x46,
- 0x17, 0xfd, 0x24, 0x51, 0xb6, 0x78, 0x25, 0x39, 0xca, 0xc4, 0x08, 0x88, 0xe4, 0x6b, 0xaa, 0x32,
- 0x29, 0xaa, 0x78, 0x61, 0xa7, 0x0a, 0x0a, 0x64, 0x55, 0x20, 0x96, 0xea, 0x26, 0xa8, 0xa3, 0x18,
- 0x23, 0x1b, 0x45, 0xbb, 0x21, 0x26, 0xe4, 0xa5, 0x3c, 0x30, 0xff, 0x30, 0x43, 0xd9, 0x96, 0x36,
- 0x45, 0x1f, 0xa6, 0x0f, 0xd1, 0xa7, 0xe8, 0xaa, 0xef, 0x53, 0xcc, 0x0c, 0x49, 0x93, 0x4e, 0x81,
- 0xee, 0xe6, 0x9e, 0x7b, 0x78, 0xee, 0xcc, 0xfd, 0x23, 0x3c, 0x8d, 0x6e, 0x83, 0xa3, 0x98, 0xaf,
- 0x04, 0xcb, 0x79, 0x9a, 0x14, 0x27, 0x3c, 0xcc, 0x44, 0x9a, 0xa7, 0xc4, 0xae, 0x1c, 0xee, 0x6f,
- 0x60, 0xbf, 0x9f, 0x7f, 0x60, 0xd9, 0xf9, 0x26, 0x43, 0xb2, 0x0f, 0x5d, 0x2e, 0xd7, 0x3c, 0x70,
- 0x5a, 0xe3, 0xf6, 0xa4, 0x47, 0x8d, 0x61, 0xd0, 0x15, 0x0f, 0x9c, 0x76, 0x89, 0xae, 0x78, 0x40,
- 0x1e, 0xc3, 0xce, 0x65, 0x2a, 0x73, 0x1e, 0x38, 0x9d, 0x71, 0x7b, 0xd2, 0xa5, 0x85, 0x45, 0x08,
- 0x58, 0x89, 0xe4, 0x81, 0x63, 0x69, 0x54, 0x9f, 0xc9, 0x13, 0xe8, 0xc5, 0x2c, 0x13, 0x2c, 0x59,
- 0xa1, 0xd3, 0xd5, 0x78, 0x65, 0xbb, 0xaf, 0x60, 0x67, 0x96, 0x26, 0x21, 0x5f, 0x91, 0x11, 0x74,
- 0xae, 0x70, 0xa3, 0x63, 0xdb, 0x54, 0x1d, 0x55, 0xe4, 0x6b, 0x16, 0xad, 0x51, 0x47, 0xb6, 0xa9,
- 0x31, 0xdc, 0x1f, 0x61, 0x67, 0x8e, 0xd7, 0xdc, 0x47, 0x1d, 0x8b, 0xc5, 0x58, 0x7c, 0xa2, 0xcf,
- 0xe4, 0x25, 0xec, 0xf8, 0x5a, 0xcf, 0x69, 0x8f, 0x3b, 0x93, 0xfe, 0xf4, 0xe1, 0x61, 0xf5, 0xd8,
- 0x43, 0x13, 0x88, 0x16, 0x04, 0xf7, 0xaf, 0x36, 0xf4, 0x96, 0x09, 0xcb, 0xe4, 0x65, 0x9a, 0xff,
- 0xab, 0xd6, 0x6b, 0xe8, 0x47, 0xa9, 0xcf, 0xa2, 0xd9, 0x7f, 0x08, 0xd6, 0x59, 0xea, 0xb1, 0x99,
- 0x48, 0x43, 0x1e, 0xa1, 0x74, 0x3a, 0xe3, 0xce, 0xc4, 0xa6, 0x95, 0x4d, 0xfe, 0x0f, 0x36, 0x66,
- 0x97, 0x18, 0xa3, 0x60, 0x91, 0xce, 0x50, 0x8f, 0xde, 0x01, 0xe4, 0x3b, 0x18, 0x68, 0x21, 0xf3,
- 0x3a, 0xe9, 0x74, 0x3f, 0x8b, 0x67, 0x3c, 0xb4, 0x41, 0x23, 0x2e, 0x0c, 0x98, 0xf0, 0x2f, 0x79,
- 0x8e, 0x7e, 0xbe, 0x16, 0xe8, 0xec, 0xe8, 0x0c, 0x37, 0x30, 0x75, 0x29, 0x99, 0xb3, 0x1c, 0xc3,
- 0x75, 0xe4, 0xec, 0xea, 0xb8, 0x95, 0x4d, 0x9e, 0xc3, 0xd0, 0x17, 0xa8, 0x03, 0x78, 0x01, 0xcb,
- 0xd1, 0xe9, 0x8d, 0x5b, 0x93, 0x0e, 0x1d, 0x94, 0xe0, 0x9c, 0xe5, 0x48, 0x5e, 0xc0, 0x5e, 0xc4,
- 0x64, 0xee, 0xad, 0x25, 0x06, 0x86, 0x65, 0x1b, 0x96, 0x42, 0x2f, 0x24, 0x06, 0x8a, 0xe5, 0xfe,
- 0xde, 0x82, 0xa1, 0x90, 0x9b, 0xc4, 0x3f, 0x41, 0xa6, 0xe2, 0x4a, 0xd5, 0x26, 0xb7, 0x2c, 0xcf,
- 0x85, 0x74, 0x5a, 0xe3, 0xd6, 0xa4, 0x47, 0x0b, 0x4b, 0xe1, 0x01, 0x46, 0x98, 0xab, 0xda, 0x6a,
- 0xdc, 0x58, 0xea, 0xa2, 0x7e, 0x1a, 0x67, 0x02, 0xa5, 0xca, 0x9e, 0xf2, 0x54, 0x36, 0x79, 0x01,
- 0xc3, 0x4f, 0x3c, 0xe0, 0x02, 0x7d, 0x75, 0x2d, 0x9d, 0x41, 0x45, 0x68, 0x82, 0xee, 0x4b, 0xe8,
- 0x6f, 0x43, 0x59, 0x5d, 0xa0, 0x2e, 0xd8, 0x6a, 0x0a, 0xba, 0x7f, 0x76, 0xe0, 0xc1, 0x87, 0x32,
- 0xb9, 0xef, 0x90, 0x05, 0x28, 0xc8, 0x01, 0xb4, 0x43, 0xa9, 0xbb, 0x60, 0x6f, 0xfa, 0xa4, 0x96,
- 0xfa, 0x8a, 0x77, 0xb2, 0x54, 0xb3, 0x42, 0xdb, 0xa1, 0x24, 0xdf, 0x80, 0xe5, 0x0b, 0xbe, 0xd6,
- 0x4f, 0xd8, 0x9b, 0x3e, 0xaa, 0x37, 0x06, 0x7d, 0x7f, 0xa1, 0x69, 0x9a, 0x40, 0x0e, 0xa0, 0xcb,
- 0x83, 0x98, 0x65, 0xba, 0x21, 0xfa, 0xd3, 0xfd, 0x1a, 0xb3, 0x9a, 0x3e, 0x6a, 0x28, 0xea, 0x95,
- 0xb2, 0x68, 0xca, 0x05, 0x8b, 0x51, 0x3a, 0x96, 0x6e, 0xa2, 0x26, 0x48, 0xbe, 0x05, 0xbb, 0x04,
- 0xca, 0x46, 0xa9, 0xc7, 0x2f, 0xdb, 0x9a, 0xde, 0xb1, 0x88, 0x03, 0xbb, 0x99, 0xc0, 0x60, 0x1d,
- 0x67, 0xce, 0xae, 0x4e, 0x44, 0x69, 0x92, 0xef, 0xef, 0x55, 0x4d, 0x77, 0x40, 0x7f, 0xea, 0xd4,
- 0x04, 0x1b, 0x7e, 0x7a, 0xaf, 0xc8, 0x0e, 0xec, 0x0a, 0x0c, 0x05, 0xca, 0x4b, 0xdd, 0x15, 0x3d,
- 0x5a, 0x9a, 0xe4, 0x4d, 0xa3, 0x18, 0x0e, 0x68, 0xdd, 0xc7, 0x35, 0xdd, 0x9a, 0x97, 0x36, 0xea,
- 0xf6, 0x0c, 0xe0, 0x3a, 0x8d, 0xd6, 0x31, 0x2e, 0xf9, 0x16, 0x9d, 0xbe, 0x6e, 0xb6, 0x1a, 0xe2,
- 0x9e, 0xc0, 0xa8, 0x2a, 0xc9, 0x2c, 0x4d, 0x72, 0x91, 0x46, 0xea, 0x1e, 0x72, 0xed, 0xfb, 0xa6,
- 0xd4, 0xaa, 0xc9, 0x4b, 0x53, 0x79, 0x62, 0x94, 0x92, 0xad, 0x4c, 0xbf, 0xd9, 0xb4, 0x34, 0xdd,
- 0xd7, 0x30, 0xac, 0x74, 0x96, 0x9b, 0xc4, 0x57, 0xe3, 0x14, 0xf2, 0x84, 0x45, 0x67, 0x02, 0xe7,
- 0x2a, 0x57, 0x46, 0xa9, 0x81, 0xb9, 0x7f, 0x74, 0x60, 0xa4, 0x32, 0xe7, 0xa9, 0x21, 0x92, 0x1e,
- 0x26, 0xb9, 0xd8, 0xa8, 0x39, 0x0a, 0x05, 0xe2, 0x96, 0x27, 0x2b, 0x2f, 0xe7, 0xc5, 0x2a, 0x19,
- 0xd2, 0x41, 0x09, 0x9e, 0xf3, 0x18, 0xc9, 0x97, 0xd0, 0x0f, 0x45, 0xba, 0xc5, 0xc4, 0x50, 0xda,
- 0x9a, 0x02, 0x06, 0xd2, 0x84, 0xaf, 0x60, 0x10, 0x63, 0xac, 0xc5, 0x35, 0xa3, 0xa3, 0x19, 0xfd,
- 0x02, 0xd3, 0x94, 0xe7, 0x30, 0x8c, 0x31, 0xbe, 0x11, 0x3c, 0x47, 0xc3, 0xb1, 0x4c, 0xa0, 0x12,
- 0x2c, 0x49, 0x19, 0x5b, 0xa1, 0xf4, 0xa4, 0xcf, 0x92, 0x04, 0x03, 0xbd, 0x78, 0x2d, 0x3a, 0xd0,
- 0xe0, 0xd2, 0x60, 0xe4, 0x15, 0xec, 0x17, 0xa4, 0x2b, 0x9e, 0x65, 0x18, 0x78, 0x19, 0x13, 0x98,
- 0xe4, 0x7a, 0x85, 0x58, 0x94, 0x18, 0xae, 0x71, 0x9d, 0x69, 0xcf, 0x9d, 0xac, 0x8a, 0x94, 0x63,
- 0xa2, 0xb7, 0x49, 0x29, 0xfb, 0xb3, 0xc1, 0x14, 0x89, 0x8b, 0x98, 0x65, 0x9e, 0x40, 0x99, 0x46,
- 0xd7, 0x66, 0xa3, 0x0c, 0xe9, 0x40, 0x83, 0xd4, 0x60, 0xe4, 0x0b, 0x00, 0xa3, 0x14, 0xb1, 0xed,
- 0xc6, 0xb1, 0xb5, 0x8c, 0xad, 0x91, 0x53, 0xb6, 0xdd, 0x94, 0x6e, 0x2f, 0xe3, 0x59, 0xd1, 0x38,
- 0x85, 0xfb, 0x4c, 0x01, 0x6a, 0x1f, 0x55, 0x6e, 0xef, 0xd3, 0x3a, 0x94, 0xba, 0x45, 0x8a, 0x8b,
- 0x28, 0xca, 0xf1, 0x3a, 0x94, 0xee, 0xdf, 0x2d, 0x78, 0x24, 0x50, 0xe6, 0xa9, 0xc0, 0x46, 0xa9,
- 0xbe, 0x36, 0x5f, 0x4b, 0x4f, 0xad, 0x02, 0x26, 0xd0, 0xfc, 0xf1, 0x2c, 0x6a, 0xde, 0x36, 0x2b,
- 0x40, 0x72, 0x00, 0x0f, 0x9b, 0xe9, 0xf1, 0xd3, 0x1b, 0x5d, 0x32, 0x8b, 0x3e, 0xa8, 0xe7, 0x66,
- 0x96, 0xde, 0xa8, 0xba, 0x85, 0xa9, 0xb8, 0xaa, 0x8a, 0x5f, 0xd4, 0xad, 0xc0, 0xca, 0xd2, 0x96,
- 0x97, 0xa9, 0x95, 0xad, 0x5f, 0x60, 0x9a, 0x52, 0x5d, 0xac, 0x00, 0x55, 0xd9, 0x5a, 0xd5, 0xc5,
- 0x68, 0x01, 0xba, 0xb7, 0xd0, 0xaf, 0x3f, 0xe7, 0x08, 0xac, 0xc0, 0xb4, 0xaa, 0x1a, 0xaf, 0xa7,
- 0xb5, 0xf1, 0xba, 0xdf, 0xa4, 0x54, 0x13, 0xc9, 0x1b, 0x35, 0xb0, 0x5a, 0x4b, 0x8f, 0x43, 0x7f,
- 0xfa, 0xac, 0x3e, 0xea, 0x9f, 0x27, 0x8c, 0x96, 0xf4, 0x83, 0x45, 0x6d, 0x63, 0x9a, 0x4d, 0x48,
- 0x6c, 0xe8, 0xd2, 0xe5, 0x2f, 0x8b, 0xd9, 0xe8, 0x7f, 0xea, 0x78, 0x7c, 0x4e, 0x4f, 0x96, 0xa3,
- 0x16, 0xd9, 0x85, 0xce, 0xaf, 0x27, 0xcb, 0x51, 0x5b, 0x1d, 0xe8, 0xf1, 0x7c, 0xd4, 0x21, 0x8f,
- 0xe0, 0xc1, 0xf1, 0xe9, 0xc7, 0xd9, 0x4f, 0xde, 0xdb, 0xc5, 0xdc, 0x33, 0x5f, 0x58, 0x07, 0x47,
- 0xd0, 0x2b, 0x77, 0x25, 0xd9, 0x03, 0x50, 0x67, 0xaf, 0xa6, 0x76, 0xf6, 0xee, 0xed, 0xc5, 0xe9,
- 0xa8, 0x45, 0x7a, 0x60, 0x2d, 0x3e, 0x2e, 0x7e, 0x18, 0xb5, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff,
- 0x6c, 0x4d, 0xa0, 0xee, 0xd9, 0x08, 0x00, 0x00,
+ // 1128 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xdd, 0x8e, 0xd3, 0x46,
+ 0x14, 0x6e, 0x12, 0x67, 0x37, 0x3e, 0x49, 0x76, 0xc3, 0x80, 0x90, 0x05, 0x2d, 0x4d, 0x0d, 0x55,
+ 0xc3, 0x56, 0x02, 0x1a, 0x54, 0x89, 0xab, 0x4a, 0x6c, 0xd2, 0x2d, 0xa8, 0x10, 0x56, 0x13, 0x50,
+ 0xd5, 0xde, 0x58, 0xb3, 0xf6, 0x71, 0x32, 0xc2, 0x7f, 0x9a, 0xb1, 0x17, 0x92, 0x9b, 0xaa, 0x0f,
+ 0xd3, 0xe7, 0xe9, 0x55, 0x1f, 0xa6, 0x77, 0xd5, 0xcc, 0xd8, 0xc6, 0xde, 0xad, 0xd4, 0xbb, 0x39,
+ 0xdf, 0xf9, 0xfc, 0x9d, 0x33, 0xe7, 0x67, 0x0c, 0x77, 0xa3, 0x8f, 0xc1, 0xe3, 0x98, 0x6f, 0x04,
+ 0xcb, 0x79, 0x9a, 0x94, 0x27, 0x7c, 0x94, 0x89, 0x34, 0x4f, 0x89, 0x5d, 0x3b, 0xdc, 0xdf, 0xc1,
+ 0x7e, 0xb9, 0x7c, 0xcd, 0xb2, 0xb7, 0xbb, 0x0c, 0xc9, 0x2d, 0xe8, 0x73, 0x59, 0xf0, 0xc0, 0xe9,
+ 0x4c, 0xbb, 0xb3, 0x01, 0x35, 0x86, 0x41, 0x37, 0x3c, 0x70, 0xba, 0x15, 0xba, 0xe1, 0x01, 0xb9,
+ 0x0d, 0x07, 0xdb, 0x54, 0xe6, 0x3c, 0x70, 0x7a, 0xd3, 0xee, 0xac, 0x4f, 0x4b, 0x8b, 0x10, 0xb0,
+ 0x12, 0xc9, 0x03, 0xc7, 0xd2, 0xa8, 0x3e, 0x93, 0x3b, 0x30, 0x88, 0x59, 0x26, 0x58, 0xb2, 0x41,
+ 0xa7, 0xaf, 0xf1, 0xda, 0x76, 0x9f, 0xc0, 0xc1, 0x22, 0x4d, 0x42, 0xbe, 0x21, 0x13, 0xe8, 0xbd,
+ 0xc7, 0x9d, 0x8e, 0x6d, 0x53, 0x75, 0x54, 0x91, 0x2f, 0x59, 0x54, 0xa0, 0x8e, 0x6c, 0x53, 0x63,
+ 0xb8, 0x3f, 0xc1, 0xc1, 0x12, 0x2f, 0xb9, 0x8f, 0x3a, 0x16, 0x8b, 0xb1, 0xfc, 0x44, 0x9f, 0xc9,
+ 0x43, 0x38, 0xf0, 0xb5, 0x9e, 0xd3, 0x9d, 0xf6, 0x66, 0xc3, 0xf9, 0x8d, 0x47, 0xf5, 0x65, 0x1f,
+ 0x99, 0x40, 0xb4, 0x24, 0xb8, 0x7f, 0x75, 0x61, 0xb0, 0x4e, 0x58, 0x26, 0xb7, 0x69, 0xfe, 0x9f,
+ 0x5a, 0x4f, 0x61, 0x18, 0xa5, 0x3e, 0x8b, 0x16, 0xff, 0x23, 0xd8, 0x64, 0xa9, 0xcb, 0x66, 0x22,
+ 0x0d, 0x79, 0x84, 0xd2, 0xe9, 0x4d, 0x7b, 0x33, 0x9b, 0xd6, 0x36, 0xf9, 0x1c, 0x6c, 0xcc, 0xb6,
+ 0x18, 0xa3, 0x60, 0x91, 0xae, 0xd0, 0x80, 0x7e, 0x02, 0xc8, 0xf7, 0x30, 0xd2, 0x42, 0xe6, 0x76,
+ 0xd2, 0xe9, 0x5f, 0x8b, 0x67, 0x3c, 0xb4, 0x45, 0x23, 0x2e, 0x8c, 0x98, 0xf0, 0xb7, 0x3c, 0x47,
+ 0x3f, 0x2f, 0x04, 0x3a, 0x07, 0xba, 0xc2, 0x2d, 0x4c, 0x25, 0x25, 0x73, 0x96, 0x63, 0x58, 0x44,
+ 0xce, 0xa1, 0x8e, 0x5b, 0xdb, 0xe4, 0x3e, 0x8c, 0x7d, 0x81, 0x3a, 0x80, 0x17, 0xb0, 0x1c, 0x9d,
+ 0xc1, 0xb4, 0x33, 0xeb, 0xd1, 0x51, 0x05, 0x2e, 0x59, 0x8e, 0xe4, 0x01, 0x1c, 0x45, 0x4c, 0xe6,
+ 0x5e, 0x21, 0x31, 0x30, 0x2c, 0xdb, 0xb0, 0x14, 0xfa, 0x4e, 0x62, 0xa0, 0x58, 0xee, 0x1f, 0x1d,
+ 0x18, 0x0b, 0xb9, 0x4b, 0xfc, 0x33, 0x64, 0x2a, 0xae, 0x54, 0x63, 0xf2, 0x91, 0xe5, 0xb9, 0x90,
+ 0x4e, 0x67, 0xda, 0x99, 0x0d, 0x68, 0x69, 0x29, 0x3c, 0xc0, 0x08, 0x73, 0xd5, 0x5b, 0x8d, 0x1b,
+ 0x4b, 0x25, 0xea, 0xa7, 0x71, 0x26, 0x50, 0xaa, 0xea, 0x29, 0x4f, 0x6d, 0x93, 0x07, 0x30, 0xbe,
+ 0xe0, 0x01, 0x17, 0xe8, 0xab, 0xb4, 0x74, 0x05, 0x15, 0xa1, 0x0d, 0xba, 0x0f, 0x61, 0xb8, 0x0f,
+ 0x65, 0x9d, 0x40, 0x53, 0xb0, 0xd3, 0x16, 0x74, 0x37, 0x30, 0xbe, 0xc8, 0x45, 0x83, 0xfc, 0x10,
+ 0x26, 0x75, 0xb1, 0xbd, 0x2d, 0xb2, 0x00, 0x45, 0xf9, 0xd1, 0x71, 0x8d, 0xbf, 0xd0, 0x30, 0xf9,
+ 0x16, 0x6e, 0x18, 0x82, 0x27, 0x8b, 0x8b, 0xcb, 0x34, 0x2a, 0x62, 0x94, 0xe5, 0x5d, 0x26, 0xc6,
+ 0xb1, 0xae, 0x71, 0xf7, 0x9f, 0x1e, 0x1c, 0xbf, 0xbe, 0x22, 0x70, 0x02, 0xdd, 0x50, 0xea, 0x71,
+ 0x3b, 0x9a, 0xdf, 0x69, 0xf4, 0xb8, 0xe6, 0x9d, 0xad, 0xd5, 0x52, 0xd2, 0x6e, 0x28, 0xc9, 0x37,
+ 0x60, 0xf9, 0x82, 0x17, 0x5a, 0xff, 0x68, 0x7e, 0xb3, 0x39, 0x81, 0xf4, 0xe5, 0x3b, 0x4d, 0xd3,
+ 0x04, 0x72, 0x02, 0x7d, 0x1e, 0xc4, 0x2c, 0xd3, 0x93, 0x37, 0x9c, 0xdf, 0x6a, 0x30, 0xeb, 0x35,
+ 0xa7, 0x86, 0xa2, 0xca, 0x29, 0xcb, 0xe9, 0x5f, 0x31, 0x95, 0xbd, 0xa5, 0xa7, 0xb5, 0x0d, 0x92,
+ 0xef, 0xc0, 0xae, 0x80, 0x6a, 0x22, 0x9b, 0xf1, 0xab, 0xfd, 0xa1, 0x9f, 0x58, 0xc4, 0x81, 0xc3,
+ 0x4c, 0x60, 0x50, 0xc4, 0x99, 0x73, 0xa8, 0x0b, 0x52, 0x99, 0xe4, 0x87, 0x2b, 0xe3, 0xa1, 0x47,
+ 0x6d, 0x38, 0x77, 0x1a, 0x82, 0x2d, 0x3f, 0xbd, 0x32, 0x4d, 0x0e, 0x1c, 0x0a, 0x0c, 0x05, 0xca,
+ 0xad, 0x1e, 0xbf, 0x01, 0xad, 0x4c, 0xf2, 0xac, 0xd5, 0x75, 0x07, 0xb4, 0xee, 0xed, 0x86, 0x6e,
+ 0xc3, 0x4b, 0x5b, 0x03, 0x72, 0x0f, 0xc0, 0xb4, 0x69, 0xcd, 0xf7, 0xe8, 0x0c, 0xf5, 0x54, 0x37,
+ 0x10, 0x95, 0x73, 0x6b, 0x48, 0x9c, 0xd1, 0xb5, 0x9c, 0x5b, 0x7e, 0xda, 0xa6, 0xbb, 0x67, 0x30,
+ 0xa9, 0x5b, 0xba, 0x48, 0x93, 0x5c, 0xa4, 0x91, 0xba, 0x87, 0x2c, 0x7c, 0xdf, 0xcc, 0xa4, 0xda,
+ 0xc6, 0xca, 0x54, 0x9e, 0x18, 0xa5, 0x64, 0x1b, 0xb3, 0x18, 0x36, 0xad, 0x4c, 0xf7, 0x29, 0x8c,
+ 0x6b, 0x9d, 0xf5, 0x2e, 0xf1, 0xd5, 0xde, 0x87, 0x3c, 0x61, 0xd1, 0xb9, 0xc0, 0xa5, 0xaa, 0xb5,
+ 0x51, 0x6a, 0x61, 0xee, 0x9f, 0x3d, 0x98, 0xa8, 0xca, 0x7b, 0x6a, 0xdb, 0xa5, 0x87, 0x49, 0x2e,
+ 0x76, 0x6a, 0xe1, 0x43, 0x81, 0xb8, 0xe7, 0xc9, 0xc6, 0xcb, 0x79, 0xf9, 0xe6, 0x8d, 0xe9, 0xa8,
+ 0x02, 0xdf, 0xf2, 0x18, 0xc9, 0x97, 0x30, 0x0c, 0x45, 0xba, 0xc7, 0xc4, 0x50, 0xba, 0x9a, 0x02,
+ 0x06, 0xd2, 0x84, 0xaf, 0x60, 0x14, 0x63, 0xac, 0xc5, 0x35, 0xa3, 0xa7, 0x19, 0xc3, 0x12, 0xd3,
+ 0x94, 0xfb, 0x30, 0x8e, 0x31, 0xfe, 0x20, 0x78, 0x8e, 0x86, 0x63, 0x99, 0x40, 0x15, 0x58, 0x91,
+ 0x32, 0xb6, 0x41, 0xe9, 0x49, 0x9f, 0x25, 0x09, 0x06, 0xfa, 0x0f, 0x61, 0xd1, 0x91, 0x06, 0xd7,
+ 0x06, 0x23, 0x4f, 0xe0, 0x56, 0x49, 0x7a, 0xcf, 0xb3, 0x0c, 0x03, 0x2f, 0x63, 0x02, 0x93, 0x5c,
+ 0xbf, 0x75, 0x16, 0x25, 0x86, 0x6b, 0x5c, 0xe7, 0xda, 0xf3, 0x49, 0x56, 0x45, 0xca, 0x31, 0xd1,
+ 0xcf, 0x5e, 0x25, 0xfb, 0x8b, 0xc1, 0x14, 0x89, 0x8b, 0x98, 0x65, 0x9e, 0x40, 0x99, 0x46, 0x97,
+ 0xe6, 0xe9, 0x1b, 0xd3, 0x91, 0x06, 0xa9, 0xc1, 0xc8, 0x17, 0x00, 0x46, 0x29, 0x62, 0xfb, 0x9d,
+ 0x63, 0x6b, 0x19, 0x5b, 0x23, 0xaf, 0xd8, 0x7e, 0x57, 0xb9, 0xbd, 0x8c, 0x67, 0xe5, 0xe0, 0x95,
+ 0xee, 0x73, 0x05, 0xa8, 0x87, 0xb3, 0x76, 0x7b, 0x17, 0x45, 0x28, 0xf5, 0x88, 0x95, 0x89, 0x28,
+ 0xca, 0x69, 0x11, 0x4a, 0xf7, 0xef, 0x0e, 0xdc, 0x14, 0x28, 0xf3, 0x54, 0x60, 0xab, 0x55, 0x5f,
+ 0x9b, 0xaf, 0xa5, 0xa7, 0xde, 0x2c, 0x26, 0xd0, 0xfc, 0x9a, 0x2d, 0x6a, 0xee, 0xb6, 0x28, 0x41,
+ 0x72, 0x02, 0x37, 0xda, 0xe5, 0xf1, 0xd3, 0x0f, 0xba, 0x65, 0x16, 0x3d, 0x6e, 0xd6, 0x66, 0x91,
+ 0x7e, 0x50, 0x7d, 0x0b, 0x53, 0xf1, 0xbe, 0x6e, 0x7e, 0xd9, 0xb7, 0x12, 0xab, 0x5a, 0x5b, 0x25,
+ 0xd3, 0x68, 0xdb, 0xb0, 0xc4, 0x34, 0xa5, 0x4e, 0xac, 0x04, 0x55, 0xdb, 0x3a, 0x75, 0x62, 0xb4,
+ 0x04, 0xdd, 0x8f, 0x30, 0x6c, 0x5e, 0xe7, 0x31, 0x58, 0x81, 0x19, 0x55, 0xb5, 0x42, 0x77, 0x1b,
+ 0x2b, 0x74, 0x75, 0x48, 0xa9, 0x26, 0x92, 0x67, 0x6a, 0xe1, 0xb5, 0x96, 0x5e, 0x87, 0xe1, 0xfc,
+ 0x5e, 0xf3, 0xa9, 0xb8, 0x5e, 0x30, 0x5a, 0xd1, 0x4f, 0x56, 0x8d, 0x17, 0xd7, 0xbc, 0xa4, 0xc4,
+ 0x86, 0x3e, 0x5d, 0xff, 0xba, 0x5a, 0x4c, 0x3e, 0x53, 0xc7, 0xd3, 0xb7, 0xf4, 0x6c, 0x3d, 0xe9,
+ 0x90, 0x43, 0xe8, 0xfd, 0x76, 0xb6, 0x9e, 0x74, 0xd5, 0x81, 0x9e, 0x2e, 0x27, 0x3d, 0x72, 0x13,
+ 0x8e, 0x4f, 0x5f, 0xbd, 0x59, 0xfc, 0xec, 0x3d, 0x5f, 0x2d, 0x3d, 0xf3, 0x85, 0x75, 0xf2, 0x18,
+ 0x06, 0xd5, 0x5b, 0x4b, 0x8e, 0x00, 0xd4, 0xd9, 0x6b, 0xa8, 0x9d, 0xbf, 0x78, 0xfe, 0xee, 0xd5,
+ 0xa4, 0x43, 0x06, 0x60, 0xad, 0xde, 0xac, 0x7e, 0x9c, 0x74, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff,
+ 0x51, 0xb3, 0x06, 0x24, 0x82, 0x09, 0x00, 0x00,
}
diff --git a/lxd/migration/migrate.proto b/lxd/migration/migrate.proto
index 721662e424..257f038e21 100644
--- a/lxd/migration/migrate.proto
+++ b/lxd/migration/migrate.proto
@@ -58,6 +58,11 @@ message zfsFeatures {
optional bool compress = 1;
}
+message btrfsFeatures {
+ optional bool migration_header = 1;
+ optional bool header_subvolumes = 2;
+}
+
message MigrationHeader {
required MigrationFSType fs = 1;
optional CRIUType criu = 2;
@@ -69,6 +74,7 @@ message MigrationHeader {
optional bool refresh = 9;
optional zfsFeatures zfsFeatures = 10;
optional int64 volumeSize = 11;
+ optional btrfsFeatures btrfsFeatures = 12;
}
message MigrationControl {
From 7217506b510fab252968072ab20bfedfd827f196 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 11:46:09 +0100
Subject: [PATCH 07/15] lxd/migration/utils: Adds GetBtrfsFeaturesSlice
function
And constants for the BTRFS features.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/migration/utils.go | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/lxd/migration/utils.go b/lxd/migration/utils.go
index 92265c2ace..9d3762dc2a 100644
--- a/lxd/migration/utils.go
+++ b/lxd/migration/utils.go
@@ -1,5 +1,11 @@
package migration
+// BTRFSFeatureMigrationHeader indicates a migration header will be sent/recv in data channel first.
+const BTRFSFeatureMigrationHeader = "migration_header"
+
+// BTRFSFeatureSubvolumes indicates migration can send/recv subvolumes.
+const BTRFSFeatureSubvolumes = "header_subvolumes"
+
// GetRsyncFeaturesSlice returns a slice of strings representing the supported RSYNC features
func (m *MigrationHeader) GetRsyncFeaturesSlice() []string {
features := []string{}
@@ -42,3 +48,23 @@ func (m *MigrationHeader) GetZfsFeaturesSlice() []string {
return features
}
+
+// GetBtrfsFeaturesSlice returns a slice of strings representing the supported BTRFS features.
+func (m *MigrationHeader) GetBtrfsFeaturesSlice() []string {
+ features := []string{}
+ if m == nil {
+ return features
+ }
+
+ if m.BtrfsFeatures != nil {
+ if m.BtrfsFeatures.MigrationHeader != nil && *m.BtrfsFeatures.MigrationHeader == true {
+ features = append(features, BTRFSFeatureMigrationHeader)
+ }
+
+ if m.BtrfsFeatures.HeaderSubvolumes != nil && *m.BtrfsFeatures.HeaderSubvolumes == true {
+ features = append(features, BTRFSFeatureSubvolumes)
+ }
+ }
+
+ return features
+}
From 58172c554abec5a8671131872e1119986536a529 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 11:46:45 +0100
Subject: [PATCH 08/15] lxd/migration/migration/volumes: Adds BTRFS feature
support to TypesToHeader
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/migration/migration_volumes.go | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/lxd/migration/migration_volumes.go b/lxd/migration/migration_volumes.go
index 331b003c22..314ea99bbe 100644
--- a/lxd/migration/migration_volumes.go
+++ b/lxd/migration/migration_volumes.go
@@ -72,6 +72,23 @@ func TypesToHeader(types ...Type) MigrationHeader {
header.ZfsFeatures = &features
}
+ // Add BTRFS features if preferred type is BTRFS.
+ if preferredType.FSType == MigrationFSType_BTRFS {
+ features := BtrfsFeatures{
+ MigrationHeader: &missingFeature,
+ HeaderSubvolumes: &missingFeature,
+ }
+ for _, feature := range preferredType.Features {
+ if feature == BTRFSFeatureMigrationHeader {
+ features.MigrationHeader = &hasFeature
+ } else if feature == BTRFSFeatureSubvolumes {
+ features.HeaderSubvolumes = &hasFeature
+ }
+ }
+
+ header.BtrfsFeatures = &features
+ }
+
// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
for _, t := range types {
if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
From d3e78fc65202a5c60a8ce228f4310a587b34cde0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 11:47:03 +0100
Subject: [PATCH 09/15] lxd/migration/migration/volumes: Adds BTRFS feature
support to MatchTypes
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/migration/migration_volumes.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/migration/migration_volumes.go b/lxd/migration/migration_volumes.go
index 314ea99bbe..261f7eaab0 100644
--- a/lxd/migration/migration_volumes.go
+++ b/lxd/migration/migration_volumes.go
@@ -145,6 +145,8 @@ func MatchTypes(offer MigrationHeader, fallbackType MigrationFSType, ourTypes []
var offeredFeatures []string
if offerFSType == MigrationFSType_ZFS {
offeredFeatures = offer.GetZfsFeaturesSlice()
+ } else if offerFSType == MigrationFSType_BTRFS {
+ offeredFeatures = offer.GetBtrfsFeaturesSlice()
} else if offerFSType == MigrationFSType_RSYNC {
offeredFeatures = offer.GetRsyncFeaturesSlice()
if !shared.StringInSlice("bidirectional", offeredFeatures) {
From 1fd33760969d4254ae3722471dac320a3383a763 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 11:47:31 +0100
Subject: [PATCH 10/15] lxd/storage/drivers/driver/btrfs: Adds BTRFS features
to MigrationTypes
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs.go | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs.go b/lxd/storage/drivers/driver_btrfs.go
index 4c4e569b81..e69d0ff18b 100644
--- a/lxd/storage/drivers/driver_btrfs.go
+++ b/lxd/storage/drivers/driver_btrfs.go
@@ -370,6 +370,7 @@ func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
+ btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}
// Only offer rsync for refreshes or if running in an unprivileged container.
if refresh || d.state.OS.RunningInUserNS {
@@ -392,7 +393,8 @@ func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migratio
if contentType == ContentTypeBlock {
return []migration.Type{
{
- FSType: migration.MigrationFSType_BTRFS,
+ FSType: migration.MigrationFSType_BTRFS,
+ Features: btrfsFeatures,
},
{
FSType: migration.MigrationFSType_BLOCK_AND_RSYNC,
@@ -403,7 +405,8 @@ func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migratio
return []migration.Type{
{
- FSType: migration.MigrationFSType_BTRFS,
+ FSType: migration.MigrationFSType_BTRFS,
+ Features: btrfsFeatures,
},
{
FSType: migration.MigrationFSType_RSYNC,
From 6fb47c0e094e23e6cf54362e38ed6a161590aa7c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 16:51:18 +0100
Subject: [PATCH 11/15] lxd/storage/memorypipe: Dont make ioutil.ReadAll panic
on cancel
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/memorypipe/memory_pipe.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/storage/memorypipe/memory_pipe.go b/lxd/storage/memorypipe/memory_pipe.go
index 28744d1c7b..4db48f874f 100644
--- a/lxd/storage/memorypipe/memory_pipe.go
+++ b/lxd/storage/memorypipe/memory_pipe.go
@@ -31,12 +31,12 @@ func (p *pipe) Read(b []byte) (int, error) {
select {
case msg := <-p.ch:
if msg.err == io.EOF {
- return -1, msg.err
+ return 0, msg.err
}
n := copy(b, msg.data)
return n, msg.err
case <-p.ctx.Done():
- return -1, p.ctx.Err()
+ return 0, p.ctx.Err()
}
}
@@ -51,7 +51,7 @@ func (p *pipe) Write(b []byte) (int, error) {
case p.otherEnd.ch <- msg: // Sent msg to the other side's Read function.
return len(msg.data), msg.err
case <-p.ctx.Done():
- return -1, p.ctx.Err()
+ return 0, p.ctx.Err()
}
}
From 7e16de785564ade6f6e9d2100b15e53330eac55f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 16:57:10 +0100
Subject: [PATCH 12/15] lxd/storage/drivers/driver/btrfs/utils: Kill btrfs send
on error in sendSubvolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_utils.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index bd48d3fc5a..bf2fd9097b 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -286,6 +286,7 @@ func (d *btrfs) sendSubvolume(path string, parent string, conn io.ReadWriteClose
_, err := io.Copy(conn, stdoutPipe)
chStdoutPipe <- err
conn.Close()
+ cmd.Process.Kill() // This closes stderr.
}()
// Run the command.
From 10d380152044cdfbf38c143ab9ad094670023b9a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 16:57:29 +0100
Subject: [PATCH 13/15] lxd/storage/drivers/driver/btrfs/utils: Support
subvolumes in receiveSubvolume
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_utils.go | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index bf2fd9097b..12df54820a 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -378,8 +378,15 @@ func (d *btrfs) receiveSubvolume(path string, targetPath string, conn io.ReadWri
return nil
}
- // Handle older LXD versions.
+ // Location we would expect to receive the root subvolume.
receivedSnapshot := fmt.Sprintf("%s/.migration-send", path)
+
+ // Handle non-root subvolumes.
+ if !shared.PathExists(receivedSnapshot) {
+ receivedSnapshot = fmt.Sprintf("%s/%s", path, filepath.Base(targetPath))
+ }
+
+ // Handle older LXD versions.
if !shared.PathExists(receivedSnapshot) {
receivedSnapshot = fmt.Sprintf("%s/.root", path)
}
From a1b54501a15a9a52efc99c9fd17251d3e045b636 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 17:17:00 +0100
Subject: [PATCH 14/15] lxd/storage/drivers/driver/btrfs/utils: Adds
migrationHeader function returning BTRFSMigrationHeader struct
Scans a volume and its snapshots for BTRFS subvolumes ready for sending to migration target.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_utils.go | 69 +++++++++++++++++++++++
1 file changed, 69 insertions(+)
diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
index 12df54820a..e5744b7690 100644
--- a/lxd/storage/drivers/driver_btrfs_utils.go
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -417,3 +417,72 @@ func (d *btrfs) volumeSize(vol Volume) string {
return size
}
+
+// BTRFSSubVolume is the structure used to store information about a subvolume.
+type BTRFSSubVolume struct {
+ Path string
+ Snapshot string
+ Readonly bool
+}
+
+// BTRFSMigrationHeader is the meta data header sent before each volume during a migration.
+type BTRFSMigrationHeader struct {
+ Subvolumes []BTRFSSubVolume
+}
+
+// migrationHeader scans the volume and any specified snapshots, returning a migration header containing subvolume
+// information.
+func (d *btrfs) migrationHeader(vol Volume, snapshots []string) (*BTRFSMigrationHeader, error) {
+ var migrationHeader BTRFSMigrationHeader
+
+ // Add snapshots to volumes list.
+ for _, snapName := range snapshots {
+ snapVol, _ := vol.NewSnapshot(snapName)
+
+ // Add snapshot root volume to volumes list.
+ migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, BTRFSSubVolume{
+ Snapshot: snapVol.name,
+ Path: string(filepath.Separator),
+ Readonly: false,
+ })
+
+ // Find any subvolumes in snapshot volume
+ subVolPaths, err := d.getSubvolumes(snapVol.MountPath())
+ if err != nil {
+ return nil, err
+ }
+ sort.Sort(sort.StringSlice(subVolPaths))
+
+ // Add any subvolumes under the same snapshot volume name, but with a different path.
+ for _, subVolPath := range subVolPaths {
+ migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, BTRFSSubVolume{
+ Snapshot: snapVol.name,
+ Path: fmt.Sprintf("%s%s", string(filepath.Separator), subVolPath),
+ Readonly: false,
+ })
+ }
+ }
+
+ // Add main root volume to volumes list.
+ migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, BTRFSSubVolume{
+ Path: string(filepath.Separator),
+ Readonly: false,
+ })
+
+ // Find any subvolumes in main volume.
+ subVolPaths, err := d.getSubvolumes(vol.MountPath())
+ if err != nil {
+ return nil, err
+ }
+ sort.Sort(sort.StringSlice(subVolPaths))
+
+ // Add any subvolumes under the main volume name, but with a different path.
+ for _, subVolPath := range subVolPaths {
+ migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, BTRFSSubVolume{
+ Path: fmt.Sprintf("%s%s", string(filepath.Separator), subVolPath),
+ Readonly: false,
+ })
+ }
+
+ return &migrationHeader, nil
+}
From e08e5688ac8fa967038d005bd211286e739abe54 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 6 May 2020 17:24:43 +0100
Subject: [PATCH 15/15] lxd/storage/drivers/driver/btrfs/volumes: Updates
CreateVolumeFromMigration to receive subvolumes
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/drivers/driver_btrfs_volumes.go | 29 +++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
index 0d5da1c297..b87666d238 100644
--- a/lxd/storage/drivers/driver_btrfs_volumes.go
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -318,6 +318,21 @@ func (d *btrfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, v
return ErrNotSupported
}
+ var migrationHeader BTRFSMigrationHeader
+
+ // Inspect negotiated features to see if we are expecting to get a metadata migration header frame.
+ if shared.StringInSlice(migration.BTRFSFeatureMigrationHeader, volTargetArgs.MigrationType.Features) {
+ buf, err := ioutil.ReadAll(conn)
+ if err != nil {
+ return errors.Wrapf(err, "Failed reading migration header")
+ }
+
+ err = json.Unmarshal(buf, &migrationHeader)
+ if err != nil {
+ return errors.Wrapf(err, "Failed decoding migration header")
+ }
+ }
+
// Handle btrfs send/receive migration.
if len(volTargetArgs.Snapshots) > 0 {
snapshotsDir := GetVolumeSnapshotDir(d.name, vol.volType, vol.name)
@@ -356,11 +371,25 @@ func (d *btrfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, v
}
wrapper := migration.ProgressWriter(op, "fs_progress", vol.name)
+ d.logger.Debug("Receiving volume", log.Ctx{"name": vol.name, "tmpPath": tmpVolumesMountPoint, "path": vol.MountPath()})
err = d.receiveSubvolume(tmpVolumesMountPoint, vol.MountPath(), conn, wrapper)
if err != nil {
return err
}
+ for _, subVol := range migrationHeader.Subvolumes {
+ if subVol.Snapshot != "" || subVol.Path == string(filepath.Separator) {
+ continue // We're only looking for subvolumes of main volume.
+ }
+
+ path := filepath.Join(vol.MountPath(), subVol.Path)
+ d.logger.Debug("Receiving volume", log.Ctx{"name": vol.name, "tmpPath": tmpVolumesMountPoint, "path": path})
+ err = d.receiveSubvolume(tmpVolumesMountPoint, filepath.Join(vol.MountPath(), subVol.Path), conn, wrapper)
+ if err != nil {
+ return err
+ }
+ }
+
return nil
}
More information about the lxc-devel
mailing list