[lxc-devel] [lxd/master] Add pre-copy migration support to LXD
brauner on Github
lxc-bot at linuxcontainers.org
Mon Dec 4 17:08:44 UTC 2017
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 1786 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20171204/2592142d/attachment.bin>
-------------- next part --------------
From 1ffcbfaa69eb7b8c91fea4ab026639a7359db3c8 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Wed, 29 Nov 2017 08:56:18 +0100
Subject: [PATCH 1/9] migrate: prepare for pre-copy migration
The upcoming pre-copy migration support needs additional parameters to
the Migrate() function. In order to have a cleaner interface this patch
modifies the Migrate() function to use one struct as parameter instead
of currently five (and more in the future).
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/container.go | 12 +++++++--
lxd/container_lxc.go | 74 ++++++++++++++++++++++++++++++++++++++--------------
lxd/migrate.go | 30 ++++++++++++++++++---
3 files changed, 91 insertions(+), 25 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index 4515d9c3a..ef577e6d2 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -416,7 +416,7 @@ type container interface {
/* actionScript here is a script called action.sh in the stateDir, to
* be passed to CRIU as --action-script
*/
- Migrate(cmd uint, stateDir string, function string, stop bool, actionScript bool) error
+ Migrate(args *CriuMigrationArgs) error
Snapshots() ([]container, error)
// Config handling
@@ -696,7 +696,15 @@ func containerCreateAsSnapshot(s *state.State, args db.ContainerArgs, sourceCont
* after snapshotting will fail.
*/
- err = sourceContainer.Migrate(lxc.MIGRATE_DUMP, stateDir, "snapshot", false, false)
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_DUMP,
+ stateDir: stateDir,
+ function: "snapshot",
+ stop: false,
+ actionScript: false,
+ }
+
+ err = sourceContainer.Migrate(&criuMigrationArgs)
if err != nil {
os.RemoveAll(sourceContainer.StatePath())
return nil, err
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 5f58821e4..da1d2c492 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -2153,7 +2153,15 @@ func (c *containerLXC) Start(stateful bool) error {
return fmt.Errorf("Container has no existing state to restore.")
}
- err := c.Migrate(lxc.MIGRATE_RESTORE, c.StatePath(), "snapshot", false, false)
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_RESTORE,
+ stateDir: c.StatePath(),
+ function: "snapshot",
+ stop: false,
+ actionScript: false,
+ }
+
+ err := c.Migrate(&criuMigrationArgs)
if err != nil && !c.IsRunning() {
return err
}
@@ -2370,8 +2378,16 @@ func (c *containerLXC) Stop(stateful bool) error {
return err
}
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_DUMP,
+ stateDir: stateDir,
+ function: "snapshot",
+ stop: true,
+ actionScript: false,
+ }
+
// Checkpoint
- err = c.Migrate(lxc.MIGRATE_DUMP, stateDir, "snapshot", true, false)
+ err = c.Migrate(&criuMigrationArgs)
if err != nil {
op.Done(err)
logger.Error("Failed stopping container", ctxMap)
@@ -2877,7 +2893,17 @@ func (c *containerLXC) Restore(sourceContainer container, stateful bool) error {
logger.Debug("Performing stateful restore", ctxMap)
c.stateful = true
- err := c.Migrate(lxc.MIGRATE_RESTORE, c.StatePath(), "snapshot", false, false)
+
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_RESTORE,
+ stateDir: c.StatePath(),
+ function: "snapshot",
+ stop: false,
+ actionScript: false,
+ }
+
+ // Checkpoint
+ err := c.Migrate(&criuMigrationArgs)
if err != nil {
return err
}
@@ -4468,14 +4494,22 @@ func getCRIULogErrors(imagesDir string, method string) (string, error) {
return strings.Join(ret, "\n"), nil
}
-func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop bool, actionScript bool) error {
+type CriuMigrationArgs struct {
+ cmd uint
+ stateDir string
+ function string
+ stop bool
+ actionScript bool
+}
+
+func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
ctxMap := log.Ctx{"name": c.name,
"created": c.creationDate,
"ephemeral": c.ephemeral,
"used": c.lastUsedDate,
- "statedir": stateDir,
- "actionscript": actionScript,
- "stop": stop}
+ "statedir": args.stateDir,
+ "actionscript": args.actionScript,
+ "stop": args.stop}
_, err := exec.LookPath("criu")
if err != nil {
@@ -4491,7 +4525,7 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
}
prettyCmd := ""
- switch cmd {
+ switch args.cmd {
case lxc.MIGRATE_PRE_DUMP:
prettyCmd = "pre-dump"
case lxc.MIGRATE_DUMP:
@@ -4500,7 +4534,7 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
prettyCmd = "restore"
default:
prettyCmd = "unknown"
- logger.Warn("unknown migrate call", log.Ctx{"cmd": cmd})
+ logger.Warn("unknown migrate call", log.Ctx{"cmd": args.cmd})
}
preservesInodes := c.storage.PreservesInodes()
@@ -4517,7 +4551,7 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
* instead of having it be a child of LXD, so let's hijack the command
* here and do the extra fork.
*/
- if cmd == lxc.MIGRATE_RESTORE {
+ if args.cmd == lxc.MIGRATE_RESTORE {
// Run the shared start
_, err := c.startCommon()
if err != nil {
@@ -4541,7 +4575,7 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
return err
}
- err = idmapset.ShiftRootfs(stateDir)
+ err = idmapset.ShiftRootfs(args.stateDir)
if ourStart {
_, err2 := c.StorageStop()
if err != nil {
@@ -4563,7 +4597,7 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
c.name,
c.state.OS.LxcPath,
configPath,
- stateDir,
+ args.stateDir,
fmt.Sprintf("%v", preservesInodes))
if out != "" {
@@ -4579,8 +4613,8 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
}
script := ""
- if actionScript {
- script = filepath.Join(stateDir, "action.sh")
+ if args.actionScript {
+ script = filepath.Join(args.stateDir, "action.sh")
}
// TODO: make this configurable? Ultimately I think we don't
@@ -4592,27 +4626,27 @@ func (c *containerLXC) Migrate(cmd uint, stateDir string, function string, stop
ghostLimit := uint64(256 * 1024 * 1024)
opts := lxc.MigrateOptions{
- Stop: stop,
- Directory: stateDir,
+ Stop: args.stop,
+ Directory: args.stateDir,
Verbose: true,
PreservesInodes: preservesInodes,
ActionScript: script,
GhostLimit: ghostLimit,
}
- migrateErr = c.c.Migrate(cmd, opts)
+ migrateErr = c.c.Migrate(args.cmd, opts)
}
- collectErr := collectCRIULogFile(c, stateDir, function, prettyCmd)
+ collectErr := collectCRIULogFile(c, args.stateDir, args.function, prettyCmd)
if collectErr != nil {
logger.Error("Error collecting checkpoint log file", log.Ctx{"err": collectErr})
}
if migrateErr != nil {
- log, err2 := getCRIULogErrors(stateDir, prettyCmd)
+ log, err2 := getCRIULogErrors(args.stateDir, prettyCmd)
if err2 == nil {
logger.Info("Failed migrating container", ctxMap)
- migrateErr = fmt.Errorf("%s %s failed\n%s", function, prettyCmd, log)
+ migrateErr = fmt.Errorf("%s %s failed\n%s", args.function, prettyCmd, log)
}
return migrateErr
diff --git a/lxd/migrate.go b/lxd/migrate.go
index ae18f19aa..553124d15 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -563,7 +563,15 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
}
go func() {
- dumpSuccess <- s.container.Migrate(lxc.MIGRATE_DUMP, checkpointDir, "migration", true, true)
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_DUMP,
+ stateDir: checkpointDir,
+ function: "migration",
+ stop: true,
+ actionScript: true,
+ }
+
+ dumpSuccess <- s.container.Migrate(&criuMigrationArgs)
os.RemoveAll(checkpointDir)
}()
@@ -577,7 +585,15 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
}
} else {
defer os.RemoveAll(checkpointDir)
- err = s.container.Migrate(lxc.MIGRATE_DUMP, checkpointDir, "migration", true, false)
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_DUMP,
+ stateDir: checkpointDir,
+ function: "migration",
+ stop: true,
+ actionScript: false,
+ }
+
+ err = s.container.Migrate(&criuMigrationArgs)
if err != nil {
return abort(err)
}
@@ -968,7 +984,15 @@ func (c *migrationSink) Do(migrateOp *operation) error {
}
if live {
- err = c.src.container.Migrate(lxc.MIGRATE_RESTORE, imagesDir, "migration", false, false)
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_RESTORE,
+ stateDir: imagesDir,
+ function: "migration",
+ stop: false,
+ actionScript: false,
+ }
+
+ err = c.src.container.Migrate(&criuMigrationArgs)
if err != nil {
restore <- err
return
From 28827c3eb6d4d58dcac92569117750ab0576c6b6 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Thu, 30 Nov 2017 08:48:53 +0000
Subject: [PATCH 2/9] migration: prepare for pre-copy migration (part 2)
In addition to the previous pre-copy migration prepare commit which
changed the parameters of the Migrate() function this extends the
protocol between migration source and destination to detect if
pre-copy migration should be used.
Currently pre-copy migration defaults to off and can be enabled by
setting 'migration.pre_copy.enabled' to true. If it is 'true' and the
migration destination side also acknowledges that it supports pre-copy
migration the variable 'use_pre_dumps' is set to 'true' and the
following commits can use this variable to use pre-copy migration or not
in the upcoming patches.
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/migrate.go | 36 +++++++++++++++++++++++++
lxd/migrate.pb.go | 76 ++++++++++++++++++++++++++++++-----------------------
lxd/migrate.proto | 1 +
shared/container.go | 2 ++
4 files changed, 82 insertions(+), 33 deletions(-)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index 553124d15..8cb11dcdb 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -412,6 +412,23 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
}
}
+ // TODO: ask CRIU if this system (kernel+criu) supports
+ // pre-copy (dirty memory tracking)
+ // The user should also be enable to influence it from the
+ // command-line.
+
+ // What does the config say about pre-copy
+ tmp := s.container.ExpandedConfig()["migration.pre_copy.enabled"]
+
+ // default to false for pre-dumps as long as libxlc has no
+ // detection for the feature
+ use_pre_dumps := false
+
+ if tmp != "" {
+ use_pre_dumps = shared.IsTrue(tmp)
+ }
+ logger.Debugf("migration.pre_copy.enabled %s", use_pre_dumps)
+
// The protocol says we have to send a header no matter what, so let's
// do that, but then immediately send an error.
myType := s.container.Storage().MigrationType()
@@ -421,6 +438,7 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
Idmap: idmaps,
SnapshotNames: snapshotNames,
Snapshots: snapshots,
+ Predump: proto.Bool(use_pre_dumps),
}
err = s.send(&header)
@@ -454,6 +472,15 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
}
}
+ // Check if the other side knows about pre-dumping and
+ // the associated rsync protocol
+ use_pre_dumps = header.GetPredump()
+ if use_pre_dumps {
+ logger.Debugf("The other side does support pre-copy")
+ } else {
+ logger.Debugf("The other side does not support pre-copy")
+ }
+
// All failure paths need to do a few things to correctly handle errors before returning.
// Unfortunately, handling errors is not well-suited to defer as the code depends on the
// status of driver and the error value. The error value is especially tricky due to the
@@ -875,6 +902,15 @@ func (c *migrationSink) Do(migrateOp *operation) error {
resp.Fs = &myType
}
+ if header.GetPredump() == true {
+ // If the other side wants pre-dump and if
+ // this side supports it, let's use it.
+ // TODO: check kernel+criu (and config?)
+ resp.Predump = proto.Bool(true)
+ } else {
+ resp.Predump = proto.Bool(false)
+ }
+
err = sender(&resp)
if err != nil {
controller(err)
diff --git a/lxd/migrate.pb.go b/lxd/migrate.pb.go
index 93dce20cf..a67c4bf3e 100644
--- a/lxd/migrate.pb.go
+++ b/lxd/migrate.pb.go
@@ -1,5 +1,6 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
+// Code generated by protoc-gen-go.
// source: lxd/migrate.proto
+// DO NOT EDIT!
/*
Package main is a generated protocol buffer package.
@@ -279,6 +280,7 @@ type MigrationHeader struct {
Idmap []*IDMapType `protobuf:"bytes,3,rep,name=idmap" json:"idmap,omitempty"`
SnapshotNames []string `protobuf:"bytes,4,rep,name=snapshotNames" json:"snapshotNames,omitempty"`
Snapshots []*Snapshot `protobuf:"bytes,5,rep,name=snapshots" json:"snapshots,omitempty"`
+ Predump *bool `protobuf:"varint,7,opt,name=predump" json:"predump,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@@ -322,6 +324,13 @@ func (m *MigrationHeader) GetSnapshots() []*Snapshot {
return nil
}
+func (m *MigrationHeader) GetPredump() bool {
+ if m != nil && m.Predump != nil {
+ return *m.Predump
+ }
+ return false
+}
+
type MigrationControl struct {
Success *bool `protobuf:"varint,1,req,name=success" json:"success,omitempty"`
// optional failure message if sending a failure
@@ -362,38 +371,39 @@ func init() {
func init() { proto.RegisterFile("lxd/migrate.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 523 bytes of a gzipped FileDescriptorProto
+ // 535 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0x4d, 0x6f, 0xd3, 0x40,
- 0x10, 0xc5, 0x5f, 0xa9, 0x3d, 0x09, 0xa9, 0x59, 0x01, 0x5a, 0x21, 0x0e, 0x91, 0xd5, 0x4a, 0x51,
+ 0x10, 0xc5, 0x5f, 0x89, 0x3d, 0x0d, 0xa9, 0x59, 0x01, 0x5a, 0x21, 0x0e, 0x96, 0xd5, 0x4a, 0x56,
0x85, 0xd2, 0x2a, 0x37, 0x8e, 0x24, 0x21, 0x6a, 0x25, 0x1a, 0xd0, 0xa6, 0x3d, 0xc0, 0x05, 0xad,
- 0xec, 0x89, 0xb3, 0xc2, 0x5f, 0xf2, 0xda, 0x15, 0x3d, 0xf1, 0x33, 0xf9, 0x2f, 0x9c, 0xd0, 0xae,
- 0x3f, 0x9a, 0x48, 0x70, 0x9b, 0xf7, 0xe6, 0x79, 0x9e, 0xe7, 0xed, 0xc0, 0x8b, 0xe4, 0x67, 0x74,
- 0x99, 0x8a, 0xb8, 0xe4, 0x15, 0xce, 0x8a, 0x32, 0xaf, 0x72, 0x62, 0xa7, 0x5c, 0x64, 0xc1, 0x2f,
- 0xf0, 0x6e, 0x56, 0xb7, 0xbc, 0xb8, 0x7b, 0x2c, 0x90, 0xbc, 0x04, 0x47, 0xc8, 0x5a, 0x44, 0xd4,
- 0x98, 0x98, 0x53, 0x97, 0x35, 0xa0, 0x61, 0x63, 0x11, 0x51, 0xb3, 0x63, 0x63, 0x11, 0x91, 0xd7,
- 0x30, 0xd8, 0xe7, 0xb2, 0x12, 0x11, 0xb5, 0x26, 0xe6, 0xd4, 0x61, 0x2d, 0x22, 0x04, 0xec, 0x4c,
- 0x8a, 0x88, 0xda, 0x9a, 0xd5, 0x35, 0x79, 0x03, 0x6e, 0xca, 0x8b, 0x92, 0x67, 0x31, 0x52, 0x47,
- 0xf3, 0x3d, 0x0e, 0xae, 0x60, 0xb0, 0xcc, 0xb3, 0x9d, 0x88, 0x89, 0x0f, 0xd6, 0x0f, 0x7c, 0xd4,
- 0xde, 0x1e, 0x53, 0xa5, 0x72, 0x7e, 0xe0, 0x49, 0x8d, 0xda, 0xd9, 0x63, 0x0d, 0x08, 0x16, 0x30,
- 0x58, 0xe1, 0x83, 0x08, 0x51, 0x7b, 0xf1, 0x14, 0xdb, 0x4f, 0x74, 0x4d, 0xce, 0x60, 0x10, 0xea,
- 0x79, 0xd4, 0x9c, 0x58, 0xd3, 0xe1, 0x7c, 0x34, 0x53, 0x7b, 0xce, 0x1a, 0x0f, 0xd6, 0xf6, 0x82,
- 0x3f, 0x06, 0xb8, 0xdb, 0x8c, 0x17, 0x72, 0x9f, 0x57, 0xff, 0x1c, 0x33, 0x83, 0x61, 0x92, 0x87,
- 0x3c, 0x59, 0xfe, 0x7f, 0xd6, 0xa1, 0x40, 0xad, 0x58, 0x94, 0xf9, 0x4e, 0x24, 0x28, 0xa9, 0x35,
- 0xb1, 0xa6, 0x1e, 0xeb, 0x31, 0x79, 0x0b, 0x1e, 0x16, 0x7b, 0x4c, 0xb1, 0xe4, 0x89, 0xce, 0xc5,
- 0x65, 0x4f, 0x04, 0xb9, 0x82, 0x91, 0x1e, 0xd4, 0xec, 0x24, 0xa9, 0x73, 0x68, 0xd5, 0x90, 0xec,
- 0x48, 0x41, 0x02, 0x18, 0xf1, 0x32, 0xdc, 0x8b, 0x0a, 0xc3, 0xaa, 0x2e, 0x91, 0x0e, 0x74, 0xa4,
- 0x47, 0x9c, 0xfa, 0x1f, 0x59, 0xf1, 0x0a, 0x77, 0x75, 0x42, 0x4f, 0xb4, 0x65, 0x8f, 0x83, 0xdf,
- 0x06, 0x9c, 0xde, 0xea, 0x5b, 0x10, 0x79, 0x76, 0x8d, 0x3c, 0xc2, 0x92, 0x9c, 0x83, 0xb9, 0x93,
- 0x3a, 0x81, 0xf1, 0xfc, 0x55, 0xe3, 0xdd, 0x4b, 0xd6, 0x5b, 0x75, 0x1d, 0xcc, 0xdc, 0x29, 0x6b,
- 0x3b, 0x2c, 0x45, 0x4d, 0xcd, 0x89, 0x31, 0x1d, 0xcf, 0xc7, 0x6d, 0x1e, 0xec, 0xe6, 0x5e, 0x2b,
- 0x74, 0x8f, 0x9c, 0x83, 0x23, 0xa2, 0x94, 0x17, 0x3a, 0x87, 0xe1, 0xfc, 0xb4, 0x11, 0xf5, 0x57,
- 0xc6, 0x9a, 0x2e, 0x39, 0x83, 0xe7, 0xb2, 0x7d, 0x81, 0x0d, 0x4f, 0x51, 0x52, 0x5b, 0xc7, 0x76,
- 0x4c, 0x92, 0x77, 0xe0, 0x75, 0x44, 0x17, 0x4d, 0xeb, 0xda, 0x3d, 0x1f, 0x7b, 0x12, 0x04, 0x6b,
- 0xf0, 0xfb, 0xbf, 0x5e, 0xe6, 0x59, 0x55, 0xe6, 0x09, 0xa1, 0x70, 0x22, 0xeb, 0x30, 0x44, 0x29,
- 0xdb, 0xb3, 0xee, 0xa0, 0xea, 0xa4, 0x28, 0x25, 0x8f, 0x51, 0xef, 0xe3, 0xb1, 0x0e, 0x5e, 0xbc,
- 0x3f, 0x08, 0xa8, 0xd9, 0x9e, 0x78, 0xe0, 0xb0, 0xed, 0xd7, 0xcd, 0xd2, 0x7f, 0xa6, 0xca, 0xc5,
- 0x1d, 0x5b, 0x6f, 0x7d, 0x83, 0x9c, 0x80, 0xf5, 0x6d, 0xbd, 0xf5, 0x4d, 0x55, 0xb0, 0xc5, 0xca,
- 0xb7, 0x2e, 0x2e, 0xc1, 0xed, 0xf2, 0x20, 0x63, 0x00, 0x55, 0x7f, 0x3f, 0xf8, 0xf0, 0xcb, 0xf5,
- 0x87, 0xfb, 0x4f, 0xbe, 0x41, 0x5c, 0xb0, 0x37, 0x9f, 0x37, 0x1f, 0x7d, 0xf3, 0x6f, 0x00, 0x00,
- 0x00, 0xff, 0xff, 0xe4, 0xf2, 0xc3, 0xd0, 0x9b, 0x03, 0x00, 0x00,
+ 0xec, 0x8d, 0xb3, 0xc2, 0x5f, 0xda, 0xb5, 0x2b, 0x7a, 0xe2, 0x37, 0x73, 0xe5, 0x84, 0x76, 0xfd,
+ 0xd1, 0x44, 0x82, 0xdb, 0xbc, 0x37, 0xcf, 0xf3, 0x3c, 0x6f, 0x07, 0x5e, 0x64, 0x3f, 0x93, 0xcb,
+ 0x9c, 0xa7, 0x82, 0xd6, 0x6c, 0x56, 0x89, 0xb2, 0x2e, 0x91, 0x9d, 0x53, 0x5e, 0x84, 0xbf, 0xc0,
+ 0xbb, 0x59, 0xdd, 0xd2, 0xea, 0xee, 0xb1, 0x62, 0xe8, 0x25, 0x38, 0x5c, 0x36, 0x3c, 0xc1, 0x46,
+ 0x60, 0x46, 0x2e, 0x69, 0x41, 0xcb, 0xa6, 0x3c, 0xc1, 0x66, 0xcf, 0xa6, 0x3c, 0x41, 0xaf, 0x61,
+ 0xb4, 0x2f, 0x65, 0xcd, 0x13, 0x6c, 0x05, 0x66, 0xe4, 0x90, 0x0e, 0x21, 0x04, 0x76, 0x21, 0x79,
+ 0x82, 0x6d, 0xcd, 0xea, 0x1a, 0xbd, 0x01, 0x37, 0xa7, 0x95, 0xa0, 0x45, 0xca, 0xb0, 0xa3, 0xf9,
+ 0x01, 0x87, 0x57, 0x30, 0x5a, 0x96, 0xc5, 0x8e, 0xa7, 0xc8, 0x07, 0xeb, 0x07, 0x7b, 0xd4, 0xde,
+ 0x1e, 0x51, 0xa5, 0x72, 0x7e, 0xa0, 0x59, 0xc3, 0xb4, 0xb3, 0x47, 0x5a, 0x10, 0x2e, 0x60, 0xb4,
+ 0x62, 0x0f, 0x3c, 0x66, 0xda, 0x8b, 0xe6, 0xac, 0xfb, 0x44, 0xd7, 0xe8, 0x0c, 0x46, 0xb1, 0x9e,
+ 0x87, 0xcd, 0xc0, 0x8a, 0x4e, 0xe6, 0x93, 0x99, 0xda, 0x73, 0xd6, 0x7a, 0x90, 0xae, 0x17, 0xfe,
+ 0x31, 0xc0, 0xdd, 0x16, 0xb4, 0x92, 0xfb, 0xb2, 0xfe, 0xe7, 0x98, 0x19, 0x9c, 0x64, 0x65, 0x4c,
+ 0xb3, 0xe5, 0xff, 0x67, 0x1d, 0x0a, 0xd4, 0x8a, 0x95, 0x28, 0x77, 0x3c, 0x63, 0x12, 0x5b, 0x81,
+ 0x15, 0x79, 0x64, 0xc0, 0xe8, 0x2d, 0x78, 0xac, 0xda, 0xb3, 0x9c, 0x09, 0x9a, 0xe9, 0x5c, 0x5c,
+ 0xf2, 0x44, 0xa0, 0x2b, 0x98, 0xe8, 0x41, 0xed, 0x4e, 0x12, 0x3b, 0x87, 0x56, 0x2d, 0x49, 0x8e,
+ 0x14, 0x28, 0x84, 0x09, 0x15, 0xf1, 0x9e, 0xd7, 0x2c, 0xae, 0x1b, 0xc1, 0xf0, 0x48, 0x47, 0x7a,
+ 0xc4, 0xa9, 0xff, 0x91, 0x35, 0xad, 0xd9, 0xae, 0xc9, 0xf0, 0x58, 0x5b, 0x0e, 0x38, 0xfc, 0x6d,
+ 0xc0, 0xe9, 0xad, 0xbe, 0x05, 0x5e, 0x16, 0xd7, 0x8c, 0x26, 0x4c, 0xa0, 0x73, 0x30, 0x77, 0x52,
+ 0x27, 0x30, 0x9d, 0xbf, 0x6a, 0xbd, 0x07, 0xc9, 0x7a, 0xab, 0xae, 0x83, 0x98, 0x3b, 0x65, 0x6d,
+ 0xc7, 0x82, 0x37, 0xd8, 0x0c, 0x8c, 0x68, 0x3a, 0x9f, 0x76, 0x79, 0x90, 0x9b, 0x7b, 0xad, 0xd0,
+ 0x3d, 0x74, 0x0e, 0x0e, 0x4f, 0x72, 0x5a, 0xe9, 0x1c, 0x4e, 0xe6, 0xa7, 0xad, 0x68, 0xb8, 0x32,
+ 0xd2, 0x76, 0xd1, 0x19, 0x3c, 0x97, 0xdd, 0x0b, 0x6c, 0x68, 0xce, 0x24, 0xb6, 0x75, 0x6c, 0xc7,
+ 0x24, 0x7a, 0x07, 0x5e, 0x4f, 0xf4, 0xd1, 0x74, 0xae, 0xfd, 0xf3, 0x91, 0x27, 0x01, 0xc2, 0x30,
+ 0xae, 0x04, 0x4b, 0x9a, 0xbc, 0xc2, 0xe3, 0xc0, 0x88, 0x5c, 0xd2, 0xc3, 0x70, 0x0d, 0xfe, 0xb0,
+ 0xcf, 0xb2, 0x2c, 0x6a, 0x51, 0x66, 0x4a, 0x2d, 0x9b, 0x38, 0x66, 0x52, 0x76, 0x07, 0xdf, 0x43,
+ 0xd5, 0xc9, 0x99, 0x94, 0x34, 0x65, 0x7a, 0x53, 0x8f, 0xf4, 0xf0, 0xe2, 0xfd, 0x41, 0x74, 0x6d,
+ 0x2e, 0xc8, 0x03, 0x87, 0x6c, 0xbf, 0x6e, 0x96, 0xfe, 0x33, 0x55, 0x2e, 0xee, 0xc8, 0x7a, 0xeb,
+ 0x1b, 0x68, 0x0c, 0xd6, 0xb7, 0xf5, 0xd6, 0x37, 0x55, 0x41, 0x16, 0x2b, 0xdf, 0xba, 0xb8, 0x04,
+ 0xb7, 0x4f, 0x0a, 0x4d, 0x01, 0x54, 0xfd, 0xfd, 0xe0, 0xc3, 0x2f, 0xd7, 0x1f, 0xee, 0x3f, 0xf9,
+ 0x06, 0x72, 0xc1, 0xde, 0x7c, 0xde, 0x7c, 0xf4, 0xcd, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x69,
+ 0xa6, 0x98, 0x6c, 0xb5, 0x03, 0x00, 0x00,
}
diff --git a/lxd/migrate.proto b/lxd/migrate.proto
index 172a940be..c81601241 100644
--- a/lxd/migrate.proto
+++ b/lxd/migrate.proto
@@ -47,6 +47,7 @@ message MigrationHeader {
repeated IDMapType idmap = 3;
repeated string snapshotNames = 4;
repeated Snapshot snapshots = 5;
+ optional bool predump = 7;
}
message MigrationControl {
diff --git a/shared/container.go b/shared/container.go
index 9b78c377d..b184a0437 100644
--- a/shared/container.go
+++ b/shared/container.go
@@ -167,6 +167,8 @@ var KnownContainerConfigKeys = map[string]func(value string) error{
"linux.kernel_modules": IsAny,
+ "migration.pre_copy.enabled": IsBool,
+
"security.nesting": IsBool,
"security.privileged": IsBool,
From b72e90cb5a4e907ff4b640003ad248beb20f2119 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Thu, 30 Nov 2017 17:45:05 +0100
Subject: [PATCH 3/9] migrate: prepare for pre-copy migration (part 3)
This introduces a protocol for the rsync transfers to support a
variable number of pre-dumps. The number of pre-dumps depends on the
workload, the threshold and the number of pre-dumps.
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/migrate.go | 41 +++++++++++++++++++++++++
lxd/migrate.pb.go | 90 +++++++++++++++++++++++++++++++++----------------------
lxd/migrate.proto | 4 +++
3 files changed, 100 insertions(+), 35 deletions(-)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index 8cb11dcdb..2de7a154b 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -1006,6 +1006,47 @@ func (c *migrationSink) Do(migrateOp *operation) error {
criuConn = c.src.criuConn
}
+ sync := &MigrationSync{
+ // temporarily set to 'true' as long as not used
+ FinalPreDump: proto.Bool(true),
+ }
+
+ if resp.GetPredump() {
+ logger.Debugf("Before the receive loop %s", sync.GetFinalPreDump())
+ for !sync.GetFinalPreDump() {
+ logger.Debugf("About to receive rsync")
+ // Transfer a CRIU pre-dump
+ err = RsyncRecv(shared.AddSlash(imagesDir), criuConn, nil)
+ if err != nil {
+ restore <- err
+ return
+ }
+ logger.Debugf("rsync receive done")
+
+ logger.Debugf("About to receive header")
+ // Check if this was the last pre-dump
+ // Only the FinalPreDump element if of interest
+ mtype, data, err := criuConn.ReadMessage()
+ if err != nil {
+ logger.Debugf("err %s", err)
+ restore <- err
+ return
+ }
+ if mtype != websocket.BinaryMessage {
+ restore <- err
+ return
+ }
+ err = proto.Unmarshal(data, sync)
+ if err != nil {
+ logger.Debugf("err %s", err)
+ restore <- err
+ return
+ }
+ logger.Debugf("At the end of the receive loop %s", sync.GetFinalPreDump())
+ }
+ }
+
+ // CRIU dump
err = RsyncRecv(shared.AddSlash(imagesDir), criuConn, nil)
if err != nil {
restore <- err
diff --git a/lxd/migrate.pb.go b/lxd/migrate.pb.go
index a67c4bf3e..348db0fe1 100644
--- a/lxd/migrate.pb.go
+++ b/lxd/migrate.pb.go
@@ -15,6 +15,7 @@ It has these top-level messages:
Snapshot
MigrationHeader
MigrationControl
+ MigrationSync
*/
package main
@@ -357,6 +358,23 @@ func (m *MigrationControl) GetMessage() string {
return ""
}
+type MigrationSync struct {
+ FinalPreDump *bool `protobuf:"varint,1,req,name=finalPreDump" json:"finalPreDump,omitempty"`
+ XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *MigrationSync) Reset() { *m = MigrationSync{} }
+func (m *MigrationSync) String() string { return proto.CompactTextString(m) }
+func (*MigrationSync) ProtoMessage() {}
+func (*MigrationSync) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *MigrationSync) GetFinalPreDump() bool {
+ if m != nil && m.FinalPreDump != nil {
+ return *m.FinalPreDump
+ }
+ return false
+}
+
func init() {
proto.RegisterType((*IDMapType)(nil), "main.IDMapType")
proto.RegisterType((*Config)(nil), "main.Config")
@@ -364,6 +382,7 @@ func init() {
proto.RegisterType((*Snapshot)(nil), "main.Snapshot")
proto.RegisterType((*MigrationHeader)(nil), "main.MigrationHeader")
proto.RegisterType((*MigrationControl)(nil), "main.MigrationControl")
+ proto.RegisterType((*MigrationSync)(nil), "main.MigrationSync")
proto.RegisterEnum("main.MigrationFSType", MigrationFSType_name, MigrationFSType_value)
proto.RegisterEnum("main.CRIUType", CRIUType_name, CRIUType_value)
}
@@ -371,39 +390,40 @@ func init() {
func init() { proto.RegisterFile("lxd/migrate.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 535 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0x4d, 0x6f, 0xd3, 0x40,
- 0x10, 0xc5, 0x5f, 0x89, 0x3d, 0x0d, 0xa9, 0x59, 0x01, 0x5a, 0x21, 0x0e, 0x96, 0xd5, 0x4a, 0x56,
- 0x85, 0xd2, 0x2a, 0x37, 0x8e, 0x24, 0x21, 0x6a, 0x25, 0x1a, 0xd0, 0xa6, 0x3d, 0xc0, 0x05, 0xad,
- 0xec, 0x8d, 0xb3, 0xc2, 0x5f, 0xda, 0xb5, 0x2b, 0x7a, 0xe2, 0x37, 0x73, 0xe5, 0x84, 0x76, 0xfd,
- 0xd1, 0x44, 0x82, 0xdb, 0xbc, 0x37, 0xcf, 0xf3, 0x3c, 0x6f, 0x07, 0x5e, 0x64, 0x3f, 0x93, 0xcb,
- 0x9c, 0xa7, 0x82, 0xd6, 0x6c, 0x56, 0x89, 0xb2, 0x2e, 0x91, 0x9d, 0x53, 0x5e, 0x84, 0xbf, 0xc0,
- 0xbb, 0x59, 0xdd, 0xd2, 0xea, 0xee, 0xb1, 0x62, 0xe8, 0x25, 0x38, 0x5c, 0x36, 0x3c, 0xc1, 0x46,
- 0x60, 0x46, 0x2e, 0x69, 0x41, 0xcb, 0xa6, 0x3c, 0xc1, 0x66, 0xcf, 0xa6, 0x3c, 0x41, 0xaf, 0x61,
- 0xb4, 0x2f, 0x65, 0xcd, 0x13, 0x6c, 0x05, 0x66, 0xe4, 0x90, 0x0e, 0x21, 0x04, 0x76, 0x21, 0x79,
- 0x82, 0x6d, 0xcd, 0xea, 0x1a, 0xbd, 0x01, 0x37, 0xa7, 0x95, 0xa0, 0x45, 0xca, 0xb0, 0xa3, 0xf9,
- 0x01, 0x87, 0x57, 0x30, 0x5a, 0x96, 0xc5, 0x8e, 0xa7, 0xc8, 0x07, 0xeb, 0x07, 0x7b, 0xd4, 0xde,
- 0x1e, 0x51, 0xa5, 0x72, 0x7e, 0xa0, 0x59, 0xc3, 0xb4, 0xb3, 0x47, 0x5a, 0x10, 0x2e, 0x60, 0xb4,
- 0x62, 0x0f, 0x3c, 0x66, 0xda, 0x8b, 0xe6, 0xac, 0xfb, 0x44, 0xd7, 0xe8, 0x0c, 0x46, 0xb1, 0x9e,
- 0x87, 0xcd, 0xc0, 0x8a, 0x4e, 0xe6, 0x93, 0x99, 0xda, 0x73, 0xd6, 0x7a, 0x90, 0xae, 0x17, 0xfe,
- 0x31, 0xc0, 0xdd, 0x16, 0xb4, 0x92, 0xfb, 0xb2, 0xfe, 0xe7, 0x98, 0x19, 0x9c, 0x64, 0x65, 0x4c,
- 0xb3, 0xe5, 0xff, 0x67, 0x1d, 0x0a, 0xd4, 0x8a, 0x95, 0x28, 0x77, 0x3c, 0x63, 0x12, 0x5b, 0x81,
- 0x15, 0x79, 0x64, 0xc0, 0xe8, 0x2d, 0x78, 0xac, 0xda, 0xb3, 0x9c, 0x09, 0x9a, 0xe9, 0x5c, 0x5c,
- 0xf2, 0x44, 0xa0, 0x2b, 0x98, 0xe8, 0x41, 0xed, 0x4e, 0x12, 0x3b, 0x87, 0x56, 0x2d, 0x49, 0x8e,
- 0x14, 0x28, 0x84, 0x09, 0x15, 0xf1, 0x9e, 0xd7, 0x2c, 0xae, 0x1b, 0xc1, 0xf0, 0x48, 0x47, 0x7a,
- 0xc4, 0xa9, 0xff, 0x91, 0x35, 0xad, 0xd9, 0xae, 0xc9, 0xf0, 0x58, 0x5b, 0x0e, 0x38, 0xfc, 0x6d,
- 0xc0, 0xe9, 0xad, 0xbe, 0x05, 0x5e, 0x16, 0xd7, 0x8c, 0x26, 0x4c, 0xa0, 0x73, 0x30, 0x77, 0x52,
- 0x27, 0x30, 0x9d, 0xbf, 0x6a, 0xbd, 0x07, 0xc9, 0x7a, 0xab, 0xae, 0x83, 0x98, 0x3b, 0x65, 0x6d,
- 0xc7, 0x82, 0x37, 0xd8, 0x0c, 0x8c, 0x68, 0x3a, 0x9f, 0x76, 0x79, 0x90, 0x9b, 0x7b, 0xad, 0xd0,
- 0x3d, 0x74, 0x0e, 0x0e, 0x4f, 0x72, 0x5a, 0xe9, 0x1c, 0x4e, 0xe6, 0xa7, 0xad, 0x68, 0xb8, 0x32,
- 0xd2, 0x76, 0xd1, 0x19, 0x3c, 0x97, 0xdd, 0x0b, 0x6c, 0x68, 0xce, 0x24, 0xb6, 0x75, 0x6c, 0xc7,
- 0x24, 0x7a, 0x07, 0x5e, 0x4f, 0xf4, 0xd1, 0x74, 0xae, 0xfd, 0xf3, 0x91, 0x27, 0x01, 0xc2, 0x30,
- 0xae, 0x04, 0x4b, 0x9a, 0xbc, 0xc2, 0xe3, 0xc0, 0x88, 0x5c, 0xd2, 0xc3, 0x70, 0x0d, 0xfe, 0xb0,
- 0xcf, 0xb2, 0x2c, 0x6a, 0x51, 0x66, 0x4a, 0x2d, 0x9b, 0x38, 0x66, 0x52, 0x76, 0x07, 0xdf, 0x43,
- 0xd5, 0xc9, 0x99, 0x94, 0x34, 0x65, 0x7a, 0x53, 0x8f, 0xf4, 0xf0, 0xe2, 0xfd, 0x41, 0x74, 0x6d,
- 0x2e, 0xc8, 0x03, 0x87, 0x6c, 0xbf, 0x6e, 0x96, 0xfe, 0x33, 0x55, 0x2e, 0xee, 0xc8, 0x7a, 0xeb,
- 0x1b, 0x68, 0x0c, 0xd6, 0xb7, 0xf5, 0xd6, 0x37, 0x55, 0x41, 0x16, 0x2b, 0xdf, 0xba, 0xb8, 0x04,
- 0xb7, 0x4f, 0x0a, 0x4d, 0x01, 0x54, 0xfd, 0xfd, 0xe0, 0xc3, 0x2f, 0xd7, 0x1f, 0xee, 0x3f, 0xf9,
- 0x06, 0x72, 0xc1, 0xde, 0x7c, 0xde, 0x7c, 0xf4, 0xcd, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x69,
- 0xa6, 0x98, 0x6c, 0xb5, 0x03, 0x00, 0x00,
+ // 560 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xcd, 0x6b, 0xdb, 0x4e,
+ 0x10, 0xfd, 0x49, 0xf2, 0x87, 0x34, 0x49, 0x1c, 0xfd, 0x96, 0xb6, 0x2c, 0xa5, 0x07, 0x21, 0x12,
+ 0x10, 0xa1, 0x38, 0xc1, 0x3d, 0xf5, 0xd8, 0xd8, 0x35, 0x09, 0x34, 0x6e, 0x58, 0x25, 0x87, 0xf6,
+ 0x52, 0x16, 0x69, 0x25, 0x2f, 0xd5, 0x17, 0xbb, 0x52, 0xa8, 0x4f, 0xfd, 0x9b, 0x7b, 0xed, 0xa9,
+ 0xec, 0xea, 0x23, 0x36, 0xb4, 0xb7, 0x79, 0x6f, 0x9e, 0xe6, 0x69, 0xde, 0x0e, 0xfc, 0x9f, 0xfd,
+ 0x88, 0x2f, 0x73, 0x9e, 0x0a, 0x5a, 0xb3, 0x79, 0x25, 0xca, 0xba, 0x44, 0xa3, 0x9c, 0xf2, 0xc2,
+ 0xff, 0x09, 0xce, 0xed, 0xea, 0x8e, 0x56, 0x0f, 0xbb, 0x8a, 0xa1, 0x17, 0x30, 0xe6, 0xb2, 0xe1,
+ 0x31, 0x36, 0x3c, 0x33, 0xb0, 0x49, 0x0b, 0x5a, 0x36, 0xe5, 0x31, 0x36, 0x7b, 0x36, 0xe5, 0x31,
+ 0x7a, 0x05, 0x93, 0x6d, 0x29, 0x6b, 0x1e, 0x63, 0xcb, 0x33, 0x83, 0x31, 0xe9, 0x10, 0x42, 0x30,
+ 0x2a, 0x24, 0x8f, 0xf1, 0x48, 0xb3, 0xba, 0x46, 0xaf, 0xc1, 0xce, 0x69, 0x25, 0x68, 0x91, 0x32,
+ 0x3c, 0xd6, 0xfc, 0x80, 0xfd, 0x2b, 0x98, 0x2c, 0xcb, 0x22, 0xe1, 0x29, 0x72, 0xc1, 0xfa, 0xce,
+ 0x76, 0xda, 0xdb, 0x21, 0xaa, 0x54, 0xce, 0x4f, 0x34, 0x6b, 0x98, 0x76, 0x76, 0x48, 0x0b, 0xfc,
+ 0x6b, 0x98, 0xac, 0xd8, 0x13, 0x8f, 0x98, 0xf6, 0xa2, 0x39, 0xeb, 0x3e, 0xd1, 0x35, 0x3a, 0x83,
+ 0x49, 0xa4, 0xe7, 0x61, 0xd3, 0xb3, 0x82, 0xa3, 0xc5, 0xf1, 0x5c, 0xed, 0x39, 0x6f, 0x3d, 0x48,
+ 0xd7, 0xf3, 0x7f, 0x1b, 0x60, 0x87, 0x05, 0xad, 0xe4, 0xb6, 0xac, 0xff, 0x3a, 0x66, 0x0e, 0x47,
+ 0x59, 0x19, 0xd1, 0x6c, 0xf9, 0xef, 0x59, 0xfb, 0x02, 0xb5, 0x62, 0x25, 0xca, 0x84, 0x67, 0x4c,
+ 0x62, 0xcb, 0xb3, 0x02, 0x87, 0x0c, 0x18, 0xbd, 0x01, 0x87, 0x55, 0x5b, 0x96, 0x33, 0x41, 0x33,
+ 0x9d, 0x8b, 0x4d, 0x9e, 0x09, 0x74, 0x05, 0xc7, 0x7a, 0x50, 0xbb, 0x93, 0xc4, 0xe3, 0x7d, 0xab,
+ 0x96, 0x24, 0x07, 0x0a, 0xe4, 0xc3, 0x31, 0x15, 0xd1, 0x96, 0xd7, 0x2c, 0xaa, 0x1b, 0xc1, 0xf0,
+ 0x44, 0x47, 0x7a, 0xc0, 0xa9, 0xff, 0x91, 0x35, 0xad, 0x59, 0xd2, 0x64, 0x78, 0xaa, 0x2d, 0x07,
+ 0xec, 0xff, 0x32, 0xe0, 0xf4, 0x4e, 0xdf, 0x02, 0x2f, 0x8b, 0x1b, 0x46, 0x63, 0x26, 0xd0, 0x39,
+ 0x98, 0x89, 0xd4, 0x09, 0xcc, 0x16, 0x2f, 0x5b, 0xef, 0x41, 0xb2, 0x0e, 0xd5, 0x75, 0x10, 0x33,
+ 0x51, 0xd6, 0xa3, 0x48, 0xf0, 0x06, 0x9b, 0x9e, 0x11, 0xcc, 0x16, 0xb3, 0x2e, 0x0f, 0x72, 0xfb,
+ 0xa8, 0x15, 0xba, 0x87, 0xce, 0x61, 0xcc, 0xe3, 0x9c, 0x56, 0x3a, 0x87, 0xa3, 0xc5, 0x69, 0x2b,
+ 0x1a, 0xae, 0x8c, 0xb4, 0x5d, 0x74, 0x06, 0x27, 0xb2, 0x7b, 0x81, 0x0d, 0xcd, 0x99, 0xc4, 0x23,
+ 0x1d, 0xdb, 0x21, 0x89, 0xde, 0x82, 0xd3, 0x13, 0x7d, 0x34, 0x9d, 0x6b, 0xff, 0x7c, 0xe4, 0x59,
+ 0x80, 0x30, 0x4c, 0x2b, 0xc1, 0xe2, 0x26, 0xaf, 0xf0, 0xd4, 0x33, 0x02, 0x9b, 0xf4, 0xd0, 0x5f,
+ 0x83, 0x3b, 0xec, 0xb3, 0x2c, 0x8b, 0x5a, 0x94, 0x99, 0x52, 0xcb, 0x26, 0x8a, 0x98, 0x94, 0xdd,
+ 0xc1, 0xf7, 0x50, 0x75, 0x72, 0x26, 0x25, 0x4d, 0x99, 0xde, 0xd4, 0x21, 0x3d, 0xf4, 0xdf, 0xc1,
+ 0xc9, 0x30, 0x27, 0xdc, 0x15, 0x91, 0x7a, 0x8c, 0x84, 0x17, 0x34, 0xbb, 0x17, 0x6c, 0xa5, 0x7c,
+ 0xdb, 0x49, 0x07, 0xdc, 0xc5, 0xfb, 0xbd, 0xbc, 0xdb, 0x30, 0x91, 0x03, 0x63, 0x12, 0x7e, 0xd9,
+ 0x2c, 0xdd, 0xff, 0x54, 0x79, 0xfd, 0x40, 0xd6, 0xa1, 0x6b, 0xa0, 0x29, 0x58, 0x5f, 0xd7, 0xa1,
+ 0x6b, 0xaa, 0x82, 0x5c, 0xaf, 0x5c, 0xeb, 0xe2, 0x12, 0xec, 0x3e, 0x5e, 0x34, 0x03, 0x50, 0xf5,
+ 0xb7, 0xbd, 0x0f, 0xef, 0x6f, 0x3e, 0x3c, 0x7e, 0x72, 0x0d, 0x64, 0xc3, 0x68, 0xf3, 0x79, 0xf3,
+ 0xd1, 0x35, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x5a, 0xe1, 0x62, 0xc7, 0xea, 0x03, 0x00, 0x00,
}
diff --git a/lxd/migrate.proto b/lxd/migrate.proto
index c81601241..f1667c678 100644
--- a/lxd/migrate.proto
+++ b/lxd/migrate.proto
@@ -56,3 +56,7 @@ message MigrationControl {
/* optional failure message if sending a failure */
optional string message = 2;
}
+
+message MigrationSync {
+ required bool finalPreDump = 1;
+}
From 6eb2a5eb9f98de8e90a5e56354eb1e71b5bb5970 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Thu, 30 Nov 2017 19:57:37 +0000
Subject: [PATCH 4/9] migrate: add option to specify max pre-dump iterations
This adds the configuration option to specify the maximum number of
pre-dump iterations:
migration.pre_copy.max (defaults to 10 right now)
If the remaining memory pages are not below (or above) the not-yet-existing
threshold the migration will end by doing one final dump and transfer
before restoring the container on the destination.
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/migrate.go | 22 ++++++++++++++++++++++
shared/container.go | 1 +
2 files changed, 23 insertions(+)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index 2de7a154b..48c48f47e 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -15,6 +15,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "strconv"
"strings"
"sync"
@@ -429,6 +430,27 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
}
logger.Debugf("migration.pre_copy.enabled %s", use_pre_dumps)
+ // migration.pre_copy.max is the value after which the container
+ // will be definetely migrated, even if the remaining number of
+ // memory pages is below the defined threshold.
+ // TODO: implement threshold (needs reading of CRIU output files)
+ var max_iterations int
+ tmp = s.container.ExpandedConfig()["migration.pre_copy.max"]
+ if tmp != "" {
+ max_iterations, _ = strconv.Atoi(tmp)
+ } else {
+ // default to 10
+ max_iterations = 10
+ }
+ if max_iterations > 999 {
+ // the pre-dump directory is hardcoded to a string
+ // with maximal 3 digits. 999 pre-dumps makes no
+ // sense at all, but let's make sure the number
+ // is not higher than this.
+ max_iterations = 999
+ }
+ logger.Debugf("using maximal %d iterations for pre-dumping", max_iterations)
+
// The protocol says we have to send a header no matter what, so let's
// do that, but then immediately send an error.
myType := s.container.Storage().MigrationType()
diff --git a/shared/container.go b/shared/container.go
index b184a0437..de2fd9fb1 100644
--- a/shared/container.go
+++ b/shared/container.go
@@ -168,6 +168,7 @@ var KnownContainerConfigKeys = map[string]func(value string) error{
"linux.kernel_modules": IsAny,
"migration.pre_copy.enabled": IsBool,
+ "migration.pre_copy.max": IsUint32,
"security.nesting": IsBool,
"security.privileged": IsBool,
From 7b4ec6aedd1a0293d8040c8374ed3915359a937c Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Thu, 30 Nov 2017 10:11:58 +0100
Subject: [PATCH 5/9] migrate.proto: silence protobuf compiler warning
The protobuf compiler complains that it defaults to 'proto2':
[libprotobuf WARNING google/protobuf/compiler/parser.cc:546]
No syntax specified for the proto file: lxd/migrate.proto.
Please use 'syntax = "proto2";' or 'syntax = "proto3";'
to specify a syntax version. (Defaulted to proto2 syntax.)
This just explicitly sets the default.
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/migrate.proto | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lxd/migrate.proto b/lxd/migrate.proto
index f1667c678..3758c949e 100644
--- a/lxd/migrate.proto
+++ b/lxd/migrate.proto
@@ -1,3 +1,6 @@
+// silence the protobuf compiler warning by setting the default
+syntax = "proto2";
+
package main;
enum MigrationFSType {
From 01ee386cb7173d1bd407017cca6a335b3b9908a1 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Thu, 30 Nov 2017 10:20:42 +0100
Subject: [PATCH 6/9] migrate: older than lxc 2.0.4 will fail
Testing the older than liblxc 2.0.4 code paths does not seem to work
anymore. This adds a debug message for that code path:
liblxc version is older than 2.0.4 and the live migration will probably fail
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/migrate.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index 48c48f47e..31b0d2137 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -633,6 +633,7 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
logger.Debugf("Dump finished, continuing with restore...")
}
} else {
+ logger.Debugf("liblxc version is older than 2.0.4 and the live migration will probably fail")
defer os.RemoveAll(checkpointDir)
criuMigrationArgs := CriuMigrationArgs{
cmd: lxc.MIGRATE_DUMP,
From fa386e30783d12e48a89aa64d9050068f50eedf1 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at ubuntu.com>
Date: Mon, 27 Nov 2017 22:50:38 +0100
Subject: [PATCH 7/9] migrate: implement pre-copy migration
This uses the CRIU's pre-dump support to implement pre-copy migration in
LXD. To implement this on the CRIU side, CRIU uses the soft-dirty bit in
the pagemap:
https://www.kernel.org/doc/Documentation/vm/soft-dirty.txt
https://criu.org/Memory_changes_tracking
As long as there is no feature detection in LXC the pre-copy migration
needs to be enabled explicitly as not all architecture/kernel/criu
combinations support dirty pages tracking.
To enable pre-copy migration in LXD the following parameter needs to be set
to 'true':
pre_copy_migration.enabled
Currently it defaults to 'false'. As LXD does not yet know how at which
point it has done enough pre-copying the following parameter exists:
pre_copy_migration.max_iterations
This defaults to '10'. This means that LXD will do 10 pre-dumps before
doing the final dump.
Thanks to protobuf the whole pre-copy migration should be transparent
enough, that if the receiving side does not know anything about pre-copy
migration (and especially the additional rsync transfers) it just does
the non-optimized migration even if 'pre_copy_migration.enabled' is set
to 'true'.
The following things will be improved in follow up commits:
* read out CRIU's 'stats-dump' file to know how many pages are
part of the previous dump
* make pre-copy migration the default if LXC tells us the
architecture/kernel/criu combination supports it
* update documentation
* update test cases
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/container.go | 2 +
lxd/container_lxc.go | 29 ++++++++++++--
lxd/migrate.go | 109 ++++++++++++++++++++++++++++++++++++++++++++++++---
3 files changed, 131 insertions(+), 9 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index ef577e6d2..44456dbe6 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -702,6 +702,8 @@ func containerCreateAsSnapshot(s *state.State, args db.ContainerArgs, sourceCont
function: "snapshot",
stop: false,
actionScript: false,
+ dumpDir: "",
+ preDumpDir: "",
}
err = sourceContainer.Migrate(&criuMigrationArgs)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index da1d2c492..69b262911 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -2159,6 +2159,8 @@ func (c *containerLXC) Start(stateful bool) error {
function: "snapshot",
stop: false,
actionScript: false,
+ dumpDir: "",
+ preDumpDir: "",
}
err := c.Migrate(&criuMigrationArgs)
@@ -2384,6 +2386,8 @@ func (c *containerLXC) Stop(stateful bool) error {
function: "snapshot",
stop: true,
actionScript: false,
+ dumpDir: "",
+ preDumpDir: "",
}
// Checkpoint
@@ -2900,6 +2904,8 @@ func (c *containerLXC) Restore(sourceContainer container, stateful bool) error {
function: "snapshot",
stop: false,
actionScript: false,
+ dumpDir: "",
+ preDumpDir: "",
}
// Checkpoint
@@ -4500,6 +4506,8 @@ type CriuMigrationArgs struct {
function string
stop bool
actionScript bool
+ dumpDir string
+ preDumpDir string
}
func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
@@ -4509,6 +4517,7 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
"used": c.lastUsedDate,
"statedir": args.stateDir,
"actionscript": args.actionScript,
+ "predumpdir": args.preDumpDir,
"stop": args.stop}
_, err := exec.LookPath("criu")
@@ -4545,6 +4554,7 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
preservesInodes = false
}
+ finalStateDir := args.stateDir
var migrateErr error
/* For restore, we need an extra fork so that we daemonize monitor
@@ -4590,6 +4600,10 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
configPath := filepath.Join(c.LogPath(), "lxc.conf")
+ if args.dumpDir != "" {
+ finalStateDir = fmt.Sprintf("%s/%s", args.stateDir, args.dumpDir)
+ }
+
var out string
out, migrateErr = shared.RunCommand(
c.state.OS.ExecPath,
@@ -4597,7 +4611,7 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
c.name,
c.state.OS.LxcPath,
configPath,
- args.stateDir,
+ finalStateDir,
fmt.Sprintf("%v", preservesInodes))
if out != "" {
@@ -4617,6 +4631,10 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
script = filepath.Join(args.stateDir, "action.sh")
}
+ if args.dumpDir != "" {
+ finalStateDir = fmt.Sprintf("%s/%s", args.stateDir, args.dumpDir)
+ }
+
// TODO: make this configurable? Ultimately I think we don't
// want to do that; what we really want to do is have "modes"
// of criu operation where one is "make this succeed" and the
@@ -4627,23 +4645,26 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
opts := lxc.MigrateOptions{
Stop: args.stop,
- Directory: args.stateDir,
+ Directory: finalStateDir,
Verbose: true,
PreservesInodes: preservesInodes,
ActionScript: script,
GhostLimit: ghostLimit,
}
+ if args.preDumpDir != "" {
+ opts.PredumpDir = fmt.Sprintf("../%s", args.preDumpDir)
+ }
migrateErr = c.c.Migrate(args.cmd, opts)
}
- collectErr := collectCRIULogFile(c, args.stateDir, args.function, prettyCmd)
+ collectErr := collectCRIULogFile(c, finalStateDir, args.function, prettyCmd)
if collectErr != nil {
logger.Error("Error collecting checkpoint log file", log.Ctx{"err": collectErr})
}
if migrateErr != nil {
- log, err2 := getCRIULogErrors(args.stateDir, prettyCmd)
+ log, err2 := getCRIULogErrors(finalStateDir, prettyCmd)
if err2 == nil {
logger.Info("Failed migrating container", ctxMap)
migrateErr = fmt.Errorf("%s %s failed\n%s", args.function, prettyCmd, log)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index 31b0d2137..a8fd05426 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -356,6 +356,67 @@ func snapshotToProtobuf(c container) *Snapshot {
}
}
+type preDumpLoopArgs struct {
+ checkpointDir string
+ bwlimit string
+ preDumpDir string
+ dumpDir string
+ final bool
+}
+
+func (s *migrationSourceWs) preDumpLoop(args *preDumpLoopArgs) error {
+
+ // Do a CRIU pre-dump. We currently only do a single
+ // pre-dump and so we can hardcode "1" here.
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_PRE_DUMP,
+ stop: false,
+ actionScript: false,
+ preDumpDir: args.preDumpDir,
+ dumpDir: args.dumpDir,
+ stateDir: args.checkpointDir,
+ function: "migration",
+ }
+
+ logger.Debugf("Doing another pre-dump in %s", args.preDumpDir)
+
+ err := s.container.Migrate(&criuMigrationArgs)
+ if err != nil {
+ return err
+ }
+
+ // Send the pre-dump.
+ ctName, _, _ := containerGetParentAndSnapshotName(s.container.Name())
+ state := s.container.DaemonState()
+ err = RsyncSend(ctName, shared.AddSlash(args.checkpointDir), s.criuConn, nil, args.bwlimit, state.OS.ExecPath)
+ if err != nil {
+ return err
+ }
+
+ // If in pre-dump mode, the receiving side
+ // expects a message to know if this was the
+ // last pre-dump
+ logger.Debugf("Sending another header")
+ sync := MigrationSync{
+ FinalPreDump: proto.Bool(args.final),
+ }
+
+ data, err := proto.Marshal(&sync)
+
+ if err != nil {
+ return err
+ }
+
+ err = s.criuConn.WriteMessage(websocket.BinaryMessage, data)
+ if err != nil {
+ s.sendControl(err)
+ return err
+ }
+ logger.Debugf("Sending another header done")
+
+ return nil
+}
+
func (s *migrationSourceWs) Do(migrateOp *operation) error {
<-s.allConnected
@@ -605,6 +666,34 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
return abort(err)
}
+ preDumpCounter := 0
+ preDumpDir := ""
+ if use_pre_dumps {
+ do_pre_dumps := true
+ for do_pre_dumps {
+ if preDumpCounter+1 < max_iterations {
+ do_pre_dumps = true
+ } else {
+ do_pre_dumps = false
+ }
+ dumpDir := fmt.Sprintf("%03d", preDumpCounter)
+ loop_args := preDumpLoopArgs{
+ checkpointDir: checkpointDir,
+ bwlimit: bwlimit,
+ preDumpDir: preDumpDir,
+ dumpDir: dumpDir,
+ final: !do_pre_dumps,
+ }
+ err = s.preDumpLoop(&loop_args)
+ if err != nil {
+ os.RemoveAll(checkpointDir)
+ return abort(err)
+ }
+ preDumpDir = fmt.Sprintf("%03d", preDumpCounter)
+ preDumpCounter++
+ }
+ }
+
_, err = actionScriptOp.Run()
if err != nil {
os.RemoveAll(checkpointDir)
@@ -614,12 +703,16 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
go func() {
criuMigrationArgs := CriuMigrationArgs{
cmd: lxc.MIGRATE_DUMP,
- stateDir: checkpointDir,
- function: "migration",
stop: true,
actionScript: true,
+ preDumpDir: preDumpDir,
+ dumpDir: "final",
+ stateDir: checkpointDir,
+ function: "migration",
}
+ // Do the final CRIU dump. This is needs no special
+ // handling if pre-dumps are used or not
dumpSuccess <- s.container.Migrate(&criuMigrationArgs)
os.RemoveAll(checkpointDir)
}()
@@ -641,6 +734,8 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
function: "migration",
stop: true,
actionScript: false,
+ dumpDir: "final",
+ preDumpDir: "",
}
err = s.container.Migrate(&criuMigrationArgs)
@@ -1030,8 +1125,7 @@ func (c *migrationSink) Do(migrateOp *operation) error {
}
sync := &MigrationSync{
- // temporarily set to 'true' as long as not used
- FinalPreDump: proto.Bool(true),
+ FinalPreDump: proto.Bool(false),
}
if resp.GetPredump() {
@@ -1069,7 +1163,7 @@ func (c *migrationSink) Do(migrateOp *operation) error {
}
}
- // CRIU dump
+ // Final CRIU dump
err = RsyncRecv(shared.AddSlash(imagesDir), criuConn, nil)
if err != nil {
restore <- err
@@ -1090,8 +1184,13 @@ func (c *migrationSink) Do(migrateOp *operation) error {
function: "migration",
stop: false,
actionScript: false,
+ dumpDir: "final",
+ preDumpDir: "",
}
+ // Currently we only do a single CRIU pre-dump so we
+ // can hardcode "final" here since we know that "final" is the
+ // folder for CRIU's final dump.
err = c.src.container.Migrate(&criuMigrationArgs)
if err != nil {
restore <- err
From 8acd2cbf6e1a4bf3b83bf2e7a60d8de45c93f51f Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at ubuntu.com>
Date: Sat, 2 Dec 2017 16:56:28 +0100
Subject: [PATCH 8/9] doc: add "migration_pre_copy" api extension
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
doc/api-extensions.md | 3 +++
doc/containers.md | 17 +++++++++++++++++
shared/version/api.go | 1 +
3 files changed, 21 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 70b1076c1..e9ae864d6 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -359,3 +359,6 @@ This adds support for SR-IOV enabled network devices.
## console
This adds support to interact with the container console device and console log.
+
+## migration\_pre\_copy
+This adds support for optimized memory transfer during live migration.
diff --git a/doc/containers.md b/doc/containers.md
index 8d93f1f0a..91651398a 100644
--- a/doc/containers.md
+++ b/doc/containers.md
@@ -379,3 +379,20 @@ used as a shortcut to set both soft and hard limit (e.g.
`limits.kernel.nofile=3000`) to the same value. A resource with no explicitly
configured limitation will be inherited from the process starting up the
container. Note that this inheritance is not enforced by LXD but by the kernel.
+
+## Live migration
+LXD supports live migration of containers using [CRIU](http://criu.org). In
+order to optimize the memory transfer for a container LXD can be instructed to
+make use of CRIU's pre-copy features. This means LXD will request CRIU to
+perform a series of memory dumps for the container. After each dump LXD will
+send the memory dump to the specified remote. In an ideal scenario each memory
+dump will decrease the delta to the previous memory dump. When the delta is
+equal or smaller than the specified threshold LXD will request CRIU to perform
+a final memory dump and transfer it. If the threshold is not reached after
+a the maximum number of allowed iterations LXD will request a final memory dump
+from CRIU and migrate the container.
+
+Key | Type | Default | Live update | API extension | Description
+:-- | :--- | :------ | :---------- | :------------ | :----------
+migration.pre\_copy.enabled | boolean | false | yes | migration\_pre\_copy | Use CRIU's pre-copy feature for optimized container memory transfer.
+migration.pre\_copy.max | integer | 10 | yes | migration\_pre\_copy | Maximum number of pre-copy iterations to be performed.
diff --git a/shared/version/api.go b/shared/version/api.go
index 13ddbb7b0..bf3167cae 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -80,4 +80,5 @@ var APIExtensions = []string{
"macaroon_authentication",
"network_sriov",
"console",
+ "migration_pre_copy",
}
From 0cedb2d2595508437b5a6fc747b2eb3a5fb15302 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Fri, 1 Dec 2017 12:19:42 +0100
Subject: [PATCH 9/9] migrate: finish pre-dumping if threshold reached
This adds support to stop the pre-dumping if a certain threshold has
been reached. Instead of blindly relying on the 'max' iterations this
commit makes it possible to finish the pre-dumping if enough memory
has already been migrated using the pre-copy migration optimization.
The configuration option is:
migration.pre_copy.pre_migrated_pages (default to 70%)
If the current pre-dump more pages (%) migrated using pre-dumping
than defined in above variable the pre-copy migration is finished even
if the 'max' iterations has not been reached.
The lower the number the sooner the pre-copy migration will finish, but
the container downtime during migration will be longer as a result of
the time required to transfer the memory pages.
The higher the number is, the shorter the container downtime during
migration.
Signed-off-by: Adrian Reber <areber at redhat.com>
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
doc/containers.md | 9 +-
lxd/migrate.go | 110 ++++++++++++++++++---
lxd/migrate.pb.go | 271 +++++++++++++++++++++++++++++++++++++++++++++-------
lxd/migrate.proto | 37 +++++++
shared/container.go | 5 +-
5 files changed, 376 insertions(+), 56 deletions(-)
diff --git a/doc/containers.md b/doc/containers.md
index 91651398a..0b7e9e12c 100644
--- a/doc/containers.md
+++ b/doc/containers.md
@@ -392,7 +392,8 @@ a final memory dump and transfer it. If the threshold is not reached after
a the maximum number of allowed iterations LXD will request a final memory dump
from CRIU and migrate the container.
-Key | Type | Default | Live update | API extension | Description
-:-- | :--- | :------ | :---------- | :------------ | :----------
-migration.pre\_copy.enabled | boolean | false | yes | migration\_pre\_copy | Use CRIU's pre-copy feature for optimized container memory transfer.
-migration.pre\_copy.max | integer | 10 | yes | migration\_pre\_copy | Maximum number of pre-copy iterations to be performed.
+Key | Type | Default | Live update | API extension | Description
+:-- | :--- | :------ | :---------- | :------------ | :----------
+migration.pre\_copy.enabled | boolean | false | yes | migration\_pre\_copy | Use CRIU's pre-copy feature for optimized container memory transfer.
+migration.pre\_copy.max | integer | 10 | yes | migration\_pre\_copy | Maximum number of pre-copy iterations to be performed.
+migration.pre\_copy.pre\_migrated\_pages | integer | 70 | yes | migration\_pre\_copy | Threshold (%) at which enough memory pages have been pre-copied.
diff --git a/lxd/migrate.go b/lxd/migrate.go
index a8fd05426..c31c24a0b 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -7,6 +7,7 @@ package main
import (
"crypto/x509"
+ "encoding/binary"
"encoding/pem"
"fmt"
"io/ioutil"
@@ -356,6 +357,46 @@ func snapshotToProtobuf(c container) *Snapshot {
}
}
+// The function readCriuStatsDump() reads the CRIU 'stats-dump' file
+// in path and returns the pages_written, pages_skipped_parent, error.
+func readCriuStatsDump(path string) (uint64, uint64, error) {
+
+ statsDump := shared.AddSlash(path) + "stats-dump"
+ in, err := ioutil.ReadFile(statsDump)
+ if err != nil {
+ logger.Errorf("Error reading CRIU's 'stats-dump' file: %s", err.Error())
+ return 0, 0, err
+ }
+
+ // According to the CRIU file image format it starts with two magic values.
+ // First magic IMG_SERVICE: 1427134784
+ if binary.LittleEndian.Uint32(in[0:4]) != 1427134784 {
+ msg := "IMG_SERVICE(1427134784) criu magic not found"
+ logger.Errorf(msg)
+ return 0, 0, fmt.Errorf(msg)
+ }
+ // Second magic STATS: 1460220678
+ if binary.LittleEndian.Uint32(in[4:8]) != 1460220678 {
+ msg := "STATS(1460220678) criu magic not found"
+ logger.Errorf(msg)
+ return 0, 0, fmt.Errorf(msg)
+ }
+
+ // Next, read the size of the image payload
+ size := binary.LittleEndian.Uint32(in[8:12])
+ logger.Debugf("stats-dump payload size %d", size)
+
+ statsEntry := &StatsEntry{}
+ if err = proto.Unmarshal(in[12:12+size], statsEntry); err != nil {
+ logger.Errorf("Failed to parse CRIU's 'stats-dump' file: %s", err.Error())
+ return 0, 0, err
+ }
+
+ written := statsEntry.GetDump().GetPagesWritten()
+ skipped := statsEntry.GetDump().GetPagesSkippedParent()
+ return written, skipped, nil
+}
+
type preDumpLoopArgs struct {
checkpointDir string
bwlimit string
@@ -364,7 +405,11 @@ type preDumpLoopArgs struct {
final bool
}
-func (s *migrationSourceWs) preDumpLoop(args *preDumpLoopArgs) error {
+// The function preDumpLoop is the main logic behind the pre-copy migration.
+// This function contains the actual pre-dump, the corresponding rsync
+// transfer and it tells the outer loop to abort if the threshold
+// of memory pages transferred by pre-dumping has been reached.
+func (s *migrationSourceWs) preDumpLoop(args *preDumpLoopArgs) (bool, error) {
// Do a CRIU pre-dump. We currently only do a single
// pre-dump and so we can hardcode "1" here.
@@ -380,9 +425,11 @@ func (s *migrationSourceWs) preDumpLoop(args *preDumpLoopArgs) error {
logger.Debugf("Doing another pre-dump in %s", args.preDumpDir)
+ final := args.final
+
err := s.container.Migrate(&criuMigrationArgs)
if err != nil {
- return err
+ return final, err
}
// Send the pre-dump.
@@ -390,7 +437,41 @@ func (s *migrationSourceWs) preDumpLoop(args *preDumpLoopArgs) error {
state := s.container.DaemonState()
err = RsyncSend(ctName, shared.AddSlash(args.checkpointDir), s.criuConn, nil, args.bwlimit, state.OS.ExecPath)
if err != nil {
- return err
+ return final, err
+ }
+
+ // Read the CRIU's 'stats-dump' file
+ dumpPath := shared.AddSlash(args.checkpointDir)
+ dumpPath += shared.AddSlash(args.dumpDir)
+ written, skipped_parent, err := readCriuStatsDump(dumpPath)
+ if err != nil {
+ return final, err
+ }
+
+ logger.Debugf("CRIU pages written %d", written)
+ logger.Debugf("CRIU pages skipped %d", skipped_parent)
+
+ total_pages := written + skipped_parent
+
+ precentage_skipped := int(100 - ((100 * written) / total_pages))
+
+ logger.Debugf("CRIU pages skipped precentage %d%%", precentage_skipped)
+
+ // threshold is the precentage of memory pages that needs
+ // to be pre-copied for the pre-copy migration to stop.
+ var threshold int
+ tmp := s.container.ExpandedConfig()["migration.pre_copy.pre_migrated_pages"]
+ if tmp != "" {
+ threshold, _ = strconv.Atoi(tmp)
+ } else {
+ // defaults to 70%
+ threshold = 70
+ }
+
+ if precentage_skipped > threshold {
+ logger.Debugf("Memory pages skipped (%d%%) due to pre-copy is larger than threshold (%d%%)", precentage_skipped, threshold)
+ logger.Debugf("This was the last pre-dump; next dump is the final dump")
+ final = true
}
// If in pre-dump mode, the receiving side
@@ -398,23 +479,23 @@ func (s *migrationSourceWs) preDumpLoop(args *preDumpLoopArgs) error {
// last pre-dump
logger.Debugf("Sending another header")
sync := MigrationSync{
- FinalPreDump: proto.Bool(args.final),
+ FinalPreDump: proto.Bool(final),
}
data, err := proto.Marshal(&sync)
if err != nil {
- return err
+ return final, err
}
err = s.criuConn.WriteMessage(websocket.BinaryMessage, data)
if err != nil {
s.sendControl(err)
- return err
+ return final, err
}
logger.Debugf("Sending another header done")
- return nil
+ return final, nil
}
func (s *migrationSourceWs) Do(migrateOp *operation) error {
@@ -669,12 +750,13 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
preDumpCounter := 0
preDumpDir := ""
if use_pre_dumps {
- do_pre_dumps := true
- for do_pre_dumps {
- if preDumpCounter+1 < max_iterations {
- do_pre_dumps = true
+ final := false
+ for !final {
+ preDumpCounter++
+ if preDumpCounter < max_iterations {
+ final = false
} else {
- do_pre_dumps = false
+ final = true
}
dumpDir := fmt.Sprintf("%03d", preDumpCounter)
loop_args := preDumpLoopArgs{
@@ -682,9 +764,9 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
bwlimit: bwlimit,
preDumpDir: preDumpDir,
dumpDir: dumpDir,
- final: !do_pre_dumps,
+ final: final,
}
- err = s.preDumpLoop(&loop_args)
+ final, err = s.preDumpLoop(&loop_args)
if err != nil {
os.RemoveAll(checkpointDir)
return abort(err)
diff --git a/lxd/migrate.pb.go b/lxd/migrate.pb.go
index 348db0fe1..85368b02d 100644
--- a/lxd/migrate.pb.go
+++ b/lxd/migrate.pb.go
@@ -16,6 +16,9 @@ It has these top-level messages:
MigrationHeader
MigrationControl
MigrationSync
+ DumpStatsEntry
+ RestoreStatsEntry
+ StatsEntry
*/
package main
@@ -375,6 +378,178 @@ func (m *MigrationSync) GetFinalPreDump() bool {
return false
}
+// This one contains statistics about dump/restore process
+type DumpStatsEntry struct {
+ FreezingTime *uint32 `protobuf:"varint,1,req,name=freezing_time,json=freezingTime" json:"freezing_time,omitempty"`
+ FrozenTime *uint32 `protobuf:"varint,2,req,name=frozen_time,json=frozenTime" json:"frozen_time,omitempty"`
+ MemdumpTime *uint32 `protobuf:"varint,3,req,name=memdump_time,json=memdumpTime" json:"memdump_time,omitempty"`
+ MemwriteTime *uint32 `protobuf:"varint,4,req,name=memwrite_time,json=memwriteTime" json:"memwrite_time,omitempty"`
+ PagesScanned *uint64 `protobuf:"varint,5,req,name=pages_scanned,json=pagesScanned" json:"pages_scanned,omitempty"`
+ PagesSkippedParent *uint64 `protobuf:"varint,6,req,name=pages_skipped_parent,json=pagesSkippedParent" json:"pages_skipped_parent,omitempty"`
+ PagesWritten *uint64 `protobuf:"varint,7,req,name=pages_written,json=pagesWritten" json:"pages_written,omitempty"`
+ IrmapResolve *uint32 `protobuf:"varint,8,opt,name=irmap_resolve,json=irmapResolve" json:"irmap_resolve,omitempty"`
+ PagesLazy *uint64 `protobuf:"varint,9,req,name=pages_lazy,json=pagesLazy" json:"pages_lazy,omitempty"`
+ PagePipes *uint64 `protobuf:"varint,10,opt,name=page_pipes,json=pagePipes" json:"page_pipes,omitempty"`
+ PagePipeBufs *uint64 `protobuf:"varint,11,opt,name=page_pipe_bufs,json=pagePipeBufs" json:"page_pipe_bufs,omitempty"`
+ XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *DumpStatsEntry) Reset() { *m = DumpStatsEntry{} }
+func (m *DumpStatsEntry) String() string { return proto.CompactTextString(m) }
+func (*DumpStatsEntry) ProtoMessage() {}
+func (*DumpStatsEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *DumpStatsEntry) GetFreezingTime() uint32 {
+ if m != nil && m.FreezingTime != nil {
+ return *m.FreezingTime
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetFrozenTime() uint32 {
+ if m != nil && m.FrozenTime != nil {
+ return *m.FrozenTime
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetMemdumpTime() uint32 {
+ if m != nil && m.MemdumpTime != nil {
+ return *m.MemdumpTime
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetMemwriteTime() uint32 {
+ if m != nil && m.MemwriteTime != nil {
+ return *m.MemwriteTime
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetPagesScanned() uint64 {
+ if m != nil && m.PagesScanned != nil {
+ return *m.PagesScanned
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetPagesSkippedParent() uint64 {
+ if m != nil && m.PagesSkippedParent != nil {
+ return *m.PagesSkippedParent
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetPagesWritten() uint64 {
+ if m != nil && m.PagesWritten != nil {
+ return *m.PagesWritten
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetIrmapResolve() uint32 {
+ if m != nil && m.IrmapResolve != nil {
+ return *m.IrmapResolve
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetPagesLazy() uint64 {
+ if m != nil && m.PagesLazy != nil {
+ return *m.PagesLazy
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetPagePipes() uint64 {
+ if m != nil && m.PagePipes != nil {
+ return *m.PagePipes
+ }
+ return 0
+}
+
+func (m *DumpStatsEntry) GetPagePipeBufs() uint64 {
+ if m != nil && m.PagePipeBufs != nil {
+ return *m.PagePipeBufs
+ }
+ return 0
+}
+
+type RestoreStatsEntry struct {
+ PagesCompared *uint64 `protobuf:"varint,1,req,name=pages_compared,json=pagesCompared" json:"pages_compared,omitempty"`
+ PagesSkippedCow *uint64 `protobuf:"varint,2,req,name=pages_skipped_cow,json=pagesSkippedCow" json:"pages_skipped_cow,omitempty"`
+ ForkingTime *uint32 `protobuf:"varint,3,req,name=forking_time,json=forkingTime" json:"forking_time,omitempty"`
+ RestoreTime *uint32 `protobuf:"varint,4,req,name=restore_time,json=restoreTime" json:"restore_time,omitempty"`
+ PagesRestored *uint64 `protobuf:"varint,5,opt,name=pages_restored,json=pagesRestored" json:"pages_restored,omitempty"`
+ XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *RestoreStatsEntry) Reset() { *m = RestoreStatsEntry{} }
+func (m *RestoreStatsEntry) String() string { return proto.CompactTextString(m) }
+func (*RestoreStatsEntry) ProtoMessage() {}
+func (*RestoreStatsEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *RestoreStatsEntry) GetPagesCompared() uint64 {
+ if m != nil && m.PagesCompared != nil {
+ return *m.PagesCompared
+ }
+ return 0
+}
+
+func (m *RestoreStatsEntry) GetPagesSkippedCow() uint64 {
+ if m != nil && m.PagesSkippedCow != nil {
+ return *m.PagesSkippedCow
+ }
+ return 0
+}
+
+func (m *RestoreStatsEntry) GetForkingTime() uint32 {
+ if m != nil && m.ForkingTime != nil {
+ return *m.ForkingTime
+ }
+ return 0
+}
+
+func (m *RestoreStatsEntry) GetRestoreTime() uint32 {
+ if m != nil && m.RestoreTime != nil {
+ return *m.RestoreTime
+ }
+ return 0
+}
+
+func (m *RestoreStatsEntry) GetPagesRestored() uint64 {
+ if m != nil && m.PagesRestored != nil {
+ return *m.PagesRestored
+ }
+ return 0
+}
+
+type StatsEntry struct {
+ Dump *DumpStatsEntry `protobuf:"bytes,1,opt,name=dump" json:"dump,omitempty"`
+ Restore *RestoreStatsEntry `protobuf:"bytes,2,opt,name=restore" json:"restore,omitempty"`
+ XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *StatsEntry) Reset() { *m = StatsEntry{} }
+func (m *StatsEntry) String() string { return proto.CompactTextString(m) }
+func (*StatsEntry) ProtoMessage() {}
+func (*StatsEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *StatsEntry) GetDump() *DumpStatsEntry {
+ if m != nil {
+ return m.Dump
+ }
+ return nil
+}
+
+func (m *StatsEntry) GetRestore() *RestoreStatsEntry {
+ if m != nil {
+ return m.Restore
+ }
+ return nil
+}
+
func init() {
proto.RegisterType((*IDMapType)(nil), "main.IDMapType")
proto.RegisterType((*Config)(nil), "main.Config")
@@ -383,6 +558,9 @@ func init() {
proto.RegisterType((*MigrationHeader)(nil), "main.MigrationHeader")
proto.RegisterType((*MigrationControl)(nil), "main.MigrationControl")
proto.RegisterType((*MigrationSync)(nil), "main.MigrationSync")
+ proto.RegisterType((*DumpStatsEntry)(nil), "main.dump_stats_entry")
+ proto.RegisterType((*RestoreStatsEntry)(nil), "main.restore_stats_entry")
+ proto.RegisterType((*StatsEntry)(nil), "main.stats_entry")
proto.RegisterEnum("main.MigrationFSType", MigrationFSType_name, MigrationFSType_value)
proto.RegisterEnum("main.CRIUType", CRIUType_name, CRIUType_value)
}
@@ -390,40 +568,61 @@ func init() {
func init() { proto.RegisterFile("lxd/migrate.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 560 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xcd, 0x6b, 0xdb, 0x4e,
- 0x10, 0xfd, 0x49, 0xf2, 0x87, 0x34, 0x49, 0x1c, 0xfd, 0x96, 0xb6, 0x2c, 0xa5, 0x07, 0x21, 0x12,
- 0x10, 0xa1, 0x38, 0xc1, 0x3d, 0xf5, 0xd8, 0xd8, 0x35, 0x09, 0x34, 0x6e, 0x58, 0x25, 0x87, 0xf6,
- 0x52, 0x16, 0x69, 0x25, 0x2f, 0xd5, 0x17, 0xbb, 0x52, 0xa8, 0x4f, 0xfd, 0x9b, 0x7b, 0xed, 0xa9,
- 0xec, 0xea, 0x23, 0x36, 0xb4, 0xb7, 0x79, 0x6f, 0x9e, 0xe6, 0x69, 0xde, 0x0e, 0xfc, 0x9f, 0xfd,
- 0x88, 0x2f, 0x73, 0x9e, 0x0a, 0x5a, 0xb3, 0x79, 0x25, 0xca, 0xba, 0x44, 0xa3, 0x9c, 0xf2, 0xc2,
- 0xff, 0x09, 0xce, 0xed, 0xea, 0x8e, 0x56, 0x0f, 0xbb, 0x8a, 0xa1, 0x17, 0x30, 0xe6, 0xb2, 0xe1,
- 0x31, 0x36, 0x3c, 0x33, 0xb0, 0x49, 0x0b, 0x5a, 0x36, 0xe5, 0x31, 0x36, 0x7b, 0x36, 0xe5, 0x31,
- 0x7a, 0x05, 0x93, 0x6d, 0x29, 0x6b, 0x1e, 0x63, 0xcb, 0x33, 0x83, 0x31, 0xe9, 0x10, 0x42, 0x30,
- 0x2a, 0x24, 0x8f, 0xf1, 0x48, 0xb3, 0xba, 0x46, 0xaf, 0xc1, 0xce, 0x69, 0x25, 0x68, 0x91, 0x32,
- 0x3c, 0xd6, 0xfc, 0x80, 0xfd, 0x2b, 0x98, 0x2c, 0xcb, 0x22, 0xe1, 0x29, 0x72, 0xc1, 0xfa, 0xce,
- 0x76, 0xda, 0xdb, 0x21, 0xaa, 0x54, 0xce, 0x4f, 0x34, 0x6b, 0x98, 0x76, 0x76, 0x48, 0x0b, 0xfc,
- 0x6b, 0x98, 0xac, 0xd8, 0x13, 0x8f, 0x98, 0xf6, 0xa2, 0x39, 0xeb, 0x3e, 0xd1, 0x35, 0x3a, 0x83,
- 0x49, 0xa4, 0xe7, 0x61, 0xd3, 0xb3, 0x82, 0xa3, 0xc5, 0xf1, 0x5c, 0xed, 0x39, 0x6f, 0x3d, 0x48,
- 0xd7, 0xf3, 0x7f, 0x1b, 0x60, 0x87, 0x05, 0xad, 0xe4, 0xb6, 0xac, 0xff, 0x3a, 0x66, 0x0e, 0x47,
- 0x59, 0x19, 0xd1, 0x6c, 0xf9, 0xef, 0x59, 0xfb, 0x02, 0xb5, 0x62, 0x25, 0xca, 0x84, 0x67, 0x4c,
- 0x62, 0xcb, 0xb3, 0x02, 0x87, 0x0c, 0x18, 0xbd, 0x01, 0x87, 0x55, 0x5b, 0x96, 0x33, 0x41, 0x33,
- 0x9d, 0x8b, 0x4d, 0x9e, 0x09, 0x74, 0x05, 0xc7, 0x7a, 0x50, 0xbb, 0x93, 0xc4, 0xe3, 0x7d, 0xab,
- 0x96, 0x24, 0x07, 0x0a, 0xe4, 0xc3, 0x31, 0x15, 0xd1, 0x96, 0xd7, 0x2c, 0xaa, 0x1b, 0xc1, 0xf0,
- 0x44, 0x47, 0x7a, 0xc0, 0xa9, 0xff, 0x91, 0x35, 0xad, 0x59, 0xd2, 0x64, 0x78, 0xaa, 0x2d, 0x07,
- 0xec, 0xff, 0x32, 0xe0, 0xf4, 0x4e, 0xdf, 0x02, 0x2f, 0x8b, 0x1b, 0x46, 0x63, 0x26, 0xd0, 0x39,
- 0x98, 0x89, 0xd4, 0x09, 0xcc, 0x16, 0x2f, 0x5b, 0xef, 0x41, 0xb2, 0x0e, 0xd5, 0x75, 0x10, 0x33,
- 0x51, 0xd6, 0xa3, 0x48, 0xf0, 0x06, 0x9b, 0x9e, 0x11, 0xcc, 0x16, 0xb3, 0x2e, 0x0f, 0x72, 0xfb,
- 0xa8, 0x15, 0xba, 0x87, 0xce, 0x61, 0xcc, 0xe3, 0x9c, 0x56, 0x3a, 0x87, 0xa3, 0xc5, 0x69, 0x2b,
- 0x1a, 0xae, 0x8c, 0xb4, 0x5d, 0x74, 0x06, 0x27, 0xb2, 0x7b, 0x81, 0x0d, 0xcd, 0x99, 0xc4, 0x23,
- 0x1d, 0xdb, 0x21, 0x89, 0xde, 0x82, 0xd3, 0x13, 0x7d, 0x34, 0x9d, 0x6b, 0xff, 0x7c, 0xe4, 0x59,
- 0x80, 0x30, 0x4c, 0x2b, 0xc1, 0xe2, 0x26, 0xaf, 0xf0, 0xd4, 0x33, 0x02, 0x9b, 0xf4, 0xd0, 0x5f,
- 0x83, 0x3b, 0xec, 0xb3, 0x2c, 0x8b, 0x5a, 0x94, 0x99, 0x52, 0xcb, 0x26, 0x8a, 0x98, 0x94, 0xdd,
- 0xc1, 0xf7, 0x50, 0x75, 0x72, 0x26, 0x25, 0x4d, 0x99, 0xde, 0xd4, 0x21, 0x3d, 0xf4, 0xdf, 0xc1,
- 0xc9, 0x30, 0x27, 0xdc, 0x15, 0x91, 0x7a, 0x8c, 0x84, 0x17, 0x34, 0xbb, 0x17, 0x6c, 0xa5, 0x7c,
- 0xdb, 0x49, 0x07, 0xdc, 0xc5, 0xfb, 0xbd, 0xbc, 0xdb, 0x30, 0x91, 0x03, 0x63, 0x12, 0x7e, 0xd9,
- 0x2c, 0xdd, 0xff, 0x54, 0x79, 0xfd, 0x40, 0xd6, 0xa1, 0x6b, 0xa0, 0x29, 0x58, 0x5f, 0xd7, 0xa1,
- 0x6b, 0xaa, 0x82, 0x5c, 0xaf, 0x5c, 0xeb, 0xe2, 0x12, 0xec, 0x3e, 0x5e, 0x34, 0x03, 0x50, 0xf5,
- 0xb7, 0xbd, 0x0f, 0xef, 0x6f, 0x3e, 0x3c, 0x7e, 0x72, 0x0d, 0x64, 0xc3, 0x68, 0xf3, 0x79, 0xf3,
- 0xd1, 0x35, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x5a, 0xe1, 0x62, 0xc7, 0xea, 0x03, 0x00, 0x00,
+ // 888 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x94, 0xcd, 0x6e, 0xdb, 0x46,
+ 0x10, 0xc7, 0x4b, 0x89, 0xb2, 0xc5, 0xa1, 0x2c, 0x2b, 0xdb, 0x34, 0x60, 0x8b, 0x16, 0x55, 0x19,
+ 0x1b, 0x10, 0x8c, 0xc2, 0x31, 0x94, 0x53, 0x8f, 0xb5, 0x5c, 0x23, 0x01, 0x12, 0xd7, 0x58, 0x39,
+ 0x28, 0xda, 0x0b, 0xb1, 0x25, 0x87, 0xf2, 0xc2, 0xfc, 0xc2, 0x2e, 0x65, 0x57, 0xbe, 0xf4, 0x69,
+ 0xfa, 0x48, 0x7d, 0x90, 0x5e, 0x7b, 0x2a, 0x76, 0x96, 0xa4, 0xa5, 0x22, 0xb9, 0xed, 0xfc, 0xe6,
+ 0xcf, 0x99, 0x9d, 0x0f, 0x2e, 0x3c, 0xcb, 0xfe, 0x48, 0x5e, 0xe5, 0x72, 0xa5, 0x44, 0x8d, 0xa7,
+ 0x95, 0x2a, 0xeb, 0x92, 0xb9, 0xb9, 0x90, 0x45, 0xf8, 0x27, 0x78, 0x6f, 0x2f, 0xde, 0x8b, 0xea,
+ 0x66, 0x53, 0x21, 0x7b, 0x0e, 0x03, 0xa9, 0xd7, 0x32, 0x09, 0x9c, 0x69, 0x6f, 0x36, 0xe4, 0xd6,
+ 0xb0, 0x74, 0x25, 0x93, 0xa0, 0xd7, 0xd2, 0x95, 0x4c, 0xd8, 0x0b, 0xd8, 0xbb, 0x2d, 0x75, 0x2d,
+ 0x93, 0xa0, 0x3f, 0xed, 0xcd, 0x06, 0xbc, 0xb1, 0x18, 0x03, 0xb7, 0xd0, 0x32, 0x09, 0x5c, 0xa2,
+ 0x74, 0x66, 0x5f, 0xc1, 0x30, 0x17, 0x95, 0x12, 0xc5, 0x0a, 0x83, 0x01, 0xf1, 0xce, 0x0e, 0xcf,
+ 0x60, 0x6f, 0x51, 0x16, 0xa9, 0x5c, 0xb1, 0x09, 0xf4, 0xef, 0x70, 0x43, 0xb9, 0x3d, 0x6e, 0x8e,
+ 0x26, 0xf3, 0xbd, 0xc8, 0xd6, 0x48, 0x99, 0x3d, 0x6e, 0x8d, 0xf0, 0x1c, 0xf6, 0x2e, 0xf0, 0x5e,
+ 0xc6, 0x48, 0xb9, 0x44, 0x8e, 0xcd, 0x27, 0x74, 0x66, 0x47, 0xb0, 0x17, 0x53, 0xbc, 0xa0, 0x37,
+ 0xed, 0xcf, 0xfc, 0xf9, 0xe8, 0xd4, 0xd4, 0x79, 0x6a, 0x73, 0xf0, 0xc6, 0x17, 0xfe, 0xeb, 0xc0,
+ 0x70, 0x59, 0x88, 0x4a, 0xdf, 0x96, 0xf5, 0x47, 0xc3, 0x9c, 0x82, 0x9f, 0x95, 0xb1, 0xc8, 0x16,
+ 0x9f, 0x8e, 0xb5, 0x2d, 0x30, 0x25, 0x56, 0xaa, 0x4c, 0x65, 0x86, 0x3a, 0xe8, 0x4f, 0xfb, 0x33,
+ 0x8f, 0x77, 0x36, 0xfb, 0x1a, 0x3c, 0xac, 0x6e, 0x31, 0x47, 0x25, 0x32, 0xea, 0xcb, 0x90, 0x3f,
+ 0x01, 0x76, 0x06, 0x23, 0x0a, 0x64, 0x6b, 0xd2, 0xc1, 0x60, 0x3b, 0x95, 0x85, 0x7c, 0x47, 0xc1,
+ 0x42, 0x18, 0x09, 0x15, 0xdf, 0xca, 0x1a, 0xe3, 0x7a, 0xad, 0x30, 0xd8, 0xa3, 0x96, 0xee, 0x30,
+ 0x73, 0x1f, 0x5d, 0x8b, 0x1a, 0xd3, 0x75, 0x16, 0xec, 0x53, 0xca, 0xce, 0x0e, 0xff, 0x71, 0xe0,
+ 0xf0, 0x3d, 0xed, 0x82, 0x2c, 0x8b, 0x37, 0x28, 0x12, 0x54, 0xec, 0x18, 0x7a, 0xa9, 0xa6, 0x0e,
+ 0x8c, 0xe7, 0x5f, 0xd8, 0xdc, 0x9d, 0xe4, 0x72, 0x69, 0xb6, 0x83, 0xf7, 0x52, 0x93, 0xda, 0x8d,
+ 0x95, 0x5c, 0x07, 0xbd, 0xa9, 0x33, 0x1b, 0xcf, 0xc7, 0x4d, 0x3f, 0xf8, 0xdb, 0x0f, 0xa4, 0x20,
+ 0x1f, 0x3b, 0x86, 0x81, 0x4c, 0x72, 0x51, 0x51, 0x1f, 0xfc, 0xf9, 0xa1, 0x15, 0x75, 0x5b, 0xc6,
+ 0xad, 0x97, 0x1d, 0xc1, 0x81, 0x6e, 0x26, 0x70, 0x25, 0x72, 0xd4, 0x81, 0x4b, 0x6d, 0xdb, 0x85,
+ 0xec, 0x7b, 0xf0, 0x5a, 0xd0, 0xb6, 0xa6, 0xc9, 0xda, 0x8e, 0x8f, 0x3f, 0x09, 0x58, 0x00, 0xfb,
+ 0x95, 0xc2, 0x64, 0x9d, 0x57, 0xc1, 0xfe, 0xd4, 0x99, 0x0d, 0x79, 0x6b, 0x86, 0x97, 0x30, 0xe9,
+ 0xea, 0x59, 0x94, 0x45, 0xad, 0xca, 0xcc, 0xa8, 0xf5, 0x3a, 0x8e, 0x51, 0xeb, 0x66, 0xe1, 0x5b,
+ 0xd3, 0x78, 0x72, 0xd4, 0x5a, 0xac, 0x90, 0x2a, 0xf5, 0x78, 0x6b, 0x86, 0xaf, 0xe1, 0xa0, 0x8b,
+ 0xb3, 0xdc, 0x14, 0xb1, 0x19, 0x46, 0x2a, 0x0b, 0x91, 0x5d, 0x2b, 0xbc, 0x30, 0x79, 0x6d, 0xa4,
+ 0x1d, 0x16, 0xfe, 0xd5, 0x87, 0x89, 0xb9, 0x45, 0x64, 0x46, 0xa0, 0x23, 0x2c, 0x6a, 0xb5, 0x61,
+ 0x2f, 0xe1, 0x20, 0x55, 0x88, 0x8f, 0xb2, 0x58, 0x45, 0xb5, 0x6c, 0xd6, 0xef, 0x80, 0x8f, 0x5a,
+ 0x78, 0x23, 0x73, 0x64, 0xdf, 0x82, 0x9f, 0xaa, 0xf2, 0x11, 0x0b, 0x2b, 0xe9, 0x91, 0x04, 0x2c,
+ 0x22, 0xc1, 0x77, 0x30, 0xca, 0x31, 0xa7, 0xe0, 0xa4, 0xe8, 0x93, 0xc2, 0x6f, 0x18, 0x49, 0x5e,
+ 0xc2, 0x41, 0x8e, 0xf9, 0x83, 0x92, 0x35, 0x5a, 0x8d, 0x6b, 0x13, 0xb5, 0xb0, 0x15, 0x55, 0x62,
+ 0x85, 0x3a, 0xd2, 0xb1, 0x28, 0x0a, 0x4c, 0xe8, 0x3f, 0x75, 0xf9, 0x88, 0xe0, 0xd2, 0x32, 0x76,
+ 0x06, 0xcf, 0x1b, 0xd1, 0x9d, 0xac, 0x2a, 0x4c, 0xa2, 0x4a, 0x28, 0x2c, 0x6a, 0x5a, 0x40, 0x97,
+ 0x33, 0xab, 0xb5, 0xae, 0x6b, 0xf2, 0x3c, 0x85, 0x35, 0x99, 0x6a, 0x2c, 0x68, 0x17, 0xdb, 0xb0,
+ 0xbf, 0x58, 0x66, 0x44, 0x52, 0xe5, 0xa2, 0x8a, 0x14, 0xea, 0x32, 0xbb, 0xc7, 0x60, 0x38, 0x75,
+ 0xcc, 0x05, 0x09, 0x72, 0xcb, 0xd8, 0x37, 0x00, 0x36, 0x52, 0x26, 0x1e, 0x37, 0x81, 0x47, 0x61,
+ 0x3c, 0x22, 0xef, 0xc4, 0xe3, 0xa6, 0x75, 0x47, 0x95, 0xac, 0x50, 0x07, 0x30, 0x75, 0x5a, 0xf7,
+ 0xb5, 0x01, 0xec, 0x08, 0xc6, 0x9d, 0x3b, 0xfa, 0x7d, 0x9d, 0xea, 0xc0, 0x27, 0xc9, 0xa8, 0x95,
+ 0x9c, 0xaf, 0x53, 0x1d, 0xfe, 0xed, 0xc0, 0xe7, 0x0a, 0x75, 0x5d, 0x2a, 0xdc, 0x19, 0xd5, 0xb1,
+ 0xfd, 0x5a, 0x47, 0x71, 0x99, 0x9b, 0x92, 0xed, 0x03, 0xe9, 0x72, 0x5b, 0xdb, 0xa2, 0x81, 0xec,
+ 0x04, 0x9e, 0xed, 0xb6, 0x27, 0x2e, 0x1f, 0x68, 0x64, 0x2e, 0x3f, 0xdc, 0xee, 0xcd, 0xa2, 0x7c,
+ 0x30, 0x73, 0x4b, 0x4b, 0x75, 0xd7, 0x0d, 0xbf, 0x99, 0x5b, 0xc3, 0xda, 0xd1, 0xb6, 0x97, 0xd9,
+ 0x1a, 0x9b, 0xdf, 0x30, 0x92, 0x74, 0x17, 0x6b, 0xa0, 0x19, 0x9b, 0xd3, 0x5d, 0x8c, 0x37, 0x30,
+ 0x2c, 0xc0, 0xdf, 0x2e, 0xe7, 0x04, 0xdc, 0xc4, 0xae, 0xaa, 0x33, 0xf3, 0xe7, 0x2f, 0xec, 0xef,
+ 0xf4, 0xff, 0xfd, 0xe4, 0xa4, 0x61, 0xaf, 0x61, 0xbf, 0x89, 0x4d, 0x7f, 0x82, 0x3f, 0xff, 0xd2,
+ 0xca, 0x3f, 0xd2, 0x26, 0xde, 0x2a, 0x4f, 0x7e, 0xd8, 0x7a, 0x5f, 0xec, 0xe3, 0xc1, 0x3c, 0x18,
+ 0xf0, 0xe5, 0xaf, 0x57, 0x8b, 0xc9, 0x67, 0xe6, 0x78, 0x7e, 0xc3, 0x2f, 0x97, 0x13, 0x87, 0xed,
+ 0x43, 0xff, 0xb7, 0xcb, 0xe5, 0xa4, 0x67, 0x0e, 0xfc, 0xfc, 0x62, 0xd2, 0x3f, 0x79, 0x05, 0xc3,
+ 0xf6, 0x39, 0x61, 0x63, 0x00, 0x73, 0x8e, 0xb6, 0x3e, 0xbc, 0x7e, 0xf3, 0xe3, 0x87, 0x77, 0x13,
+ 0x87, 0x0d, 0xc1, 0xbd, 0xfa, 0xf9, 0xea, 0xa7, 0x49, 0xef, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff,
+ 0x0d, 0x9a, 0xe8, 0xf0, 0xda, 0x06, 0x00, 0x00,
}
diff --git a/lxd/migrate.proto b/lxd/migrate.proto
index 3758c949e..8bcfdc295 100644
--- a/lxd/migrate.proto
+++ b/lxd/migrate.proto
@@ -63,3 +63,40 @@ message MigrationControl {
message MigrationSync {
required bool finalPreDump = 1;
}
+
+// Taken from criu: images/stats.proto
+// Needed to read out pages_written and pages_skipped_parent for
+// pre-copy migration final dump condition
+
+// This one contains statistics about dump/restore process
+message dump_stats_entry {
+ required uint32 freezing_time = 1;
+ required uint32 frozen_time = 2;
+ required uint32 memdump_time = 3;
+ required uint32 memwrite_time = 4;
+
+ required uint64 pages_scanned = 5;
+ required uint64 pages_skipped_parent = 6;
+ required uint64 pages_written = 7;
+
+ optional uint32 irmap_resolve = 8;
+
+ required uint64 pages_lazy = 9;
+ optional uint64 page_pipes = 10;
+ optional uint64 page_pipe_bufs = 11;
+}
+
+message restore_stats_entry {
+ required uint64 pages_compared = 1;
+ required uint64 pages_skipped_cow = 2;
+
+ required uint32 forking_time = 3;
+ required uint32 restore_time = 4;
+
+ optional uint64 pages_restored = 5;
+}
+
+message stats_entry {
+ optional dump_stats_entry dump = 1;
+ optional restore_stats_entry restore = 2;
+}
diff --git a/shared/container.go b/shared/container.go
index de2fd9fb1..2062cf868 100644
--- a/shared/container.go
+++ b/shared/container.go
@@ -167,8 +167,9 @@ var KnownContainerConfigKeys = map[string]func(value string) error{
"linux.kernel_modules": IsAny,
- "migration.pre_copy.enabled": IsBool,
- "migration.pre_copy.max": IsUint32,
+ "migration.pre_copy.enabled": IsBool,
+ "migration.pre_copy.max": IsUint32,
+ "migration.pre_copy.pre_migrated_pages": IsUint32,
"security.nesting": IsBool,
"security.privileged": IsBool,
More information about the lxc-devel
mailing list