[lxc-devel] [lxd/master] mirgation: enable pre-dumping based on CRIU's feature check
adrianreber on Github
lxc-bot at linuxcontainers.org
Sun Dec 17 20:24:34 UTC 2017
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 812 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20171217/da37bcf8/attachment.bin>
-------------- next part --------------
From 2582ae0fa8e5f1d6fc91893bf242ab27df1b07a6 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Sun, 17 Dec 2017 20:37:57 +0100
Subject: [PATCH 1/4] migration: add handler for CRIU feature checking
Now that lxc and go-lxc know about CRIU feature checking also start to
include feature checking in LXD.
Signed-off-by: Adrian Reber <areber at redhat.com>
---
lxd/container_lxc.go | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 5851f36a7..e2a89b9c1 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -4646,6 +4646,7 @@ type CriuMigrationArgs struct {
actionScript bool
dumpDir string
preDumpDir string
+ features lxc.CriuFeatures
}
func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
@@ -4656,6 +4657,7 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
"statedir": args.stateDir,
"actionscript": args.actionScript,
"predumpdir": args.preDumpDir,
+ "features": args.features,
"stop": args.stop}
_, err := exec.LookPath("criu")
@@ -4679,6 +4681,8 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
prettyCmd = "dump"
case lxc.MIGRATE_RESTORE:
prettyCmd = "restore"
+ case lxc.MIGRATE_FEATURE_CHECK:
+ prettyCmd = "feature-check"
default:
prettyCmd = "unknown"
logger.Warn("unknown migrate call", log.Ctx{"cmd": args.cmd})
@@ -4757,7 +4761,17 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
logger.Debugf("forkmigrate: %s", line)
}
}
+ } else if args.cmd == lxc.MIGRATE_FEATURE_CHECK {
+ opts := lxc.MigrateOptions{
+ FeaturesToCheck: args.features,
+ }
+ migrateErr = c.c.Migrate(args.cmd, opts)
+ if migrateErr != nil {
+ logger.Info("CRIU feature check failed", ctxMap)
+ return migrateErr
+ }
+ return nil
} else {
err := c.initLXC(true)
if err != nil {
From 29b84750eacb7e75ab25f18b1531c43ab4faa37a Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Sun, 17 Dec 2017 20:40:03 +0100
Subject: [PATCH 2/4] migration: remove obsolete TODO comment
Initially the idea was to check for pre-copy support also on the
receiving side. CRIU supports pre-copying for a 'very long' time already
and there is no special kernel support required to restore a pre-copy
checkpoint. If the sender wants to do pre-copy migration let's just do it.
If the receiving side does not include the code for pre-copy migration
the protobuf header will neither be filled with 'true' or 'false'
concerning the pre-copy support of the receiving side which will be
interpreted as 'false' on the side of the sender as the protocol to
handle multiple rsync transfers for each pre-copy iteration is only
available after the pre-copy support has been added.
Interestingly this is a rather long commit message for a single line
comment removal.
Signed-off-by: Adrian Reber <areber at redhat.com>
---
lxd/migrate.go | 1 -
1 file changed, 1 deletion(-)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index e7789fddb..d237a65a0 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -1102,7 +1102,6 @@ func (c *migrationSink) Do(migrateOp *operation) error {
if header.GetPredump() == true {
// If the other side wants pre-dump and if
// this side supports it, let's use it.
- // TODO: check kernel+criu (and config?)
resp.Predump = proto.Bool(true)
} else {
resp.Predump = proto.Bool(false)
From a6e77fd7b4c71e810dbe4525587515f97a7db975 Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Sun, 17 Dec 2017 20:55:07 +0100
Subject: [PATCH 3/4] migration: move pre-dump check to its own function
This just moves code around to decrease the size of the main migration
function. The pre-dump configuration variable reading is moved to its
own function.
Signed-off-by: Adrian Reber <areber at redhat.com>
---
lxd/migrate.go | 83 ++++++++++++++++++++++++++++++++--------------------------
1 file changed, 46 insertions(+), 37 deletions(-)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index d237a65a0..4067a184f 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -357,6 +357,51 @@ func snapshotToProtobuf(c container) *Snapshot {
}
}
+// Check if CRIU supports pre-dumping and number of
+// pre-dump iterations
+func (s *migrationSourceWs) checkForPreDumpSupport() (bool, int) {
+ // TODO: ask CRIU if this system (kernel+criu) supports
+ // pre-copy (dirty memory tracking)
+ // The user should also be enable to influence it from the
+ // command-line.
+
+ // What does the configuration say about pre-copy
+ tmp := s.container.ExpandedConfig()["migration.incremental.memory"]
+
+ // default to false for pre-dumps as long as libxlc has no
+ // detection for the feature
+ use_pre_dumps := false
+
+ if tmp != "" {
+ use_pre_dumps = shared.IsTrue(tmp)
+ }
+ logger.Debugf("migration.incremental.memory %d", use_pre_dumps)
+
+
+ // migration.incremental.memory.iterations is the value after which the
+ // container will be definitely migrated, even if the remaining number
+ // of memory pages is below the defined threshold.
+ // TODO: implement threshold (needs reading of CRIU output files)
+ var max_iterations int
+ tmp = s.container.ExpandedConfig()["migration.incremental.memory.iterations"]
+ if tmp != "" {
+ max_iterations, _ = strconv.Atoi(tmp)
+ } else {
+ // default to 10
+ max_iterations = 10
+ }
+ if max_iterations > 999 {
+ // the pre-dump directory is hardcoded to a string
+ // with maximal 3 digits. 999 pre-dumps makes no
+ // sense at all, but let's make sure the number
+ // is not higher than this.
+ max_iterations = 999
+ }
+ logger.Debugf("using maximal %d iterations for pre-dumping", max_iterations)
+
+ return use_pre_dumps, max_iterations
+}
+
// The function readCriuStatsDump() reads the CRIU 'stats-dump' file
// in path and returns the pages_written, pages_skipped_parent, error.
func readCriuStatsDump(path string) (uint64, uint64, error) {
@@ -552,43 +597,7 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
}
}
- // TODO: ask CRIU if this system (kernel+criu) supports
- // pre-copy (dirty memory tracking)
- // The user should also be enable to influence it from the
- // command-line.
-
- // What does the config say about pre-copy
- tmp := s.container.ExpandedConfig()["migration.incremental.memory"]
-
- // default to false for pre-dumps as long as libxlc has no
- // detection for the feature
- use_pre_dumps := false
-
- if tmp != "" {
- use_pre_dumps = shared.IsTrue(tmp)
- }
- logger.Debugf("migration.incremental.memory %d", use_pre_dumps)
-
- // migration.incremental.memory.iterations is the value after which the
- // container will be definitely migrated, even if the remaining number
- // of memory pages is below the defined threshold.
- // TODO: implement threshold (needs reading of CRIU output files)
- var max_iterations int
- tmp = s.container.ExpandedConfig()["migration.incremental.memory.iterations"]
- if tmp != "" {
- max_iterations, _ = strconv.Atoi(tmp)
- } else {
- // default to 10
- max_iterations = 10
- }
- if max_iterations > 999 {
- // the pre-dump directory is hardcoded to a string
- // with maximal 3 digits. 999 pre-dumps makes no
- // sense at all, but let's make sure the number
- // is not higher than this.
- max_iterations = 999
- }
- logger.Debugf("using maximal %d iterations for pre-dumping", max_iterations)
+ use_pre_dumps, max_iterations := s.checkForPreDumpSupport()
// The protocol says we have to send a header no matter what, so let's
// do that, but then immediately send an error.
From f7399e045a4643eed1e1df3e03f39fde3e3e755f Mon Sep 17 00:00:00 2001
From: Adrian Reber <areber at redhat.com>
Date: Sun, 17 Dec 2017 21:07:48 +0100
Subject: [PATCH 4/4] migration: default to pre-copy migration if CRIU supports
it
Now that everything has been prepared to be able to handle pre-copy
migration detection (lxc, go-lxc, lxd) this is the final step to default
to pre-copy migration if all involved layers support it.
The user can still opt-out of pre-copy migration by setting
'migration.incremental.memory' to 'false'.
Signed-off-by: Adrian Reber <areber at redhat.com>
---
lxd/migrate.go | 35 +++++++++++++++++++++++++----------
1 file changed, 25 insertions(+), 10 deletions(-)
diff --git a/lxd/migrate.go b/lxd/migrate.go
index 4067a184f..f11eeaeb2 100644
--- a/lxd/migrate.go
+++ b/lxd/migrate.go
@@ -360,29 +360,44 @@ func snapshotToProtobuf(c container) *Snapshot {
// Check if CRIU supports pre-dumping and number of
// pre-dump iterations
func (s *migrationSourceWs) checkForPreDumpSupport() (bool, int) {
- // TODO: ask CRIU if this system (kernel+criu) supports
- // pre-copy (dirty memory tracking)
- // The user should also be enable to influence it from the
- // command-line.
+ use_pre_dumps := false
+
+ // Ask CRIU if this architecture/kernel/criu combination
+ // supports pre-copy (dirty memory tracking)
+ criuMigrationArgs := CriuMigrationArgs{
+ cmd: lxc.MIGRATE_FEATURE_CHECK,
+ stateDir: "",
+ function: "feature-check",
+ stop: false,
+ actionScript: false,
+ dumpDir: "",
+ preDumpDir: "",
+ features: lxc.FEATURE_MEM_TRACK,
+ }
+ err := s.container.Migrate(&criuMigrationArgs)
+
+ if err != nil {
+ // CRIU says it does not know about dirty memory tracking.
+ // This means the rest of this function is irrelevant.
+ return false, 0
+ }
+
+ // CRIU says it can actually do pre-dump
+ use_pre_dumps = true
// What does the configuration say about pre-copy
tmp := s.container.ExpandedConfig()["migration.incremental.memory"]
- // default to false for pre-dumps as long as libxlc has no
- // detection for the feature
- use_pre_dumps := false
-
if tmp != "" {
use_pre_dumps = shared.IsTrue(tmp)
}
logger.Debugf("migration.incremental.memory %d", use_pre_dumps)
+ var max_iterations int
// migration.incremental.memory.iterations is the value after which the
// container will be definitely migrated, even if the remaining number
// of memory pages is below the defined threshold.
- // TODO: implement threshold (needs reading of CRIU output files)
- var max_iterations int
tmp = s.container.ExpandedConfig()["migration.incremental.memory.iterations"]
if tmp != "" {
max_iterations, _ = strconv.Atoi(tmp)
More information about the lxc-devel
mailing list