[lxc-devel] [lxd/master] Snapshot scheduling
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Wed Nov 14 01:28:47 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 321 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20181114/290bf109/attachment.bin>
-------------- next part --------------
From 9f0ee9422ca1f01016435d2d2bfcf84fd6677da0 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 13 Nov 2018 19:36:34 +0100
Subject: [PATCH 1/4] db: Find next snapshot with prefix
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/db/containers.go | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
diff --git a/lxd/db/containers.go b/lxd/db/containers.go
index dcca00e342..9df6ed379c 100644
--- a/lxd/db/containers.go
+++ b/lxd/db/containers.go
@@ -873,13 +873,10 @@ WHERE projects.name=? AND containers.type=? AND SUBSTR(containers.name,1,?)=?
return result, nil
}
-// ContainerNextSnapshot returns the index the next snapshot of the container
-// with the given name should have.
-//
-// Note, the code below doesn't deal with snapshots of snapshots.
-// To do that, we'll need to weed out based on # slashes in names
-func (c *Cluster) ContainerNextSnapshot(project string, name string) int {
- base := name + shared.SnapshotDelimiter + "snap"
+// ContainerNextSnapshotWithPrefix returns the index the next snapshot of the container
+// with the given name and prefix should have.
+func (c *Cluster) ContainerNextSnapshotWithPrefix(project string, name string, prefix string) int {
+ base := name + shared.SnapshotDelimiter + prefix
length := len(base)
q := `
SELECT containers.name
@@ -914,6 +911,15 @@ SELECT containers.name
return max
}
+// ContainerNextSnapshot returns the index the next snapshot of the container
+// with the given name should have.
+//
+// Note, the code below doesn't deal with snapshots of snapshots.
+// To do that, we'll need to weed out based on # slashes in names
+func (c *Cluster) ContainerNextSnapshot(project string, name string) int {
+ return c.ContainerNextSnapshotWithPrefix(project, name, "snap")
+}
+
// ContainerPool returns the storage pool of a given container.
//
// This is a non-transactional variant of ClusterTx.ContainerPool().
From fa55643bdf556d1d37de64aac50246ebb1bac1d3 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 14 Nov 2018 02:26:20 +0100
Subject: [PATCH 2/4] shared: Add pongo2 render function
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
shared/util.go | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/shared/util.go b/shared/util.go
index 85706702f5..6641e16d05 100644
--- a/shared/util.go
+++ b/shared/util.go
@@ -26,6 +26,7 @@ import (
"strings"
"time"
+ "github.com/flosch/pongo2"
"github.com/lxc/lxd/shared/cancel"
"github.com/lxc/lxd/shared/ioprogress"
"github.com/pkg/errors"
@@ -1104,3 +1105,25 @@ func (r *ReadSeeker) Read(p []byte) (n int, err error) {
func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
return r.Seeker.Seek(offset, whence)
}
+
+// RenderTemplate renders a pongo2 template.
+func RenderTemplate(template string, ctx pongo2.Context) (string, error) {
+ // Load template from string
+ tpl, err := pongo2.FromString("{% autoescape off %}" + template + "{% endautoescape %}")
+ if err != nil {
+ return "", err
+ }
+
+ // Get rendered template
+ ret, err := tpl.Execute(ctx)
+ if err != nil {
+ return ret, err
+ }
+
+ // Looks like we're nesting templates so run pongo again
+ if strings.Contains(ret, "{{") || strings.Contains(ret, "{%") {
+ return RenderTemplate(ret, ctx)
+ }
+
+ return ret, err
+}
From b87a5edfa2af729aa08f4abe34e3b7655df844d3 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 9 Aug 2018 16:01:55 +0200
Subject: [PATCH 3/4] shared/container: Add snapshots.* config keys
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
shared/container.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/shared/container.go b/shared/container.go
index 0996a49138..47f2f75a88 100644
--- a/shared/container.go
+++ b/shared/container.go
@@ -263,6 +263,10 @@ var KnownContainerConfigKeys = map[string]func(value string) error{
"security.syscalls.blacklist": IsAny,
"security.syscalls.whitelist": IsAny,
+ "snapshots.schedule": IsAny,
+ "snapshots.schedule.stopped": IsBool,
+ "snapshots.pattern": IsAny,
+
// Caller is responsible for full validation of any raw.* value
"raw.apparmor": IsAny,
"raw.lxc": IsAny,
From fe35c3f197236ec54c96cfd2d37454049e12ced9 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 17 Aug 2018 07:33:40 +0200
Subject: [PATCH 4/4] lxd: Support container snapshot scheduling
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/container.go | 167 +++++++++++++++++++++++++++++++++++++++++++++++
lxd/daemon.go | 3 +
2 files changed, 170 insertions(+)
diff --git a/lxd/container.go b/lxd/container.go
index aca66cddf3..cdbabc11c9 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -9,17 +9,22 @@ import (
"strings"
"time"
+ "golang.org/x/net/context"
"gopkg.in/lxc/go-lxc.v2"
+ "gopkg.in/robfig/cron.v2"
+ "github.com/flosch/pongo2"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/sys"
+ "github.com/lxc/lxd/lxd/task"
"github.com/lxc/lxd/lxd/types"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/idmap"
+ log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/lxc/lxd/shared/osarch"
"github.com/pkg/errors"
@@ -1531,3 +1536,165 @@ func containerCompareSnapshots(source container, target container) ([]container,
return toSync, toDelete, nil
}
+
+func autoCreateContainerSnapshotsTask(d *Daemon) (task.Func, task.Schedule) {
+ f := func(ctx context.Context) {
+ opRun := func(op *operation) error {
+ return autoCreateContainerSnapshots(ctx, d)
+ }
+
+ op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationSnapshotCreate, nil, nil, opRun, nil, nil)
+ if err != nil {
+ logger.Error("Failed to start create snapshot operation", log.Ctx{"err": err})
+ }
+
+ logger.Info("Creating scheduled container snapshots")
+
+ _, err = op.Run()
+ if err != nil {
+ logger.Error("Failed to create scheduled container snapshots", log.Ctx{"err": err})
+ }
+
+ logger.Info("Done creating scheduled container snapshots")
+ }
+
+ f(context.Background())
+
+ first := true
+ schedule := func() (time.Duration, error) {
+ interval := time.Hour
+
+ if first {
+ first = false
+ return interval, task.ErrSkip
+ }
+
+ return interval, nil
+ }
+
+ return f, schedule
+}
+
+func autoCreateContainerSnapshots(ctx context.Context, d *Daemon) error {
+ logger.Info("Creating scheduled container snapshots")
+
+ containers, err := d.cluster.ContainersNodeList(db.CTypeRegular)
+ if err != nil {
+ return errors.Wrap(err, "Unable to retrieve the list of containers")
+ }
+
+ for _, container := range containers {
+ cId, err := d.cluster.ContainerID(container)
+
+ c, err := containerLoadById(d.State(), cId)
+ if err != nil {
+ return errors.Wrap(err, "Error loading container")
+ }
+
+ schedule := c.LocalConfig()["snapshots.schedule"]
+
+ if len(strings.Split(schedule, " ")) != 4 {
+ logger.Error("Schedule must be of the form: <hour> <day-of-month> <month> <day-of-week>")
+ continue
+ }
+
+ if schedule == "" || (!shared.IsTrue(c.LocalConfig()["snapshots.schedule.stopped"]) && c.IsRunning()) {
+ continue
+ }
+
+ // Extend our schedule to one that is accepted by the used cron parser
+ sched, err := cron.Parse(fmt.Sprintf("* * %s", schedule))
+ if err != nil {
+ logger.Error("Error parsing schedule", log.Ctx{"err": err,
+ "container": container, "schedule": schedule})
+ continue
+ }
+
+ now := time.Now()
+ next := sched.Next(now).Format("2006-01-02T15")
+ skip := false
+
+ pattern := c.LocalConfig()["snapshots.pattern"]
+ if pattern == "" {
+ pattern = "auto-snapshot"
+ }
+
+ pattern, err = shared.RenderTemplate(pattern, pongo2.Context{
+ "creation_date": now,
+ })
+ if err != nil {
+ logger.Error("Error rendering template", log.Ctx{"err": err,
+ "container": container, "template": c.LocalConfig()["snapshots.pattern"]})
+ continue
+ }
+
+ snapshots, err := c.Snapshots()
+ if err != nil {
+ logger.Error("Error retrieving snapshots", log.Ctx{"err": err,
+ "container": container})
+ continue
+ }
+
+ snapshotExists := false
+
+ for _, snap := range snapshots {
+ _, snapshotOnlyName, _ := containerGetParentAndSnapshotName(snap.Name())
+ // Check whether the container has been snapshotted within (at least)
+ // the last hour.
+ if strings.HasPrefix(snapshotOnlyName, pattern) && snap.CreationDate().Format("2006-01-02T15") == next {
+ skip = true
+ break
+ }
+
+ if snap.Name() == pattern {
+ snapshotExists = true
+ }
+ }
+
+ // Skip snapshotting if the container has been snapshotted within (at
+ // least) the last hour, or it's not time yet.
+ if skip || now.Format("2006-01-02T15") != next {
+ continue
+ }
+
+ ch := make(chan struct{})
+ go func() {
+ var snapshotName string
+
+ if snapshotExists {
+ snapshotName = fmt.Sprintf("%s%s%s-%d",
+ c.Name(),
+ shared.SnapshotDelimiter,
+ pattern,
+ d.cluster.ContainerNextSnapshotWithPrefix(c.Project(), c.Name(), pattern))
+ } else {
+ snapshotName = fmt.Sprintf("%s%s%s",
+ c.Name(),
+ shared.SnapshotDelimiter,
+ pattern)
+ }
+
+ args := db.ContainerArgs{
+ Project: c.Project(),
+ Architecture: c.Architecture(),
+ Config: c.LocalConfig(),
+ Ctype: db.CTypeSnapshot,
+ Devices: c.LocalDevices(),
+ Ephemeral: c.IsEphemeral(),
+ Name: snapshotName,
+ Profiles: c.Profiles(),
+ Stateful: false,
+ }
+
+ containerCreateAsSnapshot(d.State(), args, c)
+ ch <- struct{}{}
+ }()
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-ch:
+ }
+ }
+
+ return nil
+}
diff --git a/lxd/daemon.go b/lxd/daemon.go
index f4201ae885..f0a78e6e74 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -812,6 +812,9 @@ func (d *Daemon) Ready() error {
// Remove expired container backups (hourly)
d.tasks.Add(pruneExpiredContainerBackupsTask(d))
+
+ // Take snapshot of containers (hourly check of configurable cron expression)
+ d.tasks.Add(autoCreateContainerSnapshotsTask(d))
}
// Start all background tasks
More information about the lxc-devel
mailing list