[lxc-devel] [lxd/master] Add custom volume snapshot expiry
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Thu Mar 12 12:40:08 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200312/0b3ddfe6/attachment.bin>
-------------- next part --------------
From af10f7c4658046164c060c5a0ada7f7022e6007a Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 5 Mar 2020 15:02:15 +0100
Subject: [PATCH 1/6] shared/version/api: Add custom_volume_snapshot_expiry
extension
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
shared/version/api.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/shared/version/api.go b/shared/version/api.go
index 3cc3c02ea8..529669d180 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -193,6 +193,7 @@ var APIExtensions = []string{
"container_syscall_intercept_hugetlbfs",
"limits_hugepages",
"container_nic_routed_gateway",
+ "custom_volume_snapshot_expiry",
}
// APIExtensionsCount returns the number of available API extensions.
From 5f9d384f1c2b1258daf9d68a6d1bb8b472e1bdca Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 12 Mar 2020 10:23:48 +0100
Subject: [PATCH 2/6] doc: Add custom_volume_snapshot_expiry
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
doc/api-extensions.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 4ae2accbe9..0d85b8b57a 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -949,3 +949,7 @@ This introduces the `ipv4.gateway` and `ipv6.gateway` NIC config keys that can t
gateway being added inside the container and the same gateway address being added to the host-side interface.
If the value is set to "none" then no default gateway nor will the address be added to the host-side interface.
This allows multiple routed NIC devices to be added to a container.
+
+## custom\_volume\_snapshot\_expiry
+This allows custom volume snapshots to expiry.
+Expiry dates can be set individually, or by setting the `snapshots.expiry` config key on the parent custom volume which then automatically applies to all created snapshots.
From f3326d2ef88fa543d7609197d6369fa0affd670b Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 12 Mar 2020 10:35:55 +0100
Subject: [PATCH 3/6] lxd/db: Add expiry_date to storage_volumes_snapshots
This adds the expiry_date column to the storage_volumes_snapshots table.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/db/cluster/schema.go | 3 +-
lxd/db/cluster/update.go | 7 +++
lxd/db/instances.mapper.go | 102 ++++++++++++++++++-------------------
3 files changed, 60 insertions(+), 52 deletions(-)
diff --git a/lxd/db/cluster/schema.go b/lxd/db/cluster/schema.go
index 5553b3709f..06f9a9a106 100644
--- a/lxd/db/cluster/schema.go
+++ b/lxd/db/cluster/schema.go
@@ -532,6 +532,7 @@ CREATE TABLE storage_volumes_snapshots (
storage_volume_id INTEGER NOT NULL,
name TEXT NOT NULL,
description TEXT,
+ expiry_date DATETIME,
UNIQUE (id),
UNIQUE (storage_volume_id, name),
FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes (id) ON DELETE CASCADE
@@ -552,5 +553,5 @@ CREATE TABLE storage_volumes_snapshots_config (
UNIQUE (storage_volume_snapshot_id, key)
);
-INSERT INTO schema (version, updated_at) VALUES (26, strftime("%s"))
+INSERT INTO schema (version, updated_at) VALUES (27, strftime("%s"))
`
diff --git a/lxd/db/cluster/update.go b/lxd/db/cluster/update.go
index 87d929c20e..2e3fb9bf91 100644
--- a/lxd/db/cluster/update.go
+++ b/lxd/db/cluster/update.go
@@ -63,6 +63,13 @@ var updates = map[int]schema.Update{
24: updateFromV23,
25: updateFromV24,
26: updateFromV25,
+ 27: updateFromV26,
+}
+
+// Add expiry date to storage volume snapshots
+func updateFromV26(tx *sql.Tx) error {
+ _, err := tx.Exec("ALTER TABLE storage_volumes_snapshots ADD COLUMN expiry_date DATETIME;")
+ return err
}
// Create new storage snapshot tables and migrate data to them.
diff --git a/lxd/db/instances.mapper.go b/lxd/db/instances.mapper.go
index bac855fa3d..19e5e9bbb1 100644
--- a/lxd/db/instances.mapper.go
+++ b/lxd/db/instances.mapper.go
@@ -235,6 +235,13 @@ func (c *ClusterTx) InstanceList(filter InstanceFilter) ([]Instance, error) {
filter.Node,
filter.Name,
}
+ } else if criteria["Project"] != nil && criteria["Type"] != nil && criteria["Name"] != nil {
+ stmt = c.stmt(instanceObjectsByProjectAndTypeAndName)
+ args = []interface{}{
+ filter.Project,
+ filter.Type,
+ filter.Name,
+ }
} else if criteria["Project"] != nil && criteria["Name"] != nil && criteria["Node"] != nil {
stmt = c.stmt(instanceObjectsByProjectAndNameAndNode)
args = []interface{}{
@@ -242,12 +249,12 @@ func (c *ClusterTx) InstanceList(filter InstanceFilter) ([]Instance, error) {
filter.Name,
filter.Node,
}
- } else if criteria["Project"] != nil && criteria["Type"] != nil && criteria["Name"] != nil {
- stmt = c.stmt(instanceObjectsByProjectAndTypeAndName)
+ } else if criteria["Project"] != nil && criteria["Type"] != nil && criteria["Node"] != nil {
+ stmt = c.stmt(instanceObjectsByProjectAndTypeAndNode)
args = []interface{}{
filter.Project,
filter.Type,
- filter.Name,
+ filter.Node,
}
} else if criteria["Type"] != nil && criteria["Name"] != nil && criteria["Node"] != nil {
stmt = c.stmt(instanceObjectsByTypeAndNameAndNode)
@@ -256,17 +263,16 @@ func (c *ClusterTx) InstanceList(filter InstanceFilter) ([]Instance, error) {
filter.Name,
filter.Node,
}
- } else if criteria["Project"] != nil && criteria["Type"] != nil && criteria["Node"] != nil {
- stmt = c.stmt(instanceObjectsByProjectAndTypeAndNode)
+ } else if criteria["Project"] != nil && criteria["Type"] != nil {
+ stmt = c.stmt(instanceObjectsByProjectAndType)
args = []interface{}{
filter.Project,
filter.Type,
- filter.Node,
}
- } else if criteria["Project"] != nil && criteria["Name"] != nil {
- stmt = c.stmt(instanceObjectsByProjectAndName)
+ } else if criteria["Type"] != nil && criteria["Name"] != nil {
+ stmt = c.stmt(instanceObjectsByTypeAndName)
args = []interface{}{
- filter.Project,
+ filter.Type,
filter.Name,
}
} else if criteria["Project"] != nil && criteria["Node"] != nil {
@@ -275,17 +281,17 @@ func (c *ClusterTx) InstanceList(filter InstanceFilter) ([]Instance, error) {
filter.Project,
filter.Node,
}
- } else if criteria["Type"] != nil && criteria["Name"] != nil {
- stmt = c.stmt(instanceObjectsByTypeAndName)
+ } else if criteria["Project"] != nil && criteria["Name"] != nil {
+ stmt = c.stmt(instanceObjectsByProjectAndName)
args = []interface{}{
- filter.Type,
+ filter.Project,
filter.Name,
}
- } else if criteria["Project"] != nil && criteria["Type"] != nil {
- stmt = c.stmt(instanceObjectsByProjectAndType)
+ } else if criteria["Type"] != nil && criteria["Node"] != nil {
+ stmt = c.stmt(instanceObjectsByTypeAndNode)
args = []interface{}{
- filter.Project,
filter.Type,
+ filter.Node,
}
} else if criteria["Node"] != nil && criteria["Name"] != nil {
stmt = c.stmt(instanceObjectsByNodeAndName)
@@ -293,11 +299,15 @@ func (c *ClusterTx) InstanceList(filter InstanceFilter) ([]Instance, error) {
filter.Node,
filter.Name,
}
- } else if criteria["Type"] != nil && criteria["Node"] != nil {
- stmt = c.stmt(instanceObjectsByTypeAndNode)
+ } else if criteria["Type"] != nil {
+ stmt = c.stmt(instanceObjectsByType)
args = []interface{}{
filter.Type,
- filter.Node,
+ }
+ } else if criteria["Name"] != nil {
+ stmt = c.stmt(instanceObjectsByName)
+ args = []interface{}{
+ filter.Name,
}
} else if criteria["Project"] != nil {
stmt = c.stmt(instanceObjectsByProject)
@@ -309,16 +319,6 @@ func (c *ClusterTx) InstanceList(filter InstanceFilter) ([]Instance, error) {
args = []interface{}{
filter.Node,
}
- } else if criteria["Type"] != nil {
- stmt = c.stmt(instanceObjectsByType)
- args = []interface{}{
- filter.Type,
- }
- } else if criteria["Name"] != nil {
- stmt = c.stmt(instanceObjectsByName)
- args = []interface{}{
- filter.Name,
- }
} else {
stmt = c.stmt(instanceObjects)
args = []interface{}{}
@@ -595,16 +595,16 @@ func (c *ClusterTx) InstanceProfilesRef(filter InstanceFilter) (map[string]map[s
filter.Project,
filter.Name,
}
- } else if criteria["Node"] != nil {
- stmt = c.stmt(instanceProfilesRefByNode)
- args = []interface{}{
- filter.Node,
- }
} else if criteria["Project"] != nil {
stmt = c.stmt(instanceProfilesRefByProject)
args = []interface{}{
filter.Project,
}
+ } else if criteria["Node"] != nil {
+ stmt = c.stmt(instanceProfilesRefByNode)
+ args = []interface{}{
+ filter.Node,
+ }
} else {
stmt = c.stmt(instanceProfilesRef)
args = []interface{}{}
@@ -674,21 +674,16 @@ func (c *ClusterTx) InstanceConfigRef(filter InstanceFilter) (map[string]map[str
var stmt *sql.Stmt
var args []interface{}
- if criteria["Project"] != nil && criteria["Node"] != nil {
- stmt = c.stmt(instanceConfigRefByProjectAndNode)
- args = []interface{}{
- filter.Project,
- filter.Node,
- }
- } else if criteria["Project"] != nil && criteria["Name"] != nil {
+ if criteria["Project"] != nil && criteria["Name"] != nil {
stmt = c.stmt(instanceConfigRefByProjectAndName)
args = []interface{}{
filter.Project,
filter.Name,
}
- } else if criteria["Node"] != nil {
- stmt = c.stmt(instanceConfigRefByNode)
+ } else if criteria["Project"] != nil && criteria["Node"] != nil {
+ stmt = c.stmt(instanceConfigRefByProjectAndNode)
args = []interface{}{
+ filter.Project,
filter.Node,
}
} else if criteria["Project"] != nil {
@@ -696,6 +691,11 @@ func (c *ClusterTx) InstanceConfigRef(filter InstanceFilter) (map[string]map[str
args = []interface{}{
filter.Project,
}
+ } else if criteria["Node"] != nil {
+ stmt = c.stmt(instanceConfigRefByNode)
+ args = []interface{}{
+ filter.Node,
+ }
} else {
stmt = c.stmt(instanceConfigRef)
args = []interface{}{}
@@ -770,21 +770,16 @@ func (c *ClusterTx) InstanceDevicesRef(filter InstanceFilter) (map[string]map[st
var stmt *sql.Stmt
var args []interface{}
- if criteria["Project"] != nil && criteria["Node"] != nil {
- stmt = c.stmt(instanceDevicesRefByProjectAndNode)
- args = []interface{}{
- filter.Project,
- filter.Node,
- }
- } else if criteria["Project"] != nil && criteria["Name"] != nil {
+ if criteria["Project"] != nil && criteria["Name"] != nil {
stmt = c.stmt(instanceDevicesRefByProjectAndName)
args = []interface{}{
filter.Project,
filter.Name,
}
- } else if criteria["Node"] != nil {
- stmt = c.stmt(instanceDevicesRefByNode)
+ } else if criteria["Project"] != nil && criteria["Node"] != nil {
+ stmt = c.stmt(instanceDevicesRefByProjectAndNode)
args = []interface{}{
+ filter.Project,
filter.Node,
}
} else if criteria["Project"] != nil {
@@ -792,6 +787,11 @@ func (c *ClusterTx) InstanceDevicesRef(filter InstanceFilter) (map[string]map[st
args = []interface{}{
filter.Project,
}
+ } else if criteria["Node"] != nil {
+ stmt = c.stmt(instanceDevicesRefByNode)
+ args = []interface{}{
+ filter.Node,
+ }
} else {
stmt = c.stmt(instanceDevicesRef)
args = []interface{}{}
From b7870c9ca743a441d8f77f77728dcdffb9f40630 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 12 Mar 2020 12:55:09 +0100
Subject: [PATCH 4/6] shared/api: Add expiry fields to StorageVolumeSnapshot*
This adds an ExpiresAt field to both StorageVolumeSnapshotsPost and
StorageVolumeSnapshotPut, the same way it's done for Instances.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
shared/api/storage_pool_volume_snapshot.go | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/shared/api/storage_pool_volume_snapshot.go b/shared/api/storage_pool_volume_snapshot.go
index 4ba21da05d..11d3b5bb7a 100644
--- a/shared/api/storage_pool_volume_snapshot.go
+++ b/shared/api/storage_pool_volume_snapshot.go
@@ -1,10 +1,15 @@
package api
+import "time"
+
// StorageVolumeSnapshotsPost represents the fields available for a new LXD storage volume snapshot
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotsPost struct {
Name string `json:"name" yaml:"name"`
+
+ // API extension: custom_volume_snapshot_expiry
+ ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"`
}
// StorageVolumeSnapshotPost represents the fields required to rename/move a LXD storage volume snapshot
@@ -18,9 +23,10 @@ type StorageVolumeSnapshotPost struct {
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshot struct {
- Name string `json:"name" yaml:"name"`
- Config map[string]string `json:"config" yaml:"config"`
- Description string `json:"description" yaml:"description"`
+ StorageVolumeSnapshotPut `json:",inline", yaml:",inline"`
+
+ Name string `json:"name" yaml:"name"`
+ Config map[string]string `json:"config" yaml:"config"`
}
// StorageVolumeSnapshotPut represents the modifiable fields of a LXD storage volume
@@ -28,4 +34,7 @@ type StorageVolumeSnapshot struct {
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotPut struct {
Description string `json:"description" yaml:"description"`
+
+ // API extension: custom_volume_snapshot_expiry
+ ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"`
}
From 303c6f897a55d2a79db751a3277ac8ccba354a7c Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 12 Mar 2020 12:59:43 +0100
Subject: [PATCH 5/6] lxd/storage: Add expiry to volume snapshot pool functions
This adds an expiry field to both CreateCustomVolumeSnapshot and
UpdateCustomVolumeSnapshot.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage/backend_lxd.go | 5 +++--
lxd/storage/backend_mock.go | 5 +++--
lxd/storage/pool_interface.go | 5 +++--
lxd/storage_volumes_snapshot.go | 5 +++--
4 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/lxd/storage/backend_lxd.go b/lxd/storage/backend_lxd.go
index 8e22694520..8ca12dfb28 100644
--- a/lxd/storage/backend_lxd.go
+++ b/lxd/storage/backend_lxd.go
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"strings"
+ "time"
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"
@@ -2547,7 +2548,7 @@ func (b *lxdBackend) UpdateCustomVolume(projectName string, volName string, newD
// UpdateCustomVolumeSnapshot updates the description of a custom volume snapshot.
// Volume config is not allowd to be updated and will return an error.
-func (b *lxdBackend) UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, op *operations.Operation) error {
+func (b *lxdBackend) UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, newExpiryDate time.Time, op *operations.Operation) error {
logger := logging.AddContext(b.logger, log.Ctx{"project": projectName, "volName": volName, "newDesc": newDesc, "newConfig": newConfig})
logger.Debug("UpdateCustomVolumeSnapshot started")
defer logger.Debug("UpdateCustomVolumeSnapshot finished")
@@ -2653,7 +2654,7 @@ func (b *lxdBackend) UnmountCustomVolume(projectName, volName string, op *operat
}
// CreateCustomVolumeSnapshot creates a snapshot of a custom volume.
-func (b *lxdBackend) CreateCustomVolumeSnapshot(projectName, volName string, newSnapshotName string, op *operations.Operation) error {
+func (b *lxdBackend) CreateCustomVolumeSnapshot(projectName, volName string, newSnapshotName string, newExpiryDate time.Time, op *operations.Operation) error {
logger := logging.AddContext(b.logger, log.Ctx{"project": projectName, "volName": volName, "newSnapshotName": newSnapshotName})
logger.Debug("CreateCustomVolumeSnapshot started")
defer logger.Debug("CreateCustomVolumeSnapshot finished")
diff --git a/lxd/storage/backend_mock.go b/lxd/storage/backend_mock.go
index 5823c333b2..5378221656 100644
--- a/lxd/storage/backend_mock.go
+++ b/lxd/storage/backend_mock.go
@@ -2,6 +2,7 @@ package storage
import (
"io"
+ "time"
"github.com/lxc/lxd/lxd/backup"
"github.com/lxc/lxd/lxd/instance"
@@ -217,7 +218,7 @@ func (b *mockBackend) UnmountCustomVolume(projectName string, volName string, op
return true, nil
}
-func (b *mockBackend) CreateCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, op *operations.Operation) error {
+func (b *mockBackend) CreateCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, expiryDate time.Time, op *operations.Operation) error {
return nil
}
@@ -229,7 +230,7 @@ func (b *mockBackend) DeleteCustomVolumeSnapshot(projectName string, volName str
return nil
}
-func (b *mockBackend) UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, op *operations.Operation) error {
+func (b *mockBackend) UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, expiryDate time.Time, op *operations.Operation) error {
return nil
}
diff --git a/lxd/storage/pool_interface.go b/lxd/storage/pool_interface.go
index ffacb2a455..cc2d14d034 100644
--- a/lxd/storage/pool_interface.go
+++ b/lxd/storage/pool_interface.go
@@ -2,6 +2,7 @@ package storage
import (
"io"
+ "time"
"github.com/lxc/lxd/lxd/backup"
"github.com/lxc/lxd/lxd/instance"
@@ -75,10 +76,10 @@ type Pool interface {
UnmountCustomVolume(projectName string, volName string, op *operations.Operation) (bool, error)
// Custom volume snapshots.
- CreateCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, op *operations.Operation) error
+ CreateCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, newExpiryDate time.Time, op *operations.Operation) error
RenameCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, op *operations.Operation) error
DeleteCustomVolumeSnapshot(projectName string, volName string, op *operations.Operation) error
- UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, op *operations.Operation) error
+ UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, newExpiryDate time.Time, op *operations.Operation) error
RestoreCustomVolume(projectName string, volName string, snapshotName string, op *operations.Operation) error
// Custom volume migration.
diff --git a/lxd/storage_volumes_snapshot.go b/lxd/storage_volumes_snapshot.go
index 1c4139ff9d..e9780933bc 100644
--- a/lxd/storage_volumes_snapshot.go
+++ b/lxd/storage_volumes_snapshot.go
@@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"strings"
+ "time"
"github.com/gorilla/mux"
@@ -122,7 +123,7 @@ func storagePoolVolumeSnapshotsTypePost(d *Daemon, r *http.Request) response.Res
return err
}
- return pool.CreateCustomVolumeSnapshot(projectName, volumeName, req.Name, op)
+ return pool.CreateCustomVolumeSnapshot(projectName, volumeName, req.Name, time.Unix(0, 0), op)
}
resources := map[string][]string{}
@@ -430,7 +431,7 @@ func storagePoolVolumeSnapshotTypePut(d *Daemon, r *http.Request) response.Respo
}
// Handle custom volume update requests.
- return pool.UpdateCustomVolumeSnapshot(projectName, vol.Name, req.Description, nil, op)
+ return pool.UpdateCustomVolumeSnapshot(projectName, vol.Name, req.Description, nil, time.Unix(0, 0), op)
}
resources := map[string][]string{}
From 3c6cf6b1a670f291deb234b5e5a8dd88ce04d0bc Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 12 Mar 2020 13:26:53 +0100
Subject: [PATCH 6/6] lxd: Add snapshots.expiry config key for storage pools
This adds the snapshots.expiry config key to storage pools. The key
needs to be set per pool.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/storage_pools_config.go | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index 71195b7e5a..0e63dfc6d2 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
+ "time"
"golang.org/x/sys/unix"
@@ -52,6 +53,13 @@ var storagePoolConfigKeys = map[string]func(value string) error{
// valid drivers: btrfs, lvm, zfs
"size": shared.IsSize,
+ // valid drivers: all
+ "snapshots.expiry": func(value string) error {
+ // Validate expression
+ _, err := shared.GetSnapshotExpiry(time.Time{}, value)
+ return err
+ },
+
// valid drivers: btrfs, dir, lvm, zfs
"source": shared.IsAny,
More information about the lxc-devel
mailing list