[lxc-devel] [lxd/master] Fix handling of image expiry
stgraber on Github
lxc-bot at linuxcontainers.org
Thu Oct 5 23:06:18 UTC 2017
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20171005/e05e2612/attachment.bin>
-------------- next part --------------
From 0dd52f7aeb357b81b5e2a3fc6f0397198968c892 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 17:05:01 -0400
Subject: [PATCH 01/10] tests: Fix shell return value masking
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
test/suites/basic.sh | 12 ++++++------
test/suites/concurrent.sh | 2 +-
test/suites/config.sh | 6 +++---
test/suites/deps.sh | 2 +-
test/suites/image.sh | 2 +-
5 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/test/suites/basic.sh b/test/suites/basic.sh
index f51bc378b..cb7d147ae 100644
--- a/test/suites/basic.sh
+++ b/test/suites/basic.sh
@@ -370,11 +370,11 @@ test_basic_usage() {
aa_namespace="lxd-lxd-apparmor-test_<$(echo "${LXD_DIR}" | sed -e 's/\//-/g' -e 's/^.//')>"
aa-status | grep -q ":${aa_namespace}:unconfined" || aa-status | grep -q ":${aa_namespace}://unconfined"
lxc stop lxd-apparmor-test --force
- ! aa-status | grep -q ":${aa_namespace}:"
+ ! aa-status | grep -q ":${aa_namespace}:" || false
else
aa-status | grep "lxd-lxd-apparmor-test_<${LXD_DIR}>"
lxc stop lxd-apparmor-test --force
- ! aa-status | grep -q "lxd-lxd-apparmor-test_<${LXD_DIR}>"
+ ! aa-status | grep -q "lxd-lxd-apparmor-test_<${LXD_DIR}>" || false
fi
lxc delete lxd-apparmor-test
[ ! -f "${LXD_DIR}/security/apparmor/profiles/lxd-lxd-apparmor-test" ]
@@ -420,9 +420,9 @@ test_basic_usage() {
lxc publish --force c3 --alias=image3
# Delete multiple images with lxc delete and confirm they're deleted
lxc image delete local:image1 local:image2 local:image3
- ! lxc image list | grep -q image1
- ! lxc image list | grep -q image2
- ! lxc image list | grep -q image3
+ ! lxc image list | grep -q image1 || false
+ ! lxc image list | grep -q image2 || false
+ ! lxc image list | grep -q image3 || false
# Cleanup the containers
lxc delete --force c1 c2 c3
@@ -453,5 +453,5 @@ test_basic_usage() {
sleep 2
lxc stop foo --force || true
- ! lxc list | grep -q foo
+ ! lxc list | grep -q foo || false
}
diff --git a/test/suites/concurrent.sh b/test/suites/concurrent.sh
index 107f4f08c..f0ec2884b 100644
--- a/test/suites/concurrent.sh
+++ b/test/suites/concurrent.sh
@@ -29,5 +29,5 @@ test_concurrent() {
wait "${pid}"
done
- ! lxc list | grep -q concurrent
+ ! lxc list | grep -q concurrent || false
}
diff --git a/test/suites/config.sh b/test/suites/config.sh
index 3231bd431..526378dfc 100644
--- a/test/suites/config.sh
+++ b/test/suites/config.sh
@@ -161,9 +161,9 @@ test_config_profiles() {
# test live-adding a nic
lxc start foo
lxc exec foo -- cat /proc/self/mountinfo | grep -q "/mnt1.*ro,"
- ! lxc config show foo | grep -q "raw.lxc"
+ ! lxc config show foo | grep -q "raw.lxc" || false
lxc config show foo --expanded | grep -q "raw.lxc"
- ! lxc config show foo | grep -v "volatile.eth0" | grep -q "eth0"
+ ! lxc config show foo | grep -v "volatile.eth0" | grep -q "eth0" || false
lxc config show foo --expanded | grep -v "volatile.eth0" | grep -q "eth0"
lxc config device add foo eth2 nic nictype=p2p name=eth10
lxc exec foo -- /sbin/ifconfig -a | grep eth0
@@ -288,7 +288,7 @@ test_container_metadata() {
# templates can be removed
lxc config template delete c my.tpl
- ! (lxc config template list c | grep -q my.tpl)
+ ! lxc config template list c | grep -q my.tpl || false
lxc delete c
}
diff --git a/test/suites/deps.sh b/test/suites/deps.sh
index 2ec30a983..69a1c879b 100644
--- a/test/suites/deps.sh
+++ b/test/suites/deps.sh
@@ -1,3 +1,3 @@
test_check_deps() {
- ! ldd "$(which lxc)" | grep -q liblxc
+ ! ldd "$(which lxc)" | grep -q liblxc || false
}
diff --git a/test/suites/image.sh b/test/suites/image.sh
index 62fa5019a..804cc84ad 100644
--- a/test/suites/image.sh
+++ b/test/suites/image.sh
@@ -26,7 +26,7 @@ test_image_expiry() {
lxc_remote config set images.remote_cache_expiry 0
lxc_remote remote set-default local
- ! lxc_remote image list l2: | grep -q "${fpbrief}"
+ ! lxc_remote image list l2: | grep -q "${fpbrief}" || false
lxc_remote delete l2:c1
From d18836e133ef0b634e7659602756f2c2fcd9756e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 17:21:22 -0400
Subject: [PATCH 02/10] images: Respect disabled cache expiry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/images.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lxd/images.go b/lxd/images.go
index eb3a3dabf..58b185d04 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -1010,6 +1010,10 @@ func pruneExpiredImages(d *Daemon) {
// Get the list of expired images.
expiry := daemonConfig["images.remote_cache_expiry"].GetInt64()
+ if expiry <= 0 {
+ return
+ }
+
images, err := db.ImagesGetExpired(d.db, expiry)
if err != nil {
logger.Error("Unable to retrieve the list of expired images", log.Ctx{"err": err})
From 45f86043b4614bf0d9eb7e53191e5b3595c0ae69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 17:38:29 -0400
Subject: [PATCH 03/10] images: Store UploadedAt as RFC3399
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/db/images.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/db/images.go b/lxd/db/images.go
index b2d259925..9f6d00a62 100644
--- a/lxd/db/images.go
+++ b/lxd/db/images.go
@@ -462,14 +462,14 @@ func ImageInsert(db *sql.DB, fp string, fname string, sz int64, public bool, aut
autoUpdateInt = 1
}
- stmt, err := tx.Prepare(`INSERT INTO images (fingerprint, filename, size, public, auto_update, architecture, creation_date, expiry_date, upload_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, strftime("%s"))`)
+ stmt, err := tx.Prepare(`INSERT INTO images (fingerprint, filename, size, public, auto_update, architecture, creation_date, expiry_date, upload_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
tx.Rollback()
return err
}
defer stmt.Close()
- result, err := stmt.Exec(fp, fname, sz, publicInt, autoUpdateInt, arch, createdAt, expiresAt)
+ result, err := stmt.Exec(fp, fname, sz, publicInt, autoUpdateInt, arch, createdAt, expiresAt, time.Now().UTC())
if err != nil {
tx.Rollback()
return err
From f29baf9f1cfed4dff19bb3b0614574b78fcdb4e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 17:48:24 -0400
Subject: [PATCH 04/10] patches: Convert UploadedAt to RFC3399
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/patches.go | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/lxd/patches.go b/lxd/patches.go
index 2c630b10f..6f2c89777 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -49,6 +49,7 @@ var patches = []patch{
{name: "storage_zfs_volume_size", run: patchStorageZFSVolumeSize},
{name: "network_dnsmasq_hosts", run: patchNetworkDnsmasqHosts},
{name: "storage_api_dir_bind_mount", run: patchStorageApiDirBindMount},
+ {name: "fix_uploaded_at", run: patchFixUploadedAt},
}
type patch struct {
@@ -2475,6 +2476,27 @@ func patchStorageApiDirBindMount(name string, d *Daemon) error {
return nil
}
+func patchFixUploadedAt(name string, d *Daemon) error {
+ images, err := db.ImagesGet(d.db, false)
+ if err != nil {
+ return err
+ }
+
+ for _, fingerprint := range images {
+ id, image, err := db.ImageGet(d.db, fingerprint, false, true)
+ if err != nil {
+ return err
+ }
+
+ _, err = db.Exec(d.db, "UPDATE images SET upload_date=? WHERE id=?", image.UploadedAt, id)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
// Patches end here
// Here are a couple of legacy patches that were originally in
From eeafcab4bb19f06d1f674a16026ee9d10d8807e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 18:05:51 -0400
Subject: [PATCH 05/10] tests: Fix image expiry test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
test/suites/image.sh | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/test/suites/image.sh b/test/suites/image.sh
index 804cc84ad..1b05cfbc5 100644
--- a/test/suites/image.sh
+++ b/test/suites/image.sh
@@ -12,29 +12,30 @@ test_image_expiry() {
# shellcheck disable=2153
lxc_remote remote add l1 "${LXD_ADDR}" --accept-certificate --password foo
fi
+
if ! lxc_remote remote list | grep -q l2; then
lxc_remote remote add l2 "${LXD2_ADDR}" --accept-certificate --password foo
fi
+
+ # Create a container from a remote image
lxc_remote init l1:testimage l2:c1
fp=$(lxc_remote image info testimage | awk -F: '/^Fingerprint/ { print $2 }' | awk '{ print $1 }')
+
+ # Confirm the image is cached
[ ! -z "${fp}" ]
fpbrief=$(echo "${fp}" | cut -c 1-10)
-
lxc_remote image list l2: | grep -q "${fpbrief}"
- lxc_remote remote set-default l2
- lxc_remote config set images.remote_cache_expiry 0
- lxc_remote remote set-default local
+ # Override the upload date
+ sqlite3 "${LXD2_DIR}/lxd.db" "UPDATE images SET last_use_date='$(date --rfc-3339=seconds -u -d "2 days ago")' WHERE fingerprint='${fp}'"
+ # Trigger the expiry
+ lxc_remote config set l2: images.remote_cache_expiry 1
! lxc_remote image list l2: | grep -q "${fpbrief}" || false
+ # Cleanup and reset
lxc_remote delete l2:c1
-
- # reset the default expiry
- lxc_remote remote set-default l2
- lxc_remote config set images.remote_cache_expiry 10
- lxc_remote remote set-default local
-
+ lxc_remote config set l2: images.remote_cache_expiry 10
lxc_remote remote remove l2
kill_lxd "$LXD2_DIR"
}
From e76c6dbc88ed0fd88f109c2507e079b0397612ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 18:23:19 -0400
Subject: [PATCH 06/10] daemon: Simplify time channels
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/daemon.go | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 0962fec52..2843ff3b6 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -806,9 +806,8 @@ func (d *Daemon) Ready() error {
pruneExpiredImages(d)
for {
timer := time.NewTimer(24 * time.Hour)
- timeChan := timer.C
select {
- case <-timeChan:
+ case <-timer.C:
/* run once per day */
pruneExpiredImages(d)
case <-d.pruneChan:
@@ -833,10 +832,9 @@ func (d *Daemon) Ready() error {
interval := daemonConfig["images.auto_update_interval"].GetInt64()
if interval > 0 {
timer := time.NewTimer(time.Duration(interval) * time.Hour)
- timeChan := timer.C
select {
- case <-timeChan:
+ case <-timer.C:
autoUpdateImages(d)
case <-d.resetAutoUpdateChan:
timer.Stop()
From 8e91ef392551cae59948c0376b45b661a62b8b26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 18:23:43 -0400
Subject: [PATCH 07/10] daemon: Fix handling of config triggers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/daemon_config.go | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lxd/daemon_config.go b/lxd/daemon_config.go
index 35b63220f..6b5ad7381 100644
--- a/lxd/daemon_config.go
+++ b/lxd/daemon_config.go
@@ -133,6 +133,11 @@ func (k *daemonConfigKey) Set(d *Daemon, value string) error {
return err
}
+ // Run the trigger (if any)
+ if k.trigger != nil {
+ k.trigger(d, k.name(), value)
+ }
+
return nil
}
From 941e0b60ccb4dbb45370902ac7eba20f9b1ecc43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 18:43:50 -0400
Subject: [PATCH 08/10] daemon: Don't update images while pruning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/daemon.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 2843ff3b6..aefa007b2 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -803,7 +803,6 @@ func (d *Daemon) Ready() error {
/* Prune images */
d.pruneChan = make(chan bool)
go func() {
- pruneExpiredImages(d)
for {
timer := time.NewTimer(24 * time.Hour)
select {
@@ -818,6 +817,9 @@ func (d *Daemon) Ready() error {
}
}()
+ // Do an initial pruning run before we start updating images
+ pruneExpiredImages(d)
+
/* Auto-update images */
d.resetAutoUpdateChan = make(chan bool)
go func() {
From ac493dcb118eb1cb8ece958bcedbccee26d27d96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 18:44:18 -0400
Subject: [PATCH 09/10] images: Actually get the list of images to remove
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/db/images.go | 2 +-
lxd/images.go | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/lxd/db/images.go b/lxd/db/images.go
index 9f6d00a62..c1b1fa470 100644
--- a/lxd/db/images.go
+++ b/lxd/db/images.go
@@ -76,7 +76,7 @@ func ImagesGetExpired(db *sql.DB, expiry int64) ([]string, error) {
results = append(results, r[0].(string))
}
- return []string{}, nil
+ return results, nil
}
func ImageSourceInsert(db *sql.DB, imageId int, server string, protocol string, certificate string, alias string) error {
diff --git a/lxd/images.go b/lxd/images.go
index 58b185d04..f915fd12f 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -1006,14 +1006,15 @@ func autoUpdateImage(d *Daemon, op *operation, id int, info *api.Image) error {
}
func pruneExpiredImages(d *Daemon) {
- logger.Infof("Pruning expired images")
-
// Get the list of expired images.
expiry := daemonConfig["images.remote_cache_expiry"].GetInt64()
+
+ // Check if we're supposed to prune something
if expiry <= 0 {
return
}
+ logger.Infof("Pruning expired images")
images, err := db.ImagesGetExpired(d.db, expiry)
if err != nil {
logger.Error("Unable to retrieve the list of expired images", log.Ctx{"err": err})
From abdc002cc572f8ad36df2036d986bb332aa7ab58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 5 Oct 2017 18:56:17 -0400
Subject: [PATCH 10/10] test: Setup basic channel handler for triggers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/main_test.go | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/lxd/main_test.go b/lxd/main_test.go
index dfb2ef2b7..c8a46e6c5 100644
--- a/lxd/main_test.go
+++ b/lxd/main_test.go
@@ -121,6 +121,18 @@ func (suite *lxdTestSuite) SetupTest() {
os.Exit(1)
}
suite.Req = require.New(suite.T())
+
+ suite.d.pruneChan = make(chan bool)
+ suite.d.resetAutoUpdateChan = make(chan bool)
+ go func() {
+ for {
+ select {
+ case <-suite.d.pruneChan:
+ case <-suite.d.resetAutoUpdateChan:
+ continue
+ }
+ }
+ }()
}
func (suite *lxdTestSuite) TearDownTest() {
More information about the lxc-devel
mailing list