[lxc-devel] [lxd/master] Handle out-of-space errors gracefully

[freeekanayaka on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191209/feedf468/attachment.bin>
-------------- next part --------------
From 76e00c876d9916b8551e815a3c7e03f7105a502c Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 9 Dec 2019 08:53:23 +0000
Subject: [PATCH 1/3] Rename database_update.sh to database.sh

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 test/suites/{database_update.sh => database.sh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename test/suites/{database_update.sh => database.sh} (100%)

diff --git a/test/suites/database_update.sh b/test/suites/database.sh
similarity index 100%
rename from test/suites/database_update.sh
rename to test/suites/database.sh

From d883f2e3e4f0ca98265553dd78e49dbbadaeebd9 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 9 Dec 2019 11:38:44 +0000
Subject: [PATCH 2/3] Don't retry in case of generic I/O errors

Leadership-related errors are propagated by go-dqlite using driver.ErrBadConn,
so this legacy check can be removed, since it would actually make us retry in
case of non-retriable errors such as out-of-disk.

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/db/query/retry.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lxd/db/query/retry.go b/lxd/db/query/retry.go
index 1b288377a4..240f75e1ac 100644
--- a/lxd/db/query/retry.go
+++ b/lxd/db/query/retry.go
@@ -51,10 +51,5 @@ func IsRetriableError(err error) bool {
 		return true
 	}
 
-	// Despite the description this is usually a lost leadership error.
-	if strings.Contains(err.Error(), "disk I/O error") {
-		return true
-	}
-
 	return false
 }

From 2f4f4712299ab9be3722afebfaa2479b37a3121b Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 9 Dec 2019 11:40:49 +0000
Subject: [PATCH 3/3] Add test_database_no_disk_space

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 test/main.sh            |  1 +
 test/suites/database.sh | 47 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/test/main.sh b/test/main.sh
index eca30ee99b..b2676b9f56 100755
--- a/test/main.sh
+++ b/test/main.sh
@@ -158,6 +158,7 @@ run_test test_check_deps "checking dependencies"
 run_test test_static_analysis "static analysis"
 run_test test_database_update "database schema updates"
 run_test test_database_restore "database restore"
+run_test test_database_no_disk_space "database out of disk space"
 run_test test_sql "lxd sql"
 run_test test_basic_usage "basic usage"
 run_test test_remote_url "remote url handling"
diff --git a/test/suites/database.sh b/test/suites/database.sh
index 3a93f23f74..fd62774b43 100644
--- a/test/suites/database.sh
+++ b/test/suites/database.sh
@@ -82,3 +82,50 @@ EOF
 
   kill_lxd "${LXD_RESTORE_DIR}"
 }
+
+test_database_no_disk_space(){
+  # shellcheck disable=2039
+  local LXD_DIR
+
+  LXD_NOSPACE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+
+  # Mount a tmpfs with limited space in the global database directory and create
+  # a very big file in it, which will eventually cause database transactions to
+  # fail.
+  GLOBAL_DB_DIR="${LXD_NOSPACE_DIR}/database/global"
+  BIG_FILE="${GLOBAL_DB_DIR}/bigfile"
+  mkdir -p "${GLOBAL_DB_DIR}"
+  mount -t tmpfs -o size=67108864 tmpfs "${GLOBAL_DB_DIR}"
+  dd bs=1024 count=49152 if=/dev/zero of="${BIG_FILE}"
+
+  spawn_lxd "${LXD_NOSPACE_DIR}" true
+
+  (
+    set -e
+    # shellcheck disable=SC2034,SC2030
+    LXD_DIR="${LXD_NOSPACE_DIR}"
+
+    ensure_import_testimage
+    lxc init testimage c
+
+    # Set a custom user property with a big value, so we eventually eat up all
+    # available disk space in the database directory.
+    DATA="$(head -c 65536 < /dev/zero | tr '\0' '\141')"
+    for i in $(seq 40); do
+        if ! lxc config set c "user.prop${i}" "${DATA}"; then
+            break
+        fi
+    done
+
+    # Commands that involve writing to the database keep failing.
+    ! lxc config set c "user.propX" "${DATA}" || false
+    ! lxc config set c "user.propY" "${DATA}" || false
+
+    # Removing the big file makes the database happy again.
+    rm "${BIG_FILE}"
+    lxc config set c "user.propZ" "${DATA}"
+  )
+
+  shutdown_lxd "${LXD_NOSPACE_DIR}"
+  umount "${GLOBAL_DB_DIR}"
+}