[lxc-devel] [lxd/master] Rename database files

freeekanayaka on Github lxc-bot at linuxcontainers.org
Tue Apr 17 06:58:32 UTC 2018


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 484 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180417/ebfd3308/attachment.bin>
-------------- next part --------------
From 5503db2205c5100e23d8adc5a9e624ade022e2f8 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 17 Apr 2018 06:15:55 +0000
Subject: [PATCH] Rename database files

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/cluster/gateway.go         |  2 +-
 lxd/cluster/gateway_test.go    |  2 +-
 lxd/cluster/membership.go      |  4 ++--
 lxd/cluster/raft.go            | 23 ++++++++++++++++-----
 lxd/daemon.go                  | 26 +++++++++++++++++-------
 lxd/db/migration.go            |  4 ++--
 lxd/db/node/open.go            |  6 +++---
 lxd/main_activateifneeded.go   | 24 ++++++++++++++--------
 lxd/sys/fs.go                  | 32 +++++++++++++++++++++++++++--
 test/includes/lxd.sh           | 46 +++++++++++++++++++++---------------------
 test/suites/database_update.sh |  3 ++-
 11 files changed, 117 insertions(+), 55 deletions(-)

diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 07546c170..b37b6db1f 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -296,7 +296,7 @@ func (g *Gateway) Reset(cert *shared.CertInfo) error {
 	if err != nil {
 		return err
 	}
-	err = os.RemoveAll(filepath.Join(g.db.Dir(), "raft"))
+	err = os.RemoveAll(filepath.Join(g.db.Dir(), "global"))
 	if err != nil {
 		return err
 	}
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index bfefaf29a..71ed46913 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -146,7 +146,7 @@ func TestGateway_RaftNodesNotLeader(t *testing.T) {
 // happens.
 func newGateway(t *testing.T, db *db.Node, certInfo *shared.CertInfo) *cluster.Gateway {
 	logging.Testing(t)
-	require.NoError(t, os.Mkdir(filepath.Join(db.Dir(), "raft"), 0755))
+	require.NoError(t, os.Mkdir(filepath.Join(db.Dir(), "global"), 0755))
 	gateway, err := cluster.NewGateway(
 		db, certInfo, cluster.Latency(0.2), cluster.LogLevel("TRACE"))
 	require.NoError(t, err)
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index 8bea809ad..865e909e2 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -286,7 +286,7 @@ func Join(state *state.State, gateway *Gateway, cert *shared.CertInfo, name stri
 	if err != nil {
 		return errors.Wrap(err, "failed to shutdown gRPC SQL gateway")
 	}
-	err = os.RemoveAll(filepath.Join(state.OS.VarDir, "raft"))
+	err = os.RemoveAll(state.OS.GlobalDatabaseDir())
 	if err != nil {
 		return errors.Wrap(err, "failed to remove existing raft data")
 	}
@@ -582,7 +582,7 @@ func Promote(state *state.State, gateway *Gateway, nodes []db.RaftNode) error {
 
 	// Wipe all existing raft data, for good measure (perhaps they were
 	// somehow leftover).
-	err = os.RemoveAll(filepath.Join(state.OS.VarDir, "raft"))
+	err = os.RemoveAll(state.OS.GlobalDatabaseDir())
 	if err != nil {
 		return errors.Wrap(err, "failed to remove existing raft data")
 	}
diff --git a/lxd/cluster/raft.go b/lxd/cluster/raft.go
index 72ce4c437..fa540ce35 100644
--- a/lxd/cluster/raft.go
+++ b/lxd/cluster/raft.go
@@ -95,11 +95,11 @@ func raftInstanceInit(
 	// FIXME: should be a parameter
 	timeout := 5 * time.Second
 
-	logger := raftLogger()
+	raftLogger := raftLogger()
 
 	// Raft config.
 	config := raftConfig(latency)
-	config.Logger = logger
+	config.Logger = raftLogger
 	config.LocalID = raft.ServerID(strconv.Itoa(int(node.ID)))
 
 	// Raft transport
@@ -122,7 +122,7 @@ func raftInstanceInit(
 			return nil, err
 		}
 
-		transport, handler, layer, err = raftNetworkTransport(db, addr, logger, timeout, dial)
+		transport, handler, layer, err = raftNetworkTransport(db, addr, raftLogger, timeout, dial)
 		if err != nil {
 			return nil, err
 		}
@@ -136,8 +136,21 @@ func raftInstanceInit(
 		return nil, errors.Wrap(err, "invalid raft configuration")
 	}
 
+	// Rename legacy data directory if needed.
+	dir := filepath.Join(db.Dir(), "global")
+	legacyDir := filepath.Join(db.Dir(), "..", "raft")
+	if shared.PathExists(legacyDir) {
+		if shared.PathExists(dir) {
+			return nil, fmt.Errorf("both legacy and new global database directories exist")
+		}
+		logger.Info("Renaming global database directory from raft/ to database/global/")
+		err := os.Rename(legacyDir, dir)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to rename legacy global database directory")
+		}
+	}
+
 	// Data directory
-	dir := filepath.Join(db.Dir(), "raft")
 	if !shared.PathExists(dir) {
 		err := os.Mkdir(dir, 0750)
 		if err != nil {
@@ -155,7 +168,7 @@ func raftInstanceInit(
 	}
 
 	// Raft snapshot store
-	snaps, err := raft.NewFileSnapshotStoreWithLogger(dir, 2, logger)
+	snaps, err := raft.NewFileSnapshotStoreWithLogger(dir, 2, raftLogger)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create file snapshot store")
 	}
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 8c8ef3bfc..2b76ab2ec 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -497,24 +497,24 @@ func (d *Daemon) init() error {
 
 	/* Migrate the node local data to the cluster database, if needed */
 	if dump != nil {
-		logger.Infof("Migrating data from lxd.db to db.bin")
+		logger.Infof("Migrating data from local to global database")
 		err = d.cluster.ImportPreClusteringData(dump)
 		if err != nil {
 			// Restore the local sqlite3 backup and wipe the raft
 			// directory, so users can fix problems and retry.
-			path := filepath.Join(d.os.VarDir, "lxd.db")
+			path := d.os.LocalDatabasePath()
 			copyErr := shared.FileCopy(path+".bak", path)
 			if copyErr != nil {
 				// Ignore errors here, there's not much we can do
-				logger.Errorf("Failed to restore lxd.db: %v", copyErr)
+				logger.Errorf("Failed to restore local database: %v", copyErr)
 			}
-			rmErr := os.RemoveAll(filepath.Join(d.os.VarDir, "raft"))
+			rmErr := os.RemoveAll(d.os.GlobalDatabaseDir())
 			if rmErr != nil {
 				// Ignore errors here, there's not much we can do
-				logger.Errorf("Failed to cleanup raft db: %v", rmErr)
+				logger.Errorf("Failed to cleanup global database: %v", rmErr)
 			}
 
-			return fmt.Errorf("Failed to migrate data to db.bin: %v", err)
+			return fmt.Errorf("Failed to migrate data to global database: %v", err)
 		}
 	}
 
@@ -859,6 +859,18 @@ func (d *Daemon) setupMAASController(server string, key string, machine string)
 
 // Create a database connection and perform any updates needed.
 func initializeDbObject(d *Daemon) (*db.Dump, error) {
+	// Rename the old database name if needed.
+	if shared.PathExists(d.os.LegacyLocalDatabasePath()) {
+		if shared.PathExists(d.os.LocalDatabasePath()) {
+			return nil, fmt.Errorf("Both legacy and new local database files exists")
+		}
+		logger.Info("Renaming local database file from lxd.db to database/local.db")
+		err := os.Rename(d.os.LegacyLocalDatabasePath(), d.os.LocalDatabasePath())
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed to rename legacy local database file")
+		}
+	}
+
 	// NOTE: we use the legacyPatches parameter to run a few
 	// legacy non-db updates that were in place before the
 	// patches mechanism was introduced in lxd/patches.go. The
@@ -898,7 +910,7 @@ func initializeDbObject(d *Daemon) (*db.Dump, error) {
 	}
 	var err error
 	var dump *db.Dump
-	d.db, dump, err = db.OpenNode(d.os.VarDir, freshHook, legacy)
+	d.db, dump, err = db.OpenNode(filepath.Join(d.os.VarDir, "database"), freshHook, legacy)
 	if err != nil {
 		return nil, fmt.Errorf("Error creating database: %s", err)
 	}
diff --git a/lxd/db/migration.go b/lxd/db/migration.go
index d2116ceb9..8f320deea 100644
--- a/lxd/db/migration.go
+++ b/lxd/db/migration.go
@@ -237,8 +237,8 @@ func importNodeAssociation(entity string, columns []string, row []interface{}, t
 	return nil
 }
 
-// Dump is a dump of all the user data in lxd.db prior the migration to the
-// cluster db.
+// Dump is a dump of all the user data in the local db prior the migration to
+// the cluster db.
 type Dump struct {
 	// Map table names to the names or their columns.
 	Schema map[string][]string
diff --git a/lxd/db/node/open.go b/lxd/db/node/open.go
index 9d77f0c2d..23048820f 100644
--- a/lxd/db/node/open.go
+++ b/lxd/db/node/open.go
@@ -12,7 +12,7 @@ import (
 
 // Open the node-local database object.
 func Open(dir string) (*sql.DB, error) {
-	path := filepath.Join(dir, "lxd.db")
+	path := filepath.Join(dir, "local.db")
 	db, err := sqliteOpen(path)
 	if err != nil {
 		return nil, fmt.Errorf("cannot open node database: %v", err)
@@ -32,8 +32,8 @@ func EnsureSchema(db *sql.DB, dir string, hook schema.Hook) (int, error) {
 	schema := Schema()
 	schema.Hook(func(version int, tx *sql.Tx) error {
 		if !backupDone {
-			logger.Infof("Updating the LXD database schema. Backup made as \"lxd.db.bak\"")
-			path := filepath.Join(dir, "lxd.db")
+			logger.Infof("Updating the LXD database schema. Backup made as \"local.db.bak\"")
+			path := filepath.Join(dir, "local.db")
 			err := shared.FileCopy(path, path+".bak")
 			if err != nil {
 				return err
diff --git a/lxd/main_activateifneeded.go b/lxd/main_activateifneeded.go
index a3c9e282d..760478f0a 100644
--- a/lxd/main_activateifneeded.go
+++ b/lxd/main_activateifneeded.go
@@ -4,7 +4,6 @@ import (
 	"database/sql"
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/CanonicalLtd/go-sqlite3"
 	"github.com/spf13/cobra"
@@ -53,14 +52,20 @@ func (c *cmdActivateifneeded) Run(cmd *cobra.Command, args []string) error {
 	// Don't start a full daemon, we just need DB access
 	d := DefaultDaemon()
 
-	if !shared.PathExists(shared.VarPath("lxd.db")) {
-		logger.Debugf("No DB, so no need to start the daemon now.")
-		return nil
+	// Check if either the local database or the legacy local database
+	// files exists.
+	path := d.os.LocalDatabasePath()
+	if !shared.PathExists(d.os.LocalDatabasePath()) {
+		path = d.os.LegacyLocalDatabasePath()
+		if !shared.PathExists(path) {
+			logger.Debugf("No DB, so no need to start the daemon now.")
+			return nil
+		}
 	}
 
 	// Open the database directly to avoid triggering any initialization
 	// code, in particular the data migration from node to cluster db.
-	sqldb, err := sql.Open("sqlite3", filepath.Join(d.os.VarDir, "lxd.db"))
+	sqldb, err := sql.Open("sqlite3", path)
 	if err != nil {
 		return err
 	}
@@ -86,10 +91,13 @@ func (c *cmdActivateifneeded) Run(cmd *cobra.Command, args []string) error {
 	}
 
 	// Look for auto-started or previously started containers
-	path := filepath.Join(d.os.VarDir, "raft", "db.bin")
+	path = d.os.GlobalDatabasePath()
 	if !shared.PathExists(path) {
-		logger.Debugf("No DB, so no need to start the daemon now.")
-		return nil
+		path = d.os.LegacyGlobalDatabasePath()
+		if !shared.PathExists(path) {
+			logger.Debugf("No DB, so no need to start the daemon now.")
+			return nil
+		}
 	}
 	sqldb, err = sql.Open("dqlite_direct_access", path+"?mode=ro")
 	if err != nil {
diff --git a/lxd/sys/fs.go b/lxd/sys/fs.go
index 8370838eb..0f4e948e5 100644
--- a/lxd/sys/fs.go
+++ b/lxd/sys/fs.go
@@ -3,8 +3,36 @@ package sys
 import (
 	"os"
 	"path/filepath"
+
+	"github.com/pkg/errors"
 )
 
+// LocalDatabasePath returns the path of the local database file.
+func (s *OS) LocalDatabasePath() string {
+	return filepath.Join(s.VarDir, "database", "local.db")
+}
+
+// LegacyLocalDatabasePath returns the path of legacy local database file.
+func (s *OS) LegacyLocalDatabasePath() string {
+	return filepath.Join(s.VarDir, "lxd.db")
+}
+
+// GlobalDatabaseDir returns the path of the global database directory.
+func (s *OS) GlobalDatabaseDir() string {
+	return filepath.Join(s.VarDir, "database", "global")
+}
+
+// GlobalDatabasePath returns the path of the global database SQLite file
+// managed by dqlite.
+func (s *OS) GlobalDatabasePath() string {
+	return filepath.Join(s.GlobalDatabaseDir(), "db.bin")
+}
+
+// LegacyGlobalDatabasePath returns the path of legacy global database file.
+func (s *OS) LegacyGlobalDatabasePath() string {
+	return filepath.Join(s.VarDir, "raft", "db.bin")
+}
+
 // Make sure all our directories are available.
 func (s *OS) initDirs() error {
 	dirs := []struct {
@@ -13,7 +41,7 @@ func (s *OS) initDirs() error {
 	}{
 		{s.VarDir, 0711},
 		{s.CacheDir, 0700},
-		{filepath.Join(s.VarDir, "raft"), 0700},
+		{filepath.Join(s.VarDir, "database"), 0700},
 		{filepath.Join(s.VarDir, "containers"), 0711},
 		{filepath.Join(s.VarDir, "devices"), 0711},
 		{filepath.Join(s.VarDir, "devlxd"), 0755},
@@ -30,7 +58,7 @@ func (s *OS) initDirs() error {
 	for _, dir := range dirs {
 		err := os.Mkdir(dir.path, dir.mode)
 		if err != nil && !os.IsExist(err) {
-			return err
+			return errors.Wrapf(err, "failed to init dir %s", dir.path)
 		}
 	}
 
diff --git a/test/includes/lxd.sh b/test/includes/lxd.sh
index ca7384ab5..58bfb9178 100644
--- a/test/includes/lxd.sh
+++ b/test/includes/lxd.sh
@@ -164,8 +164,8 @@ kill_lxd() {
         done
 
         echo "==> Checking for locked DB tables"
-        for table in $(echo .tables | sqlite3 "${daemon_dir}/lxd.db"); do
-            echo "SELECT * FROM ${table};" | sqlite3 "${daemon_dir}/lxd.db" >/dev/null
+        for table in $(echo .tables | sqlite3 "${daemon_dir}/local.db"); do
+            echo "SELECT * FROM ${table};" | sqlite3 "${daemon_dir}/local.db" >/dev/null
         done
 
         # Kill the daemon
@@ -203,27 +203,27 @@ kill_lxd() {
         echo "==> Checking for leftover cluster DB entries"
         # FIXME: we should not use the command line sqlite client, since it's
         #        not compatible with dqlite
-        check_empty_table "${daemon_dir}/raft/db.bin" "containers"
-        check_empty_table "${daemon_dir}/raft/db.bin" "containers_config"
-        check_empty_table "${daemon_dir}/raft/db.bin" "containers_devices"
-        check_empty_table "${daemon_dir}/raft/db.bin" "containers_devices_config"
-        check_empty_table "${daemon_dir}/raft/db.bin" "containers_profiles"
-        check_empty_table "${daemon_dir}/raft/db.bin" "images"
-        check_empty_table "${daemon_dir}/raft/db.bin" "images_aliases"
-        check_empty_table "${daemon_dir}/raft/db.bin" "images_properties"
-        check_empty_table "${daemon_dir}/raft/db.bin" "images_source"
-        check_empty_table "${daemon_dir}/raft/db.bin" "images_nodes"
-        check_empty_table "${daemon_dir}/raft/db.bin" "networks"
-        check_empty_table "${daemon_dir}/raft/db.bin" "networks_config"
-        check_empty_table "${daemon_dir}/raft/db.bin" "profiles"
-        check_empty_table "${daemon_dir}/raft/db.bin" "profiles_config"
-        check_empty_table "${daemon_dir}/raft/db.bin" "profiles_devices"
-        check_empty_table "${daemon_dir}/raft/db.bin" "profiles_devices_config"
-        check_empty_table "${daemon_dir}/raft/db.bin" "storage_pools"
-        check_empty_table "${daemon_dir}/raft/db.bin" "storage_pools_nodes"
-        check_empty_table "${daemon_dir}/raft/db.bin" "storage_pools_config"
-        check_empty_table "${daemon_dir}/raft/db.bin" "storage_volumes"
-        check_empty_table "${daemon_dir}/raft/db.bin" "storage_volumes_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "containers"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "containers_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "containers_devices"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "containers_devices_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "containers_profiles"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "images"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "images_aliases"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "images_properties"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "images_source"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "images_nodes"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "networks"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "networks_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "profiles"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "profiles_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "profiles_devices"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "profiles_devices_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "storage_pools"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "storage_pools_nodes"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "storage_pools_config"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "storage_volumes"
+        check_empty_table "${daemon_dir}/database/global/db.bin" "storage_volumes_config"
     fi
 
     # teardown storage
diff --git a/test/suites/database_update.sh b/test/suites/database_update.sh
index 4987802a0..6259d5c34 100644
--- a/test/suites/database_update.sh
+++ b/test/suites/database_update.sh
@@ -1,6 +1,7 @@
 test_database_update(){
   LXD_MIGRATE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
-  MIGRATE_DB=${LXD_MIGRATE_DIR}/lxd.db
+  mkdir -p "${LXD_MIGRATE_DIR}/database"
+  MIGRATE_DB=${LXD_MIGRATE_DIR}/database/local.db
 
   # Create the version 1 schema as the database
   sqlite3 "${MIGRATE_DB}" > /dev/null < deps/schema1.sql


More information about the lxc-devel mailing list