[lxc-devel] [lxd/master] DEBUG: clustering
stgraber on Github
lxc-bot at linuxcontainers.org
Tue May 7 20:24:28 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 354 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190507/f535c08d/attachment.bin>
-------------- next part --------------
From b524a268b35ea08899a3e073ad704504b225a674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 7 May 2019 16:12:00 -0400
Subject: [PATCH] DEBUG: clustering
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/cluster/heartbeat.go | 16 +++++
test/main.sh | 136 +++++++++++++++++++--------------------
2 files changed, 84 insertions(+), 68 deletions(-)
diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go
index 980f8c7018..c84debd651 100644
--- a/lxd/cluster/heartbeat.go
+++ b/lxd/cluster/heartbeat.go
@@ -24,14 +24,24 @@ import (
// It will update the heartbeat timestamp column of the nodes table
// accordingly, and also notify them of the current list of database nodes.
func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule) {
+ serverName := "unknown"
+ cluster.Transaction(func(tx *db.ClusterTx) error {
+ var err error
+ serverName, err = tx.NodeName()
+ return err
+ })
+
heartbeat := func(ctx context.Context) {
+ logger.Errorf("[%s] stgraber: in hearbeat", serverName)
if gateway.server == nil || gateway.memoryDial != nil {
// We're not a raft node or we're not clustered
+ logger.Errorf("[%s] stgraber: not clustered => out", serverName)
return
}
raftNodes, err := gateway.currentRaftNodes()
if err == raft.ErrNotLeader {
+ logger.Errorf("[%s] stgraber: not the leader => out", serverName)
return
}
logger.Debugf("Starting heartbeat round")
@@ -52,6 +62,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule)
logger.Warnf("Failed to replace local raft nodes: %v", err)
return
}
+ logger.Errorf("[%s] stgraber: set raft nodes to: %+v", serverName, raftNodes)
var nodes []db.NodeInfo
var nodeAddress string // Address of this node
@@ -71,6 +82,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule)
logger.Warnf("Failed to get current cluster nodes: %v", err)
return
}
+ logger.Errorf("[%s] stgraber: local=%v nodes=%v", serverName, nodeAddress, nodes)
heartbeats := make([]time.Time, len(nodes))
heartbeatsLock := sync.Mutex{}
@@ -93,6 +105,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule)
// Spread in time by waiting up to 3s less than the interval
time.Sleep(time.Duration(rand.Intn((heartbeatInterval*1000)-3000)) * time.Millisecond)
logger.Debugf("Sending heartbeat to %s", address)
+ logger.Errorf("[%s] stgraber: Contacting: %s", serverName, address)
err := heartbeatNode(ctx, address, gateway.cert, raftNodes)
if err == nil {
@@ -103,9 +116,11 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule)
} else {
logger.Debugf("Failed heartbeat for %s: %v", address, err)
}
+ logger.Errorf("[%s] stgraber: Got reply from: %s", serverName, address)
}(i, node.Address)
}
heartbeatsWg.Wait()
+ logger.Errorf("[%s] stgraber: Got all replies", serverName)
// If the context has been cancelled, return immediately.
if ctx.Err() != nil {
@@ -130,6 +145,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule)
logger.Warnf("Failed to update heartbeat: %v", err)
}
logger.Debugf("Completed heartbeat round")
+ logger.Errorf("[%s] stgraber: done => out", serverName)
}
// Since the database APIs are blocking we need to wrap the core logic
diff --git a/test/main.sh b/test/main.sh
index 6d245c431c..f51357c75e 100755
--- a/test/main.sh
+++ b/test/main.sh
@@ -154,74 +154,74 @@ if [ "$#" -gt 0 ]; then
exit
fi
-run_test test_check_deps "checking dependencies"
-run_test test_static_analysis "static analysis"
-run_test test_database_update "database schema updates"
-run_test test_database_restore "database restore"
-run_test test_sql "lxd sql"
-run_test test_projects_default "default project"
-run_test test_projects_crud "projects CRUD operations"
-run_test test_projects_containers "containers inside projects"
-run_test test_projects_snapshots "snapshots inside projects"
-run_test test_projects_backups "backups inside projects"
-run_test test_projects_profiles "profiles inside projects"
-run_test test_projects_profiles_default "profiles from the global default project"
-run_test test_projects_images "images inside projects"
-run_test test_projects_images_default "images from the global default project"
-run_test test_projects_storage "projects and storage pools"
-run_test test_projects_network "projects and networks"
-run_test test_remote_url "remote url handling"
-run_test test_remote_admin "remote administration"
-run_test test_remote_usage "remote usage"
-run_test test_basic_usage "basic usage"
-run_test test_container_devices_nic "container devices - nic"
-run_test test_security "security features"
-run_test test_security_protection "container protection"
-run_test test_image_expiry "image expiry"
-run_test test_image_list_all_aliases "image list all aliases"
-run_test test_image_auto_update "image auto-update"
-run_test test_image_prefer_cached "image prefer cached"
-run_test test_image_import_dir "import image from directory"
-run_test test_concurrent_exec "concurrent exec"
-run_test test_concurrent "concurrent startup"
-run_test test_snapshots "container snapshots"
-run_test test_snap_restore "snapshot restores"
-run_test test_snap_expiry "snapshot expiry"
-run_test test_config_profiles "profiles and configuration"
-run_test test_config_edit "container configuration edit"
-run_test test_config_edit_container_snapshot_pool_config "container and snapshot volume configuration edit"
-run_test test_container_metadata "manage container metadata and templates"
-run_test test_container_snapshot_config "container snapshot configuration"
-run_test test_server_config "server configuration"
-run_test test_filemanip "file manipulations"
-run_test test_network "network management"
-run_test test_idmap "id mapping"
-run_test test_template "file templating"
-run_test test_pki "PKI mode"
-run_test test_devlxd "/dev/lxd"
-run_test test_fuidshift "fuidshift"
-run_test test_migration "migration"
-run_test test_fdleak "fd leak"
-run_test test_storage "storage"
-run_test test_storage_volume_snapshots "storage volume snapshots"
-run_test test_init_auto "lxd init auto"
-run_test test_init_interactive "lxd init interactive"
-run_test test_init_preseed "lxd init preseed"
-run_test test_storage_profiles "storage profiles"
-run_test test_container_import "container import"
-run_test test_storage_volume_attach "attaching storage volumes"
-run_test test_storage_driver_ceph "ceph storage driver"
-run_test test_resources "resources"
-run_test test_kernel_limits "kernel limits"
-run_test test_macaroon_auth "macaroon authentication"
-run_test test_console "console"
-run_test test_query "query"
-run_test test_proxy_device "proxy device"
-run_test test_storage_local_volume_handling "storage local volume handling"
-run_test test_backup_import "backup import"
-run_test test_backup_export "backup export"
-run_test test_container_local_cross_pool_handling "container local cross pool handling"
-run_test test_incremental_copy "incremental container copy"
+#run_test test_check_deps "checking dependencies"
+#run_test test_static_analysis "static analysis"
+#run_test test_database_update "database schema updates"
+#run_test test_database_restore "database restore"
+#run_test test_sql "lxd sql"
+#run_test test_projects_default "default project"
+#run_test test_projects_crud "projects CRUD operations"
+#run_test test_projects_containers "containers inside projects"
+#run_test test_projects_snapshots "snapshots inside projects"
+#run_test test_projects_backups "backups inside projects"
+#run_test test_projects_profiles "profiles inside projects"
+#run_test test_projects_profiles_default "profiles from the global default project"
+#run_test test_projects_images "images inside projects"
+#run_test test_projects_images_default "images from the global default project"
+#run_test test_projects_storage "projects and storage pools"
+#run_test test_projects_network "projects and networks"
+#run_test test_remote_url "remote url handling"
+#run_test test_remote_admin "remote administration"
+#run_test test_remote_usage "remote usage"
+#run_test test_basic_usage "basic usage"
+#run_test test_container_devices_nic "container devices - nic"
+#run_test test_security "security features"
+#run_test test_security_protection "container protection"
+#run_test test_image_expiry "image expiry"
+#run_test test_image_list_all_aliases "image list all aliases"
+#run_test test_image_auto_update "image auto-update"
+#run_test test_image_prefer_cached "image prefer cached"
+#run_test test_image_import_dir "import image from directory"
+#run_test test_concurrent_exec "concurrent exec"
+#run_test test_concurrent "concurrent startup"
+#run_test test_snapshots "container snapshots"
+#run_test test_snap_restore "snapshot restores"
+#run_test test_snap_expiry "snapshot expiry"
+#run_test test_config_profiles "profiles and configuration"
+#run_test test_config_edit "container configuration edit"
+#run_test test_config_edit_container_snapshot_pool_config "container and snapshot volume configuration edit"
+#run_test test_container_metadata "manage container metadata and templates"
+#run_test test_container_snapshot_config "container snapshot configuration"
+#run_test test_server_config "server configuration"
+#run_test test_filemanip "file manipulations"
+#run_test test_network "network management"
+#run_test test_idmap "id mapping"
+#run_test test_template "file templating"
+#run_test test_pki "PKI mode"
+#run_test test_devlxd "/dev/lxd"
+#run_test test_fuidshift "fuidshift"
+#run_test test_migration "migration"
+#run_test test_fdleak "fd leak"
+#run_test test_storage "storage"
+#run_test test_storage_volume_snapshots "storage volume snapshots"
+#run_test test_init_auto "lxd init auto"
+#run_test test_init_interactive "lxd init interactive"
+#run_test test_init_preseed "lxd init preseed"
+#run_test test_storage_profiles "storage profiles"
+#run_test test_container_import "container import"
+#run_test test_storage_volume_attach "attaching storage volumes"
+#run_test test_storage_driver_ceph "ceph storage driver"
+#run_test test_resources "resources"
+#run_test test_kernel_limits "kernel limits"
+#run_test test_macaroon_auth "macaroon authentication"
+#run_test test_console "console"
+#run_test test_query "query"
+#run_test test_proxy_device "proxy device"
+#run_test test_storage_local_volume_handling "storage local volume handling"
+#run_test test_backup_import "backup import"
+#run_test test_backup_export "backup export"
+#run_test test_container_local_cross_pool_handling "container local cross pool handling"
+#run_test test_incremental_copy "incremental container copy"
run_test test_clustering_enable "clustering enable"
run_test test_clustering_membership "clustering membership"
run_test test_clustering_containers "clustering containers"
More information about the lxc-devel
mailing list