[lxc-devel] [lxd/master] lxd/cluster: Add time skew detection

[stgraber on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 370 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200521/db819849/attachment.bin>
-------------- next part --------------
From 029e18d71d9b498bb6522fb779130634fdbe3c4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 21 May 2020 12:11:28 -0400
Subject: [PATCH] lxd/cluster: Add time skew detection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes #7321

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cluster/gateway.go   | 19 +++++++++++++++++++
 lxd/cluster/heartbeat.go |  5 +++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 2195ca6d9f..14cad5378d 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -117,6 +117,9 @@ type Gateway struct {
 
 	// Abstract unix socket that the local dqlite task is listening to.
 	bindAddress string
+
+	// Keep track of skews
+	timeSkew bool
 }
 
 // Current dqlite protocol version.
@@ -183,6 +186,22 @@ func (g *Gateway) HandlerFuncs(nodeRefreshTask func(*APIHeartbeat)) map[string]h
 				return
 			}
 
+			// Look for time skews
+			if heartbeatData.Time.Add(5 * time.Second).Before(time.Now().UTC()) {
+				if !g.timeSkew {
+					logger.Warnf("Time skew detected between local member and leader (%s vs %s)", heartbeatData.Time, time.Now().UTC())
+				}
+				g.timeSkew = true
+			} else if heartbeatData.Time.Add(-5 * time.Second).After(time.Now().UTC()) {
+				if !g.timeSkew {
+					logger.Warnf("Time skew detected between local member and leader (%s vs %s)", heartbeatData.Time, time.Now().UTC())
+				}
+				g.timeSkew = true
+			} else {
+				logger.Warnf("Time skew resolved")
+				g.timeSkew = false
+			}
+
 			raftNodes := make([]db.RaftNode, 0)
 			for _, node := range heartbeatData.Members {
 				if node.RaftID > 0 {
diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go
index 6db52b4ed9..b43be4d9b1 100644
--- a/lxd/cluster/heartbeat.go
+++ b/lxd/cluster/heartbeat.go
@@ -52,7 +52,6 @@ type APIHeartbeat struct {
 // If allNodes provided is an empty set then this is considered a non-full state list.
 func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, allNodes []db.NodeInfo, offlineThreshold time.Duration) {
 	var maxSchemaVersion, maxAPIExtensionsVersion int
-	hbState.Time = time.Now()
 
 	if hbState.Members == nil {
 		hbState.Members = make(map[int64]APIHeartbeatMember)
@@ -121,8 +120,10 @@ func (hbState *APIHeartbeat) Send(ctx context.Context, cert *shared.CertInfo, lo
 		}
 		logger.Debugf("Sending heartbeat to %s", address)
 
-		err := HeartbeatNode(ctx, address, cert, heartbeatData)
+		// Update timestamp to current, used for time skew detection
+		heartbeatData.Time = time.Now().UTC()
 
+		err := HeartbeatNode(ctx, address, cert, heartbeatData)
 		if err == nil {
 			hbState.Lock()
 			// Ensure only update nodes that exist in Members already.