[lxc-devel] [lxd/master] Improve RBAC handling

[stgraber on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200924/47e6d2a8/attachment.bin>
-------------- next part --------------
From 3d2cc67697b656a544800b08a604fa6e60d3f127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 24 Sep 2020 16:21:54 -0400
Subject: [PATCH 1/2] lxd/rbac: Avoid tight retry loop
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Also, remove short path in cache flush so we get consistent logging.

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/rbac/server.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lxd/rbac/server.go b/lxd/rbac/server.go
index a3f9eb0b76..78604b434a 100644
--- a/lxd/rbac/server.go
+++ b/lxd/rbac/server.go
@@ -157,6 +157,7 @@ func (r *Server) StartStatusCheck() {
 			if resp.StatusCode != 200 {
 				resp.Body.Close()
 				logger.Debugf("RBAC server disconnected, re-connecting. (code=%v)", resp.StatusCode)
+				time.Sleep(10)
 				continue
 			}
 
@@ -164,6 +165,7 @@ func (r *Server) StartStatusCheck() {
 			resp.Body.Close()
 			if err != nil {
 				logger.Errorf("Failed to parse RBAC response, re-trying: %v", err)
+				time.Sleep(10)
 				continue
 			}
 
@@ -359,10 +361,6 @@ func (r *Server) flushCache() {
 	r.permissionsLock.Lock()
 	defer r.permissionsLock.Unlock()
 
-	if len(r.permissions) == 0 {
-		return
-	}
-
 	logger.Info("Flushing RBAC permissions cache")
 
 	for k, v := range r.permissions {

From 228860dc44aeb45ab6c66e7c5e2d77db2639938e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 24 Sep 2020 16:47:29 -0400
Subject: [PATCH 2/2] lxd/rbac: Directly handle re-tries on 504
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/rbac/server.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lxd/rbac/server.go b/lxd/rbac/server.go
index 78604b434a..444882c097 100644
--- a/lxd/rbac/server.go
+++ b/lxd/rbac/server.go
@@ -154,7 +154,14 @@ func (r *Server) StartStatusCheck() {
 				return
 			}
 
+			if resp.StatusCode == 504 {
+				// 504 indicates the server timed out the background connection, just re-connect.
+				resp.Body.Close()
+				continue
+			}
+
 			if resp.StatusCode != 200 {
+				// For other errors we assume a server restart and give it a few seconds.
 				resp.Body.Close()
 				logger.Debugf("RBAC server disconnected, re-connecting. (code=%v)", resp.StatusCode)
 				time.Sleep(10)