[lxc-devel] [lxd/master] Cluster logging improvement
stgraber on Github
lxc-bot at linuxcontainers.org
Fri Mar 29 22:15:52 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190329/9f1f15f8/attachment-0001.bin>
-------------- next part --------------
From 2943c14cbab6762e95b32a9f3465117552ede2e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 29 Mar 2019 17:32:15 -0400
Subject: [PATCH 1/5] shared/api: Add Location field to api.Event
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
shared/api/event.go | 3 +++
1 file changed, 3 insertions(+)
diff --git a/shared/api/event.go b/shared/api/event.go
index f8e038f661..cf52588c9c 100644
--- a/shared/api/event.go
+++ b/shared/api/event.go
@@ -10,6 +10,9 @@ type Event struct {
Type string `yaml:"type" json:"type"`
Timestamp time.Time `yaml:"timestamp" json:"timestamp"`
Metadata json.RawMessage `yaml:"metadata" json:"metadata"`
+
+ // API extension: event_location
+ Location string `yaml:"location" json:"location"`
}
// EventLogging represents a logging type event entry (admin only)
From f93950ddfb2874ad3a8b11ee7c6dd9236b180910 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 29 Mar 2019 17:32:28 -0400
Subject: [PATCH 2/5] api: Add event_location API extension
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
doc/api-extensions.md | 3 +++
shared/version/api.go | 1 +
2 files changed, 4 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 20395ea9bf..03fea2c807 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -722,3 +722,6 @@ This effectively gives us:
This is required to implement environments where the on-disk map isn't
changed but the kernel map is (e.g. shiftfs).
+
+## event\_location
+Expose the location of the generation of API events.
diff --git a/shared/version/api.go b/shared/version/api.go
index 6d056c5747..1b398634be 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -145,6 +145,7 @@ var APIExtensions = []string{
"resources_numa",
"kernel_features",
"id_map_current",
+ "event_location",
}
// APIExtensionsCount returns the number of available API extensions.
From c19d96d470bc0061ad7dd489d6f3d68894777a04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 29 Mar 2019 17:32:51 -0400
Subject: [PATCH 3/5] lxd/events: Expose Location of events
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/events.go | 51 ++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 40 insertions(+), 11 deletions(-)
diff --git a/lxd/events.go b/lxd/events.go
index 32419382a9..a51de1d02b 100644
--- a/lxd/events.go
+++ b/lxd/events.go
@@ -12,6 +12,7 @@ import (
log "github.com/lxc/lxd/shared/log15"
"github.com/pborman/uuid"
+ "github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/logger"
@@ -69,6 +70,7 @@ type eventListener struct {
id string
lock sync.Mutex
done bool
+ location string
// If true, this listener won't get events forwarded from other
// nodes. It only used by listeners created internally by LXD nodes
@@ -78,34 +80,49 @@ type eventListener struct {
type eventsServe struct {
req *http.Request
+ d *Daemon
}
func (r *eventsServe) Render(w http.ResponseWriter) error {
- return eventsSocket(r.req, w)
+ return eventsSocket(r.d, r.req, w)
}
func (r *eventsServe) String() string {
return "event handler"
}
-func eventsSocket(r *http.Request, w http.ResponseWriter) error {
+func eventsSocket(d *Daemon, r *http.Request, w http.ResponseWriter) error {
project := projectParam(r)
typeStr := r.FormValue("type")
if typeStr == "" {
typeStr = "logging,operation,lifecycle"
}
+ // Upgrade the connection to websocket
c, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
if err != nil {
return err
}
+ // Get the current local serverName and store it for the events
+ // We do that now to avoid issues with changes to the name and to limit
+ // the number of DB access to just one per connection
+ var serverName string
+ err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
+ serverName, err = tx.NodeName()
+ return err
+ })
+ if err != nil {
+ return err
+ }
+
listener := eventListener{
project: project,
active: make(chan bool, 1),
connection: c,
id: uuid.NewRandom().String(),
messageTypes: strings.Split(typeStr, ","),
+ location: serverName,
}
// If this request is an internal one initiated by another node wanting
@@ -125,7 +142,7 @@ func eventsSocket(r *http.Request, w http.ResponseWriter) error {
}
func eventsGet(d *Daemon, r *http.Request) Response {
- return &eventsServe{req: r}
+ return &eventsServe{req: r, d: d}
}
func eventSend(project, eventType string, eventMessage interface{}) error {
@@ -143,11 +160,6 @@ func eventSend(project, eventType string, eventMessage interface{}) error {
}
func eventBroadcast(project string, event api.Event, isForward bool) error {
- body, err := json.Marshal(event)
- if err != nil {
- return err
- }
-
eventsLock.Lock()
listeners := eventListeners
for _, listener := range listeners {
@@ -163,7 +175,7 @@ func eventBroadcast(project string, event api.Event, isForward bool) error {
continue
}
- go func(listener *eventListener, body []byte) {
+ go func(listener *eventListener, event api.Event) {
// Check that the listener still exists
if listener == nil {
return
@@ -178,7 +190,24 @@ func eventBroadcast(project string, event api.Event, isForward bool) error {
return
}
- err := listener.connection.WriteMessage(websocket.TextMessage, body)
+ // Set the Location to the expected serverName
+ if event.Location == "" {
+ eventCopy := api.Event{}
+ err := shared.DeepCopy(&event, &eventCopy)
+ if err != nil {
+ return
+ }
+ eventCopy.Location = listener.location
+
+ event = eventCopy
+ }
+
+ body, err := json.Marshal(event)
+ if err != nil {
+ return
+ }
+
+ err = listener.connection.WriteMessage(websocket.TextMessage, body)
if err != nil {
// Remove the listener from the list
eventsLock.Lock()
@@ -191,7 +220,7 @@ func eventBroadcast(project string, event api.Event, isForward bool) error {
listener.done = true
logger.Debugf("Disconnected event listener: %s", listener.id)
}
- }(listener, body)
+ }(listener, event)
}
eventsLock.Unlock()
From 4e4d52b478cfe91a282c37e908c58cabc15a7724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 29 Mar 2019 18:08:52 -0400
Subject: [PATCH 4/5] client: Properly generate events URL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
client/lxd_events.go | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/client/lxd_events.go b/client/lxd_events.go
index 6cfd7db354..3c83132053 100644
--- a/client/lxd_events.go
+++ b/client/lxd_events.go
@@ -2,7 +2,6 @@ package lxd
import (
"encoding/json"
- "fmt"
"time"
"github.com/lxc/lxd/shared"
@@ -33,7 +32,12 @@ func (r *ProtocolLXD) GetEvents() (*EventListener, error) {
r.eventListeners = []*EventListener{}
// Setup a new connection with LXD
- conn, err := r.websocket(fmt.Sprintf("/events?project=%s", r.project))
+ url, err := r.setQueryAttributes("/events")
+ if err != nil {
+ return nil, err
+ }
+
+ conn, err := r.websocket(url)
if err != nil {
return nil, err
}
From db857ea0b04cbe75ef83b0779037b5e34c645f8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 29 Mar 2019 18:14:52 -0400
Subject: [PATCH 5/5] lxd/cluster: Limit log message forwarding
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/events.go | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/lxd/events.go b/lxd/events.go
index a51de1d02b..10434b415a 100644
--- a/lxd/events.go
+++ b/lxd/events.go
@@ -229,6 +229,23 @@ func eventBroadcast(project string, event api.Event, isForward bool) error {
// Forward to the local events dispatcher an event received from another node .
func eventForward(id int64, event api.Event) {
+ if event.Type == "logging" {
+ // Parse the message
+ logEntry := api.EventLogging{}
+ err := json.Unmarshal(event.Metadata, &logEntry)
+ if err != nil {
+ return
+ }
+
+ if !debug && logEntry.Level == "dbug" {
+ return
+ }
+
+ if !debug && !verbose && logEntry.Level == "info" {
+ return
+ }
+ }
+
err := eventBroadcast("", event, true)
if err != nil {
logger.Warnf("Failed to forward event from node %d: %v", id, err)
More information about the lxc-devel
mailing list