[lxc-devel] [lxd/master] Support /dev/lxd in VMs
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Thu Jul 2 21:19:57 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200702/1825fd8e/attachment.bin>
-------------- next part --------------
From 5f642c22dacf67f0e733a0e9ee810a4a92e1baba Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 24 Jun 2020 17:35:48 +0200
Subject: [PATCH 1/3] lxd/instance/drivers: Provide instance-data file
This writes an instance-data file to the config drive, containing the
name and user config. It will be used for devlxd inside VMs.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 34 +++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 7617db6da1..634f978a18 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1387,6 +1387,12 @@ echo "To start it now, unmount this filesystem and run: systemctl start lxd-agen
return err
}
+ // Instance data for devlxd.
+ err = vm.writeInstanceData()
+ if err != nil {
+ return err
+ }
+
// Templated files.
err = os.MkdirAll(filepath.Join(configDrivePath, "files"), 0500)
if err != nil {
@@ -4589,3 +4595,31 @@ func (vm *qemu) cpuTopology(limit string) (int, int, int, map[uint64]uint64, map
return nrSockets, nrCores, nrThreads, vcpus, numaNodes, nil
}
+func (vm *qemu) writeInstanceData() error {
+ // Instance data for devlxd.
+ configDrivePath := filepath.Join(vm.Path(), "config")
+ userConfig := make(map[string]string)
+
+ for k, v := range vm.ExpandedConfig() {
+ if !strings.HasPrefix(k, "user.") {
+ continue
+ }
+
+ userConfig[k] = v
+ }
+
+ out, err := json.Marshal(struct {
+ Name string `json:"name"`
+ Config map[string]string `json:"config,omitempty"`
+ }{vm.Name(), userConfig})
+ if err != nil {
+ return err
+ }
+
+ err = ioutil.WriteFile(filepath.Join(configDrivePath, "instance-data"), out, 0600)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
From dd91b95bf4d79ce410d82b1bc9debabeb5671f91 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 30 Jun 2020 12:35:01 +0200
Subject: [PATCH 2/3] lxd-agent: Support /dev/lxd
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd-agent/devlxd.go | 246 ++++++++++++++++++++++++++++++++++++++++
lxd-agent/events.go | 23 +++-
lxd-agent/main_agent.go | 44 ++++++-
3 files changed, 305 insertions(+), 8 deletions(-)
create mode 100644 lxd-agent/devlxd.go
diff --git a/lxd-agent/devlxd.go b/lxd-agent/devlxd.go
new file mode 100644
index 0000000000..c96a17b0fb
--- /dev/null
+++ b/lxd-agent/devlxd.go
@@ -0,0 +1,246 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/gorilla/mux"
+ "github.com/lxc/lxd/lxd/daemon"
+ "github.com/lxc/lxd/lxd/util"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/logger"
+ "github.com/lxc/lxd/shared/version"
+)
+
+// DevLxdServer creates an http.Server capable of handling requests against the
+// /dev/lxd Unix socket endpoint created inside VMs.
+func devLxdServer(d *Daemon) *http.Server {
+ return &http.Server{
+ Handler: devLxdAPI(d),
+ }
+}
+
+type devLxdResponse struct {
+ content interface{}
+ code int
+ ctype string
+}
+
+type instanceData struct {
+ Name string `json:"name"`
+ Config map[string]string `json:"config,omitempty"`
+}
+
+func okResponse(ct interface{}, ctype string) *devLxdResponse {
+ return &devLxdResponse{ct, http.StatusOK, ctype}
+}
+
+type devLxdHandler struct {
+ path string
+
+ /*
+ * This API will have to be changed slightly when we decide to support
+ * websocket events upgrading, but since we don't have events on the
+ * server side right now either, I went the simple route to avoid
+ * needless noise.
+ */
+ f func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse
+}
+
+var devlxdConfigGet = devLxdHandler{"/1.0/config", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
+ data, err := ioutil.ReadFile("instance-data")
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ var instance instanceData
+
+ err = json.Unmarshal(data, &instance)
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ filtered := []string{}
+ for k := range instance.Config {
+ if strings.HasPrefix(k, "user.") {
+ filtered = append(filtered, fmt.Sprintf("/1.0/config/%s", k))
+ }
+ }
+ return okResponse(filtered, "json")
+}}
+
+var devlxdConfigKeyGet = devLxdHandler{"/1.0/config/{key}", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
+ key := mux.Vars(r)["key"]
+ if !strings.HasPrefix(key, "user.") {
+ return &devLxdResponse{"not authorized", http.StatusForbidden, "raw"}
+ }
+
+ data, err := ioutil.ReadFile("instance-data")
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ var instance instanceData
+
+ err = json.Unmarshal(data, &instance)
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ value, ok := instance.Config[key]
+ if !ok {
+ return &devLxdResponse{"not found", http.StatusNotFound, "raw"}
+ }
+
+ return okResponse(value, "raw")
+}}
+
+var devlxdMetadataGet = devLxdHandler{"/1.0/meta-data", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
+ data, err := ioutil.ReadFile("instance-data")
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ var instance instanceData
+
+ err = json.Unmarshal(data, &instance)
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ value := instance.Config["user.meta-data"]
+ return okResponse(fmt.Sprintf("#cloud-config\ninstance-id: %s\nlocal-hostname: %s\n%s", instance.Name, instance.Name, value), "raw")
+}}
+
+var devLxdEventsGet = devLxdHandler{"/1.0/events", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
+ err := eventsGet(d, r).Render(w)
+ if err != nil {
+ return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
+ }
+
+ return okResponse("", "raw")
+}}
+
+var handlers = []devLxdHandler{
+ {"/", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
+ return okResponse([]string{"/1.0"}, "json")
+ }},
+ {"/1.0", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
+ return okResponse(shared.Jmap{"api_version": version.APIVersion}, "json")
+ }},
+ devlxdConfigGet,
+ devlxdConfigKeyGet,
+ devlxdMetadataGet,
+ devLxdEventsGet,
+}
+
+func hoistReq(f func(*Daemon, http.ResponseWriter, *http.Request) *devLxdResponse, d *Daemon) func(http.ResponseWriter, *http.Request) {
+ return func(w http.ResponseWriter, r *http.Request) {
+ resp := f(d, w, r)
+ if resp.code != http.StatusOK {
+ http.Error(w, fmt.Sprintf("%s", resp.content), resp.code)
+ } else if resp.ctype == "json" {
+ w.Header().Set("Content-Type", "application/json")
+ util.WriteJSON(w, resp.content, daemon.Debug)
+ } else if resp.ctype != "websocket" {
+ w.Header().Set("Content-Type", "application/octet-stream")
+ fmt.Fprintf(w, resp.content.(string))
+ }
+ }
+}
+
+func devLxdAPI(d *Daemon) http.Handler {
+ m := mux.NewRouter()
+
+ for _, handler := range handlers {
+ m.HandleFunc(handler.path, hoistReq(handler.f, d))
+ }
+
+ return m
+}
+
+// Create a new net.Listener bound to the unix socket of the devlxd endpoint.
+func createDevLxdlListener(dir string) (net.Listener, error) {
+ path := filepath.Join(dir, "lxd", "sock")
+
+ err := os.MkdirAll(filepath.Dir(path), 0755)
+ if err != nil {
+ return nil, err
+ }
+
+ // If this socket exists, that means a previous LXD instance died and
+ // didn't clean up. We assume that such LXD instance is actually dead
+ // if we get this far, since localCreateListener() tries to connect to
+ // the actual lxd socket to make sure that it is actually dead. So, it
+ // is safe to remove it here without any checks.
+ //
+ // Also, it would be nice to SO_REUSEADDR here so we don't have to
+ // delete the socket, but we can't:
+ // http://stackoverflow.com/questions/15716302/so-reuseaddr-and-af-unix
+ //
+ // Note that this will force clients to reconnect when LXD is restarted.
+ err = socketUnixRemoveStale(path)
+ if err != nil {
+ return nil, err
+ }
+
+ listener, err := socketUnixListen(path)
+ if err != nil {
+ return nil, err
+ }
+
+ err = socketUnixSetPermissions(path, 0600)
+ if err != nil {
+ listener.Close()
+ return nil, err
+ }
+
+ return listener, nil
+}
+
+// Remove any stale socket file at the given path.
+func socketUnixRemoveStale(path string) error {
+ // If there's no socket file at all, there's nothing to do.
+ if !shared.PathExists(path) {
+ return nil
+ }
+
+ logger.Debugf("Detected stale unix socket, deleting")
+ err := os.Remove(path)
+ if err != nil {
+ return fmt.Errorf("could not delete stale local socket: %v", err)
+ }
+
+ return nil
+}
+
+// Change the file mode of the given unix socket file,
+func socketUnixSetPermissions(path string, mode os.FileMode) error {
+ err := os.Chmod(path, mode)
+ if err != nil {
+ return fmt.Errorf("cannot set permissions on local socket: %v", err)
+ }
+ return nil
+}
+
+// Bind to the given unix socket path.
+func socketUnixListen(path string) (net.Listener, error) {
+ addr, err := net.ResolveUnixAddr("unix", path)
+ if err != nil {
+ return nil, fmt.Errorf("cannot resolve socket address: %v", err)
+ }
+
+ listener, err := net.ListenUnix("unix", addr)
+ if err != nil {
+ return nil, fmt.Errorf("cannot bind socket: %v", err)
+ }
+
+ return listener, err
+
+}
diff --git a/lxd-agent/events.go b/lxd-agent/events.go
index 33be3c2a1a..e88b4e6914 100644
--- a/lxd-agent/events.go
+++ b/lxd-agent/events.go
@@ -2,18 +2,21 @@ package main
import (
"context"
+ "encoding/json"
"net/http"
"strings"
"github.com/lxc/lxd/lxd/response"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/logger"
)
var eventsCmd = APIEndpoint{
Path: "events",
- Get: APIEndpointAction{Handler: eventsGet},
+ Get: APIEndpointAction{Handler: eventsGet},
+ Post: APIEndpointAction{Handler: eventsPost},
}
type eventsServe struct {
@@ -32,7 +35,7 @@ func (r *eventsServe) String() string {
func eventsSocket(d *Daemon, r *http.Request, w http.ResponseWriter) error {
typeStr := r.FormValue("type")
if typeStr == "" {
- typeStr = "logging,operation,lifecycle"
+ typeStr = "logging,operation,lifecycle,config"
}
// Upgrade the connection to websocket
@@ -83,3 +86,19 @@ func eventsSocket(d *Daemon, r *http.Request, w http.ResponseWriter) error {
func eventsGet(d *Daemon, r *http.Request) response.Response {
return &eventsServe{req: r, d: d}
}
+
+func eventsPost(d *Daemon, r *http.Request) response.Response {
+ var event api.Event
+
+ err := json.NewDecoder(r.Body).Decode(&event)
+ if err != nil {
+ return response.InternalError(err)
+ }
+
+ err = d.events.Send("", event.Type, event.Metadata)
+ if err != nil {
+ return response.InternalError(err)
+ }
+
+ return response.SyncResponse(true, nil)
+}
diff --git a/lxd-agent/main_agent.go b/lxd-agent/main_agent.go
index 887428f12f..da77f9755c 100644
--- a/lxd-agent/main_agent.go
+++ b/lxd-agent/main_agent.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/ioutil"
+ "net/http"
"os"
"os/signal"
"strings"
@@ -135,8 +136,18 @@ func (c *cmdAgent) Run(cmd *cobra.Command, args []string) error {
d := newDaemon(c.global.flagLogDebug, c.global.flagLogVerbose)
+ servers := make(map[string]*http.Server, 2)
+
// Prepare the HTTP server.
- httpServer := restServer(tlsConfig, cert, c.global.flagLogDebug, d)
+ servers["http"] = restServer(tlsConfig, cert, c.global.flagLogDebug, d)
+
+ // Prepare the devlxd server.
+ devlxdListener, err := createDevLxdlListener("/dev")
+ if err != nil {
+ return err
+ }
+
+ servers["devlxd"] = devLxdServer(d)
// Create a cancellation context.
ctx, cancelFunc := context.WithCancel(context.Background())
@@ -144,17 +155,38 @@ func (c *cmdAgent) Run(cmd *cobra.Command, args []string) error {
// Start status notifier in background.
go c.startStatusNotifier(ctx)
+ errChan := make(chan error, 1)
+
+ // Start the server.
+ go func() {
+ err := servers["http"].ServeTLS(networkTLSListener(l, tlsConfig), "agent.crt", "agent.key")
+ if err != nil {
+ errChan <- err
+ }
+ }()
+
+ go func() {
+ err := servers["devlxd"].Serve(devlxdListener)
+ if err != nil {
+ errChan <- err
+ }
+ }()
+
// Cancel context when SIGTEM is received.
chSignal := make(chan os.Signal, 1)
signal.Notify(chSignal, unix.SIGTERM)
- go func() {
- <-chSignal
+
+ select {
+ case <-chSignal:
cancelFunc()
os.Exit(0)
- }()
+ case err := <-errChan:
+ fmt.Fprintln(os.Stderr, err)
+ cancelFunc()
+ os.Exit(1)
+ }
- // Start the server.
- return httpServer.ServeTLS(networkTLSListener(l, tlsConfig), "agent.crt", "agent.key")
+ return nil
}
// startStatusNotifier sends status of agent to vserial ring buffer every 10s or when context is done.
From da619b3f00883cedcd2913a851ee1e6fa828f8a5 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Thu, 2 Jul 2020 23:08:55 +0200
Subject: [PATCH 3/3] lxd/instance/drivers: Allow updating running VMs
This allows user.* config keys to be updated on running VMs.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 165 +++++++++++++++++++++++++++-
1 file changed, 164 insertions(+), 1 deletion(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 634f978a18..f571e7c6b8 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -2577,8 +2577,130 @@ func (vm *qemu) Rename(newName string) error {
// Update the instance config.
func (vm *qemu) Update(args db.InstanceArgs, userRequested bool) error {
+ // Only user.* keys can be changed on a running VM
if vm.IsRunning() {
- return fmt.Errorf("Update whilst running not supported")
+ if args.Config == nil {
+ args.Config = map[string]string{}
+ }
+
+ if userRequested {
+ // Validate the new config.
+ err := instance.ValidConfig(vm.state.OS, args.Config, false, false)
+ if err != nil {
+ return errors.Wrap(err, "Invalid config")
+ }
+ }
+
+ oldExpandedConfig := map[string]string{}
+ err := shared.DeepCopy(&vm.expandedConfig, &oldExpandedConfig)
+ if err != nil {
+ return err
+ }
+
+ oldLocalConfig := map[string]string{}
+ err = shared.DeepCopy(&vm.localConfig, &oldLocalConfig)
+ if err != nil {
+ return err
+ }
+
+ undoChanges := true
+ defer func() {
+ if undoChanges {
+ vm.expandedConfig = oldExpandedConfig
+ vm.localConfig = oldLocalConfig
+ }
+ }()
+
+ vm.localConfig = args.Config
+
+ // Expand the config and refresh the LXC config.
+ err = vm.expandConfig(nil)
+ if err != nil {
+ return errors.Wrap(err, "Expand config")
+ }
+
+ // Diff the configurations.
+ changedConfig := []string{}
+ for key := range oldExpandedConfig {
+ if oldExpandedConfig[key] != vm.expandedConfig[key] {
+ if !shared.StringInSlice(key, changedConfig) {
+ changedConfig = append(changedConfig, key)
+ }
+ }
+ }
+
+ for key := range vm.expandedConfig {
+ if oldExpandedConfig[key] != vm.expandedConfig[key] {
+ if !shared.StringInSlice(key, changedConfig) {
+ changedConfig = append(changedConfig, key)
+ }
+ }
+ }
+
+ for _, key := range changedConfig {
+ if !strings.HasPrefix(key, "user.") {
+ return fmt.Errorf("Only user.* keys can be updated on running VMs")
+ }
+ }
+
+ if userRequested {
+ // Do some validation of the config diff.
+ err = instance.ValidConfig(vm.state.OS, vm.expandedConfig, false, true)
+ if err != nil {
+ return errors.Wrap(err, "Invalid expanded config")
+ }
+ }
+
+ err = vm.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
+ object, err := tx.GetInstance(vm.project, vm.name)
+ if err != nil {
+ return err
+ }
+
+ object.Config = vm.localConfig
+
+ return tx.UpdateInstance(vm.project, vm.name, *object)
+ })
+ if err != nil {
+ return errors.Wrap(err, "Failed to update database")
+ }
+
+ err = vm.UpdateBackupFile()
+ if err != nil && !os.IsNotExist(err) {
+ return errors.Wrap(err, "Failed to write backup file")
+ }
+
+ // Success, update the closure to mark that the changes should be kept.
+ undoChanges = false
+
+ err = vm.writeInstanceData()
+ if err != nil {
+ return errors.Wrap(err, "Failed to write instance-data file")
+ }
+
+ // Send devlxd notifications only for user.* key changes
+ for _, key := range changedConfig {
+ if !strings.HasPrefix(key, "user.") {
+ continue
+ }
+
+ msg := map[string]string{
+ "key": key,
+ "old_value": oldExpandedConfig[key],
+ "value": vm.expandedConfig[key],
+ }
+
+ err = vm.devlxdEventSend("config", msg)
+ if err != nil {
+ return err
+ }
+ }
+
+ endpoint := fmt.Sprintf("/1.0/virtual-machines/%s", vm.name)
+
+ vm.state.Events.SendLifecycle(vm.project, "virtual-machine-updated", endpoint, nil)
+
+ return nil
}
// Set sane defaults for unset keys.
@@ -4595,6 +4717,47 @@ func (vm *qemu) cpuTopology(limit string) (int, int, int, map[uint64]uint64, map
return nrSockets, nrCores, nrThreads, vcpus, numaNodes, nil
}
+
+func (vm *qemu) expandConfig(profiles []api.Profile) error {
+ if profiles == nil && len(vm.profiles) > 0 {
+ var err error
+ profiles, err = vm.state.Cluster.GetProfiles(vm.project, vm.profiles)
+ if err != nil {
+ return err
+ }
+ }
+
+ vm.expandedConfig = db.ExpandInstanceConfig(vm.localConfig, profiles)
+
+ return nil
+}
+
+func (vm *qemu) devlxdEventSend(eventType string, eventMessage interface{}) error {
+ event := shared.Jmap{}
+ event["type"] = eventType
+ event["timestamp"] = time.Now()
+ event["metadata"] = eventMessage
+
+ client, err := vm.getAgentClient()
+ if err != nil {
+ return err
+ }
+
+ agent, err := lxdClient.ConnectLXDHTTP(nil, client)
+ if err != nil {
+ logger.Errorf("Failed to connect to lxd-agent on %s: %v", vm.Name(), err)
+ return fmt.Errorf("Failed to connect to lxd-agent")
+ }
+ defer agent.Disconnect()
+
+ _, _, err = agent.RawQuery("POST", "/1.0/events", &event, "")
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
func (vm *qemu) writeInstanceData() error {
// Instance data for devlxd.
configDrivePath := filepath.Join(vm.Path(), "config")
More information about the lxc-devel
mailing list