[lxc-devel] [lxd/master] Support virtio-fs
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Tue Oct 20 07:45:15 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201020/cdf7265e/attachment.bin>
-------------- next part --------------
From 8bcad318eea963979740453fcf9e28884f0fffa4 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 20 Oct 2020 08:39:23 +0200
Subject: [PATCH 1/5] lxd/instance/drivers: Change memory backend
This changes the memory backend to memory-backend-memfd. Also, this
turns on `share`. Both are required for virtio-fs.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/instance/drivers/driver_qemu_templates.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index 29d4aa6b40..17051c1510 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -279,9 +279,10 @@ mem-path = "{{$hugepages}}"
prealloc = "on"
discard-data = "on"
{{- else}}
-qom-type = "memory-backend-ram"
+qom-type = "memory-backend-memfd"
{{- end }}
size = "{{$memory}}M"
+share = "on"
[numa]
type = "node"
From a4aeded8bcd15f278acd58e8d5ea2427c718c79f Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 20 Oct 2020 09:15:48 +0200
Subject: [PATCH 2/5] lxd/instance/drivers: Add qemu virtio-fs template
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/instance/drivers/driver_qemu_templates.go | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index 17051c1510..306e6a0412 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -541,3 +541,15 @@ driver = "usb-host"
bus = "qemu_usb.0"
hostdevice = "{{.hostDevice}}"
`))
+
+var qemuVirtioFSDev = template.Must(template.New("qemuVirtioFSDev").Parse(`
+# Virtio FS
+[chardev "qemu_virtiofs-{{.devName}}"]
+backend = "socket"
+path = "{{.path}}"
+
+[device "dev-lxd_{{.devName}}"]
+driver = "vhost-user-fs-pci"
+chardev = "qemu_virtiofs-{{.devName}}"
+tag = "{{.tag}}"
+`))
From c384d07f23d503153d577d3bb84ac93a01894888 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 20 Oct 2020 09:24:11 +0200
Subject: [PATCH 3/5] lxd/device/disk: Support virtio-fs
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/device/disk.go | 64 +++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 61 insertions(+), 3 deletions(-)
diff --git a/lxd/device/disk.go b/lxd/device/disk.go
index 4f728007e7..4b5797cecf 100644
--- a/lxd/device/disk.go
+++ b/lxd/device/disk.go
@@ -152,7 +152,7 @@ func (d *disk) validateConfig(instConf instance.ConfigReader) error {
// contains the name of the storage volume, not the path where it is mounted. So only check
// for the existence of "source" when "pool" is empty and ceph type source not being used.
if d.config["pool"] == "" && d.config["source"] != "" && d.config["source"] != diskSourceCloudInit && d.isRequired(d.config) && !shared.PathExists(shared.HostPath(d.config["source"])) &&
- !strings.HasPrefix(d.config["source"], "ceph:") && !strings.HasPrefix(d.config["source"], "cephfs:") {
+ !strings.HasPrefix(d.config["source"], "ceph:") && !strings.HasPrefix(d.config["source"], "cephfs:") && !strings.HasPrefix(d.config["source"], "virtiofs:") {
return fmt.Errorf("Missing source %q for disk %q", d.config["source"], d.name)
}
@@ -449,6 +449,53 @@ func (d *disk) startVM() (*deviceConfig.RunConfig, error) {
DevName: d.name,
},
}
+ } else if strings.HasPrefix(d.config["source"], "virtiofs:") {
+ fields := strings.SplitN(d.config["source"], ":", 2)
+ srcPath := fields[1]
+ // Create the socket in this directory instead of the devices directory. QEMU will otherwise
+ // fail with "Permission Denied" which is probably caused by qemu being called with --chroot.
+ sockPath := filepath.Join(d.inst.Path(), fmt.Sprintf("%s.sock", d.name))
+ logPath := filepath.Join(d.inst.LogPath(), fmt.Sprintf("disk.%s.log", d.name))
+
+ // Remove old socket if needed.
+ os.Remove(sockPath)
+
+ // Start the virtiofsd process in non-daemon mode.
+ proc, err := subprocess.NewProcess("/usr/lib/qemu/virtiofsd", []string{fmt.Sprintf("--socket-path=%s", sockPath), "-o", fmt.Sprintf("source=%s", srcPath)}, logPath, logPath)
+ if err != nil {
+ return nil, err
+ }
+
+ err = proc.Start()
+ if err != nil {
+ return nil, errors.Wrapf(err, "Failed to start virtiofsd for device %q", d.name)
+ }
+
+ revert.Add(func() { proc.Stop() })
+
+ pidPath := filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("%s.pid", d.name))
+ err = proc.Save(pidPath)
+ if err != nil {
+ return nil, errors.Wrapf(err, "Failed to save virtiofsd state for device %q", d.name)
+ }
+
+ // Wait for socket file to exist (as otherwise qemu can race the creation of this file).
+ for i := 0; i < 10; i++ {
+ if shared.PathExists(sockPath) {
+ break
+ }
+
+ time.Sleep(50 * time.Millisecond)
+ }
+
+ runConf.Mounts = []deviceConfig.MountEntryItem{
+ {
+ DevName: d.name,
+ DevPath: sockPath,
+ FSType: "virtiofs",
+ TargetPath: d.config["path"],
+ },
+ }
} else {
srcPath := shared.HostPath(d.config["source"])
var err error
@@ -1139,13 +1186,24 @@ func (d *disk) stopVM() (*deviceConfig.RunConfig, error) {
}
err = proc.Stop()
- if err != nil {
+ // virtiofsd will terminate automatically once the VM has stopped. We therefore should only
+ // return an error if it's a running process.
+ if err != nil && err != subprocess.ErrNotRunning {
return &deviceConfig.RunConfig{}, err
}
// Remove PID file and socket file.
os.Remove(pidPath)
- os.Remove(filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("%s.sock", d.name)))
+
+ var sockPath string
+
+ if strings.HasPrefix(d.config["source"], "virtiofs:") {
+ sockPath = filepath.Join(d.inst.Path(), fmt.Sprintf("%s.sock", d.name))
+ } else {
+ sockPath = filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("%s.sock", d.name))
+ }
+
+ os.Remove(sockPath)
}
runConf := deviceConfig.RunConfig{
From 7279b2569967d36cd5d7705de6189bbedb6232a8 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 20 Oct 2020 09:24:25 +0200
Subject: [PATCH 4/5] lxd/instance/drivers: Support virtio-fs
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 1abaddbbb2..446ddc3f97 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1823,7 +1823,7 @@ func (vm *qemu) generateQemuConfigFile(busName string, devConfs []*deviceConfig.
for _, drive := range runConf.Mounts {
if drive.TargetPath == "/" {
err = vm.addRootDriveConfig(sb, bootIndexes, drive)
- } else if drive.FSType == "9p" {
+ } else if drive.FSType == "9p" || drive.FSType == "virtiofs" {
err = vm.addDriveDirConfig(sb, bus, fdFiles, &agentMounts, drive)
} else {
err = vm.addDriveConfig(sb, bootIndexes, drive)
@@ -2056,6 +2056,14 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus *qemuBus, fdFiles *[]
devBus, devAddr, multi := bus.allocate(busFunctionGroup9p)
+ if driveConf.FSType == "virtiofs" {
+ return qemuVirtioFSDev.Execute(sb, map[string]interface{}{
+ "devName": driveConf.DevName,
+ "path": driveConf.DevPath,
+ "tag": mountTag,
+ })
+ }
+
// For read only shares, do not use proxy.
if shared.StringInSlice("ro", driveConf.Opts) {
return qemuDriveDir.Execute(sb, map[string]interface{}{
From 6240c1951f02d8cf063eda8dbc56e5eab7c5a3d2 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 20 Oct 2020 09:24:57 +0200
Subject: [PATCH 5/5] doc: Add virtio-fs
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
doc/instances.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/doc/instances.md b/doc/instances.md
index 10e4630a33..ca209e7a4a 100644
--- a/doc/instances.md
+++ b/doc/instances.md
@@ -662,6 +662,11 @@ Example command.
lxc config device add <instance> config disk source=cloud-init:config
```
+- Virtio-fs: A shared file system that lets virtual machines access a directory tree on the host. It is designed to offer local file system semantics and performance.
+```
+lxc config device add <instance> config disk source=virtiofs:/<some-path> path=/virtiofs
+```
+
Currently only the root disk (path=/) and config drive (source=cloud-init:config) are supported with virtual machines.
More information about the lxc-devel
mailing list