[lxc-devel] [lxd/master] VM: Adds MACVLAN support

tomponline on Github lxc-bot at linuxcontainers.org
Wed Jan 22 18:29:10 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200122/20fd0c38/attachment.bin>
-------------- next part --------------
From 2e4484146b2d6ad039439f4e648dc50fe6c3079b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 22 Jan 2020 18:15:05 +0000
Subject: [PATCH 1/4] lxd/device: Relaxes requirement for name property when
 not using containers

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/infiniband_physical.go | 2 +-
 lxd/device/infiniband_sriov.go    | 2 +-
 lxd/device/nic_bridged.go         | 2 +-
 lxd/device/nic_ipvlan.go          | 2 +-
 lxd/device/nic_macvlan.go         | 3 ++-
 lxd/device/nic_p2p.go             | 2 +-
 lxd/device/nic_physical.go        | 2 +-
 lxd/device/nic_routed.go          | 2 +-
 lxd/device/nic_sriov.go           | 2 +-
 9 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/lxd/device/infiniband_physical.go b/lxd/device/infiniband_physical.go
index f1ba582621..6c6947b5bc 100644
--- a/lxd/device/infiniband_physical.go
+++ b/lxd/device/infiniband_physical.go
@@ -45,7 +45,7 @@ func (d *infinibandPhysical) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *infinibandPhysical) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/infiniband_sriov.go b/lxd/device/infiniband_sriov.go
index ea37fda93c..cb673e5ee5 100644
--- a/lxd/device/infiniband_sriov.go
+++ b/lxd/device/infiniband_sriov.go
@@ -46,7 +46,7 @@ func (d *infinibandSRIOV) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *infinibandSRIOV) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_bridged.go b/lxd/device/nic_bridged.go
index 6359052328..dd1e534799 100644
--- a/lxd/device/nic_bridged.go
+++ b/lxd/device/nic_bridged.go
@@ -83,7 +83,7 @@ func (d *nicBridged) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicBridged) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go
index 480b8af1df..ee61a8353e 100644
--- a/lxd/device/nic_ipvlan.go
+++ b/lxd/device/nic_ipvlan.go
@@ -58,7 +58,7 @@ func (d *nicIPVLAN) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicIPVLAN) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_macvlan.go b/lxd/device/nic_macvlan.go
index 5471bc869b..45df01b8d2 100644
--- a/lxd/device/nic_macvlan.go
+++ b/lxd/device/nic_macvlan.go
@@ -5,6 +5,7 @@ import (
 
 	deviceConfig "github.com/lxc/lxd/lxd/device/config"
 	"github.com/lxc/lxd/lxd/instance/instancetype"
+	"github.com/lxc/lxd/lxd/revert"
 	"github.com/lxc/lxd/shared"
 )
 
@@ -30,7 +31,7 @@ func (d *nicMACVLAN) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicMACVLAN) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_p2p.go b/lxd/device/nic_p2p.go
index 038cb420a2..0bfa97462f 100644
--- a/lxd/device/nic_p2p.go
+++ b/lxd/device/nic_p2p.go
@@ -40,7 +40,7 @@ func (d *nicP2P) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicP2P) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_physical.go b/lxd/device/nic_physical.go
index 09f4002568..54d98c6a0d 100644
--- a/lxd/device/nic_physical.go
+++ b/lxd/device/nic_physical.go
@@ -38,7 +38,7 @@ func (d *nicPhysical) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicPhysical) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_routed.go b/lxd/device/nic_routed.go
index 8b9db5eb31..3f6d0cb66f 100644
--- a/lxd/device/nic_routed.go
+++ b/lxd/device/nic_routed.go
@@ -63,7 +63,7 @@ func (d *nicRouted) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicRouted) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 
diff --git a/lxd/device/nic_sriov.go b/lxd/device/nic_sriov.go
index 7c426e01e5..b9fb4a8f30 100644
--- a/lxd/device/nic_sriov.go
+++ b/lxd/device/nic_sriov.go
@@ -49,7 +49,7 @@ func (d *nicSRIOV) validateConfig() error {
 
 // validateEnvironment checks the runtime environment for correctness.
 func (d *nicSRIOV) validateEnvironment() error {
-	if d.config["name"] == "" {
+	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
 		return fmt.Errorf("Requires name property to start")
 	}
 

From 254921475b1d9837cb1ea20e756131df1c2285b7 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 22 Jan 2020 18:15:58 +0000
Subject: [PATCH 2/4] lxd/device/nic/macvlan: Clean up valid fields

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/nic_macvlan.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lxd/device/nic_macvlan.go b/lxd/device/nic_macvlan.go
index 45df01b8d2..e1a9780748 100644
--- a/lxd/device/nic_macvlan.go
+++ b/lxd/device/nic_macvlan.go
@@ -20,7 +20,15 @@ func (d *nicMACVLAN) validateConfig() error {
 	}
 
 	requiredFields := []string{"parent"}
-	optionalFields := []string{"name", "mtu", "hwaddr", "vlan", "maas.subnet.ipv4", "maas.subnet.ipv6", "boot.priority"}
+	optionalFields := []string{
+		"name",
+		"mtu",
+		"hwaddr",
+		"vlan",
+		"maas.subnet.ipv4",
+		"maas.subnet.ipv6",
+		"boot.priority",
+	}
 	err := d.config.Validate(nicValidationRules(requiredFields, optionalFields))
 	if err != nil {
 		return err

From 860005ce080d1bf0167d36a7f462242103d32fa3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 22 Jan 2020 18:17:55 +0000
Subject: [PATCH 3/4] lxd/device/nic/macvlan: Adds VM support and improves
 revert

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/nic_macvlan.go | 52 +++++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 13 deletions(-)

diff --git a/lxd/device/nic_macvlan.go b/lxd/device/nic_macvlan.go
index e1a9780748..43cb93a651 100644
--- a/lxd/device/nic_macvlan.go
+++ b/lxd/device/nic_macvlan.go
@@ -15,7 +15,7 @@ type nicMACVLAN struct {
 
 // validateConfig checks the supplied config for correctness.
 func (d *nicMACVLAN) validateConfig() error {
-	if d.inst.Type() != instancetype.Container {
+	if d.inst.Type() != instancetype.Container && d.inst.Type() != instancetype.VM {
 		return ErrUnsupportedDevType
 	}
 
@@ -61,6 +61,9 @@ func (d *nicMACVLAN) Start() (*deviceConfig.RunConfig, error) {
 	networkCreateSharedDeviceLock.Lock()
 	defer networkCreateSharedDeviceLock.Unlock()
 
+	revert := revert.New()
+	defer revert.Fail()
+
 	saveData := make(map[string]string)
 
 	// Decide which parent we should use based on VLAN setting.
@@ -78,20 +81,32 @@ func (d *nicMACVLAN) Start() (*deviceConfig.RunConfig, error) {
 	// Record whether we created the parent device or not so it can be removed on stop.
 	saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing")
 
-	// Create MACVLAN interface.
-	_, err = shared.RunCommand("ip", "link", "add", "dev", saveData["host_name"], "link", parentName, "type", "macvlan", "mode", "bridge")
-	if err != nil {
-		return nil, err
+	if shared.IsTrue(saveData["last_state.created"]) {
+		revert.Add(func() {
+			NetworkRemoveInterfaceIfNeeded(d.state, parentName, d.inst, d.config["parent"], d.config["vlan"])
+		})
+	}
+
+	if d.inst.Type() == instancetype.Container {
+		// Create MACVLAN interface.
+		_, err = shared.RunCommand("ip", "link", "add", "dev", saveData["host_name"], "link", parentName, "type", "macvlan", "mode", "bridge")
+		if err != nil {
+			return nil, err
+		}
+	} else if d.inst.Type() == instancetype.VM {
+		// Create MACVTAP interface.
+		_, err = shared.RunCommand("ip", "link", "add", "dev", saveData["host_name"], "link", parentName, "type", "macvtap", "mode", "bridge")
+		if err != nil {
+			return nil, err
+		}
 	}
 
+	revert.Add(func() { NetworkRemoveInterface(saveData["host_name"]) })
+
 	// Set the MAC address.
 	if d.config["hwaddr"] != "" {
 		_, err := shared.RunCommand("ip", "link", "set", "dev", saveData["host_name"], "address", d.config["hwaddr"])
 		if err != nil {
-			if statusDev == "created" {
-				NetworkRemoveInterface(saveData["host_name"])
-			}
-
 			return nil, fmt.Errorf("Failed to set the MAC address: %s", err)
 		}
 	}
@@ -100,14 +115,18 @@ func (d *nicMACVLAN) Start() (*deviceConfig.RunConfig, error) {
 	if d.config["mtu"] != "" {
 		_, err := shared.RunCommand("ip", "link", "set", "dev", saveData["host_name"], "mtu", d.config["mtu"])
 		if err != nil {
-			if statusDev == "created" {
-				NetworkRemoveInterface(saveData["host_name"])
-			}
-
 			return nil, fmt.Errorf("Failed to set the MTU: %s", err)
 		}
 	}
 
+	if d.inst.Type() == instancetype.VM {
+		// Bring the interface up on host side.
+		_, err := shared.RunCommand("ip", "link", "set", "dev", saveData["host_name"], "up")
+		if err != nil {
+			return nil, fmt.Errorf("Failed to bring up interface %s: %v", saveData["host_name"], err)
+		}
+	}
+
 	err = d.volatileSet(saveData)
 	if err != nil {
 		return nil, err
@@ -122,6 +141,13 @@ func (d *nicMACVLAN) Start() (*deviceConfig.RunConfig, error) {
 		{Key: "link", Value: saveData["host_name"]},
 	}
 
+	if d.inst.Type() == instancetype.VM {
+		runConf.NetworkInterface = append(runConf.NetworkInterface,
+			deviceConfig.RunConfigItem{Key: "hwaddr", Value: d.config["hwaddr"]},
+		)
+	}
+
+	revert.Success()
 	return &runConf, nil
 }
 

From 9fc15f83510188df9dfc0031c2d72d262d8b4730 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 22 Jan 2020 18:18:45 +0000
Subject: [PATCH 4/4] lxd/instance/drivers/driver/qemu: Adds macvtap support

Uses macvtap device created by device layer to get a file handle to it and pass it to qemu to use.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/instance/drivers/driver_qemu.go | 60 +++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 8 deletions(-)

diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 0639b907b2..a86bc54aa6 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -657,7 +657,10 @@ func (vm *qemu) Start(stateful bool) error {
 		return err
 	}
 
-	confFile, err := vm.generateQemuConfigFile(qemuType, qemuConfig, devConfs)
+	// Define a set of files to open and pass their file descriptors to qemu command.
+	fdFiles := make([]string, 0)
+
+	confFile, err := vm.generateQemuConfigFile(qemuType, qemuConfig, devConfs, &fdFiles)
 	if err != nil {
 		op.Done(err)
 		return err
@@ -726,8 +729,25 @@ func (vm *qemu) Start(stateful bool) error {
 		args = append(args, fields...)
 	}
 
-	_, err = shared.RunCommand(qemuBinary, args...)
+	cmd := exec.Command(qemuBinary, args...)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	// Open any extra files and pass their file handles to qemu command.
+	for _, file := range fdFiles {
+		f, err := os.OpenFile(file, os.O_RDWR, 0)
+		if err != nil {
+			return errors.Wrapf(err, "Error opening exta file %q", file)
+		}
+		defer f.Close() // Close file after qemu has started.
+		cmd.ExtraFiles = append(cmd.ExtraFiles, f)
+	}
+
+	err = cmd.Run()
 	if err != nil {
+		err = errors.Wrapf(err, "Failed to run: %s %s: %s", qemuBinary, strings.Join(args, " "), strings.TrimSpace(string(stderr.Bytes())))
 		op.Done(err)
 		return err
 	}
@@ -1180,7 +1200,7 @@ func (vm *qemu) deviceBootPriorities() (map[string]int, error) {
 
 // generateQemuConfigFile writes the qemu config file and returns its location.
 // It writes the config file inside the VM's log path.
-func (vm *qemu) generateQemuConfigFile(qemuType string, qemuConf string, devConfs []*deviceConfig.RunConfig) (string, error) {
+func (vm *qemu) generateQemuConfigFile(qemuType string, qemuConf string, devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
 	var sb *strings.Builder = &strings.Builder{}
 
 	// Base config. This is common for all VMs and has no variables in it.
@@ -1324,7 +1344,7 @@ backend = "pty"
 
 		// Add network device.
 		if len(runConf.NetworkInterface) > 0 {
-			err = vm.addNetDevConfig(sb, nicIndex, bootIndexes, runConf.NetworkInterface)
+			err = vm.addNetDevConfig(sb, nicIndex, bootIndexes, runConf.NetworkInterface, fdFiles)
 			if err != nil {
 				return "", err
 			}
@@ -1548,26 +1568,49 @@ bootindex = "{{.bootIndex}}"
 }
 
 // addNetDevConfig adds the qemu config required for adding a network device.
-func (vm *qemu) addNetDevConfig(sb *strings.Builder, nicIndex int, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem) error {
-	var devName, devTap, devHwaddr string
+func (vm *qemu) addNetDevConfig(sb *strings.Builder, nicIndex int, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
+	var devName, nicName, devHwaddr string
+	var tapFD int
 	for _, nicItem := range nicConfig {
 		if nicItem.Key == "devName" {
 			devName = nicItem.Value
 		} else if nicItem.Key == "link" {
-			devTap = nicItem.Value
+			nicName = nicItem.Value
 		} else if nicItem.Key == "hwaddr" {
 			devHwaddr = nicItem.Value
 		}
 	}
 
+	// Detect MACVTAP interface types and figure out which tap device is being used.
+	// This is so we can open a file handle to the tap device and pass it to the qemu process.
+	if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/macvtap", nicName)) {
+		content, err := ioutil.ReadFile(fmt.Sprintf("/sys/class/net/%s/ifindex", nicName))
+		if err != nil {
+			return errors.Wrapf(err, "Error getting tap device ifindex")
+		}
+
+		ifindex, err := strconv.Atoi(strings.TrimSpace(string(content)))
+		if err != nil {
+			return errors.Wrapf(err, "Error parsing tap device ifindex")
+		}
+
+		// Append the tap device file path to the list of files to be opened and passed to qemu.
+		*fdFiles = append(*fdFiles, fmt.Sprintf("/dev/tap%d", ifindex))
+		tapFD = 2 + len(*fdFiles) // Use 2+fdFiles count, as first file descriptor available is 3.
+	}
+
 	// Devices use "lxd_" prefix indicating that this is a user named device.
 	t := template.Must(template.New("").Parse(`
 # Network card ("{{.devName}}" device)
 [netdev "lxd_{{.devName}}"]
 type = "tap"
+{{ if ne .tapFD 0 }}
+fd = "{{.tapFD}}"
+{{ else }}
 ifname = "{{.ifName}}"
 script = "no"
 downscript = "no"
+{{ end }}
 
 [device "qemu_pcie{{.chassisIndex}}"]
 driver = "pcie-root-port"
@@ -1587,12 +1630,13 @@ bootindex = "{{.bootIndex}}"
 
 	m := map[string]interface{}{
 		"devName":      devName,
-		"ifName":       devTap,
+		"ifName":       nicName,
 		"chassisIndex": 5 + nicIndex,
 		"portIndex":    14 + nicIndex,
 		"pcieAddr":     4 + nicIndex,
 		"devHwaddr":    devHwaddr,
 		"bootIndex":    bootIndexes[devName],
+		"tapFD":        tapFD,
 	}
 	return t.Execute(sb, m)
 }


More information about the lxc-devel mailing list