[lxc-devel] [lxd/master] Network: Adds support for OVN physical uplink interface to be a bridge

tomponline on Github lxc-bot at linuxcontainers.org
Wed Dec 16 12:14:04 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 516 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201216/a28b7bdb/attachment.bin>
-------------- next part --------------
From 12ae61c323fd04f2b18a9076c88e2ca545484d93 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 16 Dec 2020 12:12:41 +0000
Subject: [PATCH] lxd/network/driver/ovn: Adds support for physical uplink
 interface to be a bridge

Either native or OVS. And then uses the existing connection functions used for managed bridge uplinks to connect OVN router to uplink.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/network/driver_ovn.go | 313 +++++++++++++++++++++++---------------
 1 file changed, 188 insertions(+), 125 deletions(-)

diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index 8b17895a8c..3e68e6018d 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -878,88 +878,118 @@ func (n *ovn) uplinkPortBridgeVars(uplinkNet Network) *ovnUplinkPortBridgeVars {
 // startUplinkPortBridge creates veth pair (if doesn't exist), creates OVS bridge (if doesn't exist) and
 // connects veth pair to uplink bridge and OVS bridge.
 func (n *ovn) startUplinkPortBridge(uplinkNet Network) error {
+	if uplinkNet.Config()["bridge.driver"] != "openvswitch" {
+		return n.startUplinkPortBridgeNative(uplinkNet, uplinkNet.Name())
+	}
+
+	return n.startUplinkPortBridgeOVS(uplinkNet, uplinkNet.Name())
+}
+
+// startUplinkPortBridgeNative connects an OVN logical router to an uplink native bridge.
+func (n *ovn) startUplinkPortBridgeNative(uplinkNet Network, bridgeDevice string) error {
 	// Do this after gaining lock so that on failure we revert before release locking.
 	revert := revert.New()
 	defer revert.Fail()
 
-	ovs := openvswitch.NewOVS()
-
 	// If uplink is a native bridge, then use a separate OVS bridge with veth pair connection to native bridge.
-	if uplinkNet.Config()["bridge.driver"] != "openvswitch" {
-		vars := n.uplinkPortBridgeVars(uplinkNet)
-
-		// Create veth pair if needed.
-		if !InterfaceExists(vars.uplinkEnd) && !InterfaceExists(vars.ovsEnd) {
-			_, err := shared.RunCommand("ip", "link", "add", "dev", vars.uplinkEnd, "type", "veth", "peer", "name", vars.ovsEnd)
-			if err != nil {
-				return errors.Wrapf(err, "Failed to create the uplink veth interfaces %q and %q", vars.uplinkEnd, vars.ovsEnd)
-			}
+	vars := n.uplinkPortBridgeVars(uplinkNet)
 
-			revert.Add(func() { shared.RunCommand("ip", "link", "delete", vars.uplinkEnd) })
+	// Create veth pair if needed.
+	if !InterfaceExists(vars.uplinkEnd) && !InterfaceExists(vars.ovsEnd) {
+		_, err := shared.RunCommand("ip", "link", "add", "dev", vars.uplinkEnd, "type", "veth", "peer", "name", vars.ovsEnd)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to create the uplink veth interfaces %q and %q", vars.uplinkEnd, vars.ovsEnd)
 		}
 
-		// Ensure that the veth interfaces inherit the uplink bridge's MTU (which the OVS bridge also inherits).
-		uplinkNetConfig := uplinkNet.Config()
-		if uplinkNetConfig["bridge.mtu"] != "" {
-			err := InterfaceSetMTU(vars.uplinkEnd, uplinkNetConfig["bridge.mtu"])
-			if err != nil {
-				return err
-			}
-
-			err = InterfaceSetMTU(vars.ovsEnd, uplinkNetConfig["bridge.mtu"])
-			if err != nil {
-				return err
-			}
-		}
+		revert.Add(func() { shared.RunCommand("ip", "link", "delete", vars.uplinkEnd) })
+	}
 
-		// Ensure correct sysctls are set on uplink veth interfaces to avoid getting IPv6 link-local addresses.
-		err := util.SysctlSet(
-			fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", vars.uplinkEnd), "1",
-			fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", vars.ovsEnd), "1",
-			fmt.Sprintf("net/ipv6/conf/%s/forwarding", vars.uplinkEnd), "0",
-			fmt.Sprintf("net/ipv6/conf/%s/forwarding", vars.ovsEnd), "0",
-		)
+	// Ensure that the veth interfaces inherit the uplink bridge's MTU (which the OVS bridge also inherits).
+	uplinkNetConfig := uplinkNet.Config()
+	if uplinkNetConfig["bridge.mtu"] != "" {
+		err := InterfaceSetMTU(vars.uplinkEnd, uplinkNetConfig["bridge.mtu"])
 		if err != nil {
-			return errors.Wrapf(err, "Failed to configure uplink veth interfaces %q and %q", vars.uplinkEnd, vars.ovsEnd)
+			return err
 		}
 
-		// Connect uplink end of veth pair to uplink bridge and bring up.
-		_, err = shared.RunCommand("ip", "link", "set", "master", uplinkNet.Name(), "dev", vars.uplinkEnd, "up")
+		err = InterfaceSetMTU(vars.ovsEnd, uplinkNetConfig["bridge.mtu"])
 		if err != nil {
-			return errors.Wrapf(err, "Failed to connect uplink veth interface %q to uplink bridge %q", vars.uplinkEnd, uplinkNet.Name())
+			return err
 		}
+	}
 
-		// Ensure uplink OVS end veth interface is up.
-		_, err = shared.RunCommand("ip", "link", "set", "dev", vars.ovsEnd, "up")
-		if err != nil {
-			return errors.Wrapf(err, "Failed to bring up uplink veth interface %q", vars.ovsEnd)
-		}
+	// Ensure correct sysctls are set on uplink veth interfaces to avoid getting IPv6 link-local addresses.
+	err := util.SysctlSet(
+		fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", vars.uplinkEnd), "1",
+		fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", vars.ovsEnd), "1",
+		fmt.Sprintf("net/ipv6/conf/%s/forwarding", vars.uplinkEnd), "0",
+		fmt.Sprintf("net/ipv6/conf/%s/forwarding", vars.ovsEnd), "0",
+	)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to configure uplink veth interfaces %q and %q", vars.uplinkEnd, vars.ovsEnd)
+	}
 
-		// Create uplink OVS bridge if needed.
-		err = ovs.BridgeAdd(vars.ovsBridge, true)
-		if err != nil {
-			return errors.Wrapf(err, "Failed to create uplink OVS bridge %q", vars.ovsBridge)
-		}
+	// Connect uplink end of veth pair to uplink bridge and bring up.
+	_, err = shared.RunCommand("ip", "link", "set", "master", bridgeDevice, "dev", vars.uplinkEnd, "up")
+	if err != nil {
+		return errors.Wrapf(err, "Failed to connect uplink veth interface %q to uplink bridge %q", vars.uplinkEnd, bridgeDevice)
+	}
 
-		// Connect OVS end veth interface to OVS bridge.
-		err = ovs.BridgePortAdd(vars.ovsBridge, vars.ovsEnd, true)
-		if err != nil {
-			return errors.Wrapf(err, "Failed to connect uplink veth interface %q to uplink OVS bridge %q", vars.ovsEnd, vars.ovsBridge)
-		}
+	// Ensure uplink OVS end veth interface is up.
+	_, err = shared.RunCommand("ip", "link", "set", "dev", vars.ovsEnd, "up")
+	if err != nil {
+		return errors.Wrapf(err, "Failed to bring up uplink veth interface %q", vars.ovsEnd)
+	}
 
-		// Associate OVS bridge to logical OVN provider.
-		err = ovs.OVNBridgeMappingAdd(vars.ovsBridge, uplinkNet.Name())
-		if err != nil {
-			return errors.Wrapf(err, "Failed to associate uplink OVS bridge %q to OVN provider %q", vars.ovsBridge, uplinkNet.Name())
-		}
-	} else {
-		// If uplink is an openvswitch bridge, have OVN logical provider connect directly to it.
-		err := ovs.OVNBridgeMappingAdd(uplinkNet.Name(), uplinkNet.Name())
-		if err != nil {
-			return errors.Wrapf(err, "Failed to associate uplink OVS bridge %q to OVN provider %q", uplinkNet.Name(), uplinkNet.Name())
-		}
+	// Create uplink OVS bridge if needed.
+	ovs := openvswitch.NewOVS()
+	err = ovs.BridgeAdd(vars.ovsBridge, true)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to create uplink OVS bridge %q", vars.ovsBridge)
+	}
+
+	// Connect OVS end veth interface to OVS bridge.
+	err = ovs.BridgePortAdd(vars.ovsBridge, vars.ovsEnd, true)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to connect uplink veth interface %q to uplink OVS bridge %q", vars.ovsEnd, vars.ovsBridge)
+	}
+
+	// Associate OVS bridge to logical OVN provider.
+	err = ovs.OVNBridgeMappingAdd(vars.ovsBridge, uplinkNet.Name())
+	if err != nil {
+		return errors.Wrapf(err, "Failed to associate uplink OVS bridge %q to OVN provider %q", vars.ovsBridge, uplinkNet.Name())
+	}
+
+	// Attempt to learn uplink MAC.
+	n.pingOVNRouterIPv6()
+
+	revert.Success()
+	return nil
+}
+
+// startUplinkPortBridgeOVS connects an OVN logical router to an uplink OVS bridge.
+func (n *ovn) startUplinkPortBridgeOVS(uplinkNet Network, bridgeDevice string) error {
+	// Do this after gaining lock so that on failure we revert before release locking.
+	revert := revert.New()
+	defer revert.Fail()
+
+	// If uplink is an openvswitch bridge, have OVN logical provider connect directly to it.
+	ovs := openvswitch.NewOVS()
+	err := ovs.OVNBridgeMappingAdd(bridgeDevice, uplinkNet.Name())
+	if err != nil {
+		return errors.Wrapf(err, "Failed to associate uplink OVS bridge %q to OVN provider %q", bridgeDevice, uplinkNet.Name())
 	}
 
+	// Attempt to learn uplink MAC.
+	n.pingOVNRouterIPv6()
+
+	revert.Success()
+	return nil
+}
+
+// pingOVNRouterIPv6 pings the OVN router's external IPv6 address to attempt to trigger MAC learning on uplink.
+// This is to work around a bug in some versions of OVN.
+func (n *ovn) pingOVNRouterIPv6() {
 	routerExtPortIPv6 := net.ParseIP(n.config[ovnVolatileUplinkIPv6])
 	if routerExtPortIPv6 != nil {
 		// Now that the OVN router is connected to the uplink bridge, attempt to ping the OVN
@@ -986,15 +1016,10 @@ func (n *ovn) startUplinkPortBridge(uplinkNet Network) error {
 			n.logger.Debug("OVN router external IPv6 address unreachable", log.Ctx{"ip": routerExtPortIPv6.String()})
 		}()
 	}
-
-	revert.Success()
-	return nil
 }
 
 // startUplinkPortPhysical creates OVS bridge (if doesn't exist) and connects uplink interface to the OVS bridge.
 func (n *ovn) startUplinkPortPhysical(uplinkNet Network) error {
-	vars := n.uplinkPortBridgeVars(uplinkNet)
-
 	// Do this after gaining lock so that on failure we revert before release locking.
 	revert := revert.New()
 	defer revert.Fail()
@@ -1003,9 +1028,24 @@ func (n *ovn) startUplinkPortPhysical(uplinkNet Network) error {
 	uplinkHostName := GetHostDevice(uplinkConfig["parent"], uplinkConfig["vlan"])
 
 	if !InterfaceExists(uplinkHostName) {
-		return fmt.Errorf("Uplink network %q is not started", uplinkNet.Name())
+		return fmt.Errorf("Uplink network %q is not started (interface %q is missing)", uplinkNet.Name(), uplinkHostName)
+	}
+
+	// Detect if uplink interface is a native bridge.
+	if IsNativeBridge(uplinkHostName) {
+		return n.startUplinkPortBridgeNative(uplinkNet, uplinkHostName)
+	}
+
+	// Detect if uplink interface is a OVS bridge.
+	ovs := openvswitch.NewOVS()
+	isOVSBridge, _ := ovs.BridgeExists(uplinkHostName)
+	if isOVSBridge {
+		return n.startUplinkPortBridgeOVS(uplinkNet, uplinkHostName)
 	}
 
+	// If uplink is a normal physical interface, then use a separate OVS bridge and connect uplink to it.
+	vars := n.uplinkPortBridgeVars(uplinkNet)
+
 	// Ensure correct sysctls are set on uplink interface to avoid getting IPv6 link-local addresses.
 	err := util.SysctlSet(
 		fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", uplinkHostName), "1",
@@ -1016,7 +1056,6 @@ func (n *ovn) startUplinkPortPhysical(uplinkNet Network) error {
 	}
 
 	// Create uplink OVS bridge if needed.
-	ovs := openvswitch.NewOVS()
 	err = ovs.BridgeAdd(vars.ovsBridge, true)
 	if err != nil {
 		return errors.Wrapf(err, "Failed to create uplink OVS bridge %q", vars.ovsBridge)
@@ -1100,66 +1139,58 @@ func (n *ovn) deleteUplinkPort() error {
 	return nil
 }
 
-// deleteUplinkPortBridge deletes uplink OVS bridge, OVN bridge mappings and veth interfaces if not in use.
+// deleteUplinkPortBridge disconnects the uplink port from the bridge and performs any cleanup.
 func (n *ovn) deleteUplinkPortBridge(uplinkNet Network) error {
-	// If uplink is a native bridge, then clean up separate OVS bridge and veth pair connection if not in use.
 	if uplinkNet.Config()["bridge.driver"] != "openvswitch" {
-		// Check OVS uplink bridge exists, if it does, check whether the uplink network is in use.
-		removeVeths := false
-		vars := n.uplinkPortBridgeVars(uplinkNet)
-		if InterfaceExists(vars.ovsBridge) {
-			uplinkUsed, err := n.checkUplinkUse()
-			if err != nil {
-				return err
-			}
-
-			// Remove OVS bridge if the uplink network isn't used by any other OVN networks.
-			if !uplinkUsed {
-				removeVeths = true
+		return n.deleteUplinkPortBridgeNative(uplinkNet)
+	}
 
-				ovs := openvswitch.NewOVS()
-				err = ovs.OVNBridgeMappingDelete(vars.ovsBridge, uplinkNet.Name())
-				if err != nil {
-					return err
-				}
+	return n.deleteUplinkPortBridgeOVS(uplinkNet)
+}
 
-				err = ovs.BridgeDelete(vars.ovsBridge)
-				if err != nil {
-					return err
-				}
-			}
-		} else {
-			removeVeths = true // Remove the veths if OVS bridge already gone.
+// deleteUplinkPortBridge deletes uplink OVS bridge, OVN bridge mappings and veth interfaces if not in use.
+func (n *ovn) deleteUplinkPortBridgeNative(uplinkNet Network) error {
+	// Check OVS uplink bridge exists, if it does, check whether the uplink network is in use.
+	removeVeths := false
+	vars := n.uplinkPortBridgeVars(uplinkNet)
+	if InterfaceExists(vars.ovsBridge) {
+		uplinkUsed, err := n.checkUplinkUse()
+		if err != nil {
+			return err
 		}
 
-		// Remove the veth interfaces if they exist.
-		if removeVeths {
-			if InterfaceExists(vars.uplinkEnd) {
-				_, err := shared.RunCommand("ip", "link", "delete", "dev", vars.uplinkEnd)
-				if err != nil {
-					return errors.Wrapf(err, "Failed to delete the uplink veth interface %q", vars.uplinkEnd)
-				}
+		// Remove OVS bridge if the uplink network isn't used by any other OVN networks.
+		if !uplinkUsed {
+			removeVeths = true
+
+			ovs := openvswitch.NewOVS()
+			err = ovs.OVNBridgeMappingDelete(vars.ovsBridge, uplinkNet.Name())
+			if err != nil {
+				return err
 			}
 
-			if InterfaceExists(vars.ovsEnd) {
-				_, err := shared.RunCommand("ip", "link", "delete", "dev", vars.ovsEnd)
-				if err != nil {
-					return errors.Wrapf(err, "Failed to delete the uplink veth interface %q", vars.ovsEnd)
-				}
+			err = ovs.BridgeDelete(vars.ovsBridge)
+			if err != nil {
+				return err
 			}
 		}
 	} else {
-		uplinkUsed, err := n.checkUplinkUse()
-		if err != nil {
-			return err
+		removeVeths = true // Remove the veths if OVS bridge already gone.
+	}
+
+	// Remove the veth interfaces if they exist.
+	if removeVeths {
+		if InterfaceExists(vars.uplinkEnd) {
+			_, err := shared.RunCommand("ip", "link", "delete", "dev", vars.uplinkEnd)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to delete the uplink veth interface %q", vars.uplinkEnd)
+			}
 		}
 
-		// Remove uplink OVS bridge mapping if not in use by other OVN networks.
-		if !uplinkUsed {
-			ovs := openvswitch.NewOVS()
-			err = ovs.OVNBridgeMappingDelete(uplinkNet.Name(), uplinkNet.Name())
+		if InterfaceExists(vars.ovsEnd) {
+			_, err := shared.RunCommand("ip", "link", "delete", "dev", vars.ovsEnd)
 			if err != nil {
-				return err
+				return errors.Wrapf(err, "Failed to delete the uplink veth interface %q", vars.ovsEnd)
 			}
 		}
 	}
@@ -1167,8 +1198,44 @@ func (n *ovn) deleteUplinkPortBridge(uplinkNet Network) error {
 	return nil
 }
 
+// deleteUplinkPortBridge deletes OVN bridge mappings if not in use.
+func (n *ovn) deleteUplinkPortBridgeOVS(uplinkNet Network) error {
+	uplinkUsed, err := n.checkUplinkUse()
+	if err != nil {
+		return err
+	}
+
+	// Remove uplink OVS bridge mapping if not in use by other OVN networks.
+	if !uplinkUsed {
+		ovs := openvswitch.NewOVS()
+		err = ovs.OVNBridgeMappingDelete(uplinkNet.Name(), uplinkNet.Name())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // deleteUplinkPortPhysical deletes uplink OVS bridge and OVN bridge mappings if not in use.
 func (n *ovn) deleteUplinkPortPhysical(uplinkNet Network) error {
+	uplinkConfig := uplinkNet.Config()
+	uplinkHostName := GetHostDevice(uplinkConfig["parent"], uplinkConfig["vlan"])
+
+	// Detect if uplink interface is a native bridge.
+	if IsNativeBridge(uplinkHostName) {
+		return n.deleteUplinkPortBridgeNative(uplinkNet)
+	}
+
+	// Detect if uplink interface is a OVS bridge.
+	ovs := openvswitch.NewOVS()
+	isOVSBridge, _ := ovs.BridgeExists(uplinkHostName)
+	if isOVSBridge {
+		return n.deleteUplinkPortBridgeOVS(uplinkNet)
+	}
+
+	// Otherwise if uplink is normal physical interface, attempt cleanup of OVS bridge.
+
 	// Check OVS uplink bridge exists, if it does, check whether the uplink network is in use.
 	releaseIF := false
 	vars := n.uplinkPortBridgeVars(uplinkNet)
@@ -1194,18 +1261,14 @@ func (n *ovn) deleteUplinkPortPhysical(uplinkNet Network) error {
 			}
 		}
 	} else {
-		releaseIF = true // Remove the veths if OVS bridge already gone.
+		releaseIF = true // Bring uplink interface down if not needed.
 	}
 
-	// Bring down uplink interface if exists.
-	if releaseIF {
-		uplinkConfig := uplinkNet.Config()
-		uplinkDev := GetHostDevice(uplinkConfig["parent"], uplinkConfig["vlan"])
-		if InterfaceExists(uplinkDev) {
-			_, err := shared.RunCommand("ip", "link", "set", uplinkDev, "down")
-			if err != nil {
-				return errors.Wrapf(err, "Failed to bring down uplink interface %q", uplinkDev)
-			}
+	// Bring down uplink interface if not used and exists.
+	if releaseIF && InterfaceExists(uplinkHostName) {
+		_, err := shared.RunCommand("ip", "link", "set", uplinkHostName, "down")
+		if err != nil {
+			return errors.Wrapf(err, "Failed to bring down uplink interface %q", uplinkHostName)
 		}
 	}
 


More information about the lxc-devel mailing list