[lxc-devel] [lxd/master] Network: Adds ability route external IPs to OVN NICs
tomponline on Github
lxc-bot at linuxcontainers.org
Wed Oct 14 14:50:18 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 1346 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201014/0b931379/attachment-0001.bin>
-------------- next part --------------
From 36502eb09df6082b16108e759cd405ce1a8cc170 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Oct 2020 10:13:46 +0100
Subject: [PATCH 01/30] lxd/api/project: Adds restricted.networks.subnets
config key
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/api_project.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/api_project.go b/lxd/api_project.go
index 770e9abf65..d70d1a2901 100644
--- a/lxd/api_project.go
+++ b/lxd/api_project.go
@@ -556,6 +556,7 @@ var projectConfigKeys = map[string]func(value string) error{
"restricted.devices.nic": isEitherAllowOrBlockOrManaged,
"restricted.devices.disk": isEitherAllowOrBlockOrManaged,
"restricted.networks.uplinks": validate.IsAny,
+ "restricted.networks.subnets": validate.Optional(validate.IsNetworkList),
}
func projectValidateConfig(config map[string]string) error {
From 5be6eede2f1c35babfabfa7ca03e930aef68ca53 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Oct 2020 10:14:24 +0100
Subject: [PATCH 02/30] lxd/network/driver/physical: Adds ipv4.routes and
ipv6.routes config keys
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_physical.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/network/driver_physical.go b/lxd/network/driver_physical.go
index bcbcc8cf1f..3d24584924 100644
--- a/lxd/network/driver_physical.go
+++ b/lxd/network/driver_physical.go
@@ -43,6 +43,8 @@ func (n *physical) Validate(config map[string]string) error {
"ipv6.gateway": validate.Optional(validate.IsNetworkAddressCIDRV6),
"ipv4.ovn.ranges": validate.Optional(validate.IsNetworkRangeV4List),
"ipv6.ovn.ranges": validate.Optional(validate.IsNetworkRangeV6List),
+ "ipv4.routes": validate.Optional(validate.IsNetworkV4List),
+ "ipv6.routes": validate.Optional(validate.IsNetworkV6List),
"dns.nameservers": validate.Optional(validate.IsNetworkAddressList),
"volatile.last_state.created": validate.Optional(validate.IsBool),
}
From 6a01cbf588a2205d5e867969ac42047f64356491 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Oct 2020 10:14:46 +0100
Subject: [PATCH 03/30] lxd/network/driver/ovn: Adds ipv4.routes and
ipv6.routes config keys
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index ca3373c9f4..972ea31828 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -99,6 +99,8 @@ func (n *ovn) Validate(config map[string]string) error {
return validate.Optional(validate.IsNetworkAddressCIDRV6)(value)
},
"ipv6.dhcp.stateful": validate.Optional(validate.IsBool),
+ "ipv4.routes": validate.Optional(validate.IsNetworkV4List),
+ "ipv6.routes": validate.Optional(validate.IsNetworkV6List),
"dns.domain": validate.IsAny,
"dns.search": validate.IsAny,
From cf9f092edb0218164eb37e737bd1d47b5d4a1456 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 14:25:00 +0100
Subject: [PATCH 04/30] lxd/network/network/utils: Adds SubnetContains function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/network_utils.go | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/lxd/network/network_utils.go b/lxd/network/network_utils.go
index 31f37282ee..b4d5811898 100644
--- a/lxd/network/network_utils.go
+++ b/lxd/network/network_utils.go
@@ -1072,3 +1072,29 @@ func InterfaceSetMTU(nic string, mtu string) error {
return nil
}
+
+// SubnetContains returns true if outerSubnet contains innerSubnet.
+func SubnetContains(outerSubnet *net.IPNet, innerSubnet *net.IPNet) bool {
+ if outerSubnet == nil || innerSubnet == nil {
+ return false
+ }
+
+ if !outerSubnet.Contains(innerSubnet.IP) {
+ return false
+ }
+
+ outerOnes, outerBits := outerSubnet.Mask.Size()
+ innerOnes, innerBits := innerSubnet.Mask.Size()
+
+ // Check number of bits in mask match.
+ if innerBits != outerBits {
+ return false
+ }
+
+ // Check that the inner subnet isn't outside of the outer subnet.
+ if innerOnes < outerOnes {
+ return false
+ }
+
+ return true
+}
From 1d68e67938d5cf43a7be1a36335dbeb6b309ffa3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 09:40:26 +0100
Subject: [PATCH 05/30] lxd/api/project: Moves projectConfigKeys inside
projectValidateConfig and adds state
This is so validators can have access to database for validation.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/api_project.go | 72 +++++++++++++++++++++++-----------------------
1 file changed, 36 insertions(+), 36 deletions(-)
diff --git a/lxd/api_project.go b/lxd/api_project.go
index d70d1a2901..cc46a61f31 100644
--- a/lxd/api_project.go
+++ b/lxd/api_project.go
@@ -15,6 +15,7 @@ import (
"github.com/lxc/lxd/lxd/operations"
projecthelpers "github.com/lxc/lxd/lxd/project"
"github.com/lxc/lxd/lxd/response"
+ "github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -527,56 +528,55 @@ func isEitherAllowOrBlockOrManaged(value string) error {
return validate.IsOneOf(value, []string{"block", "allow", "managed"})
}
-// Validate the project configuration
-var projectConfigKeys = map[string]func(value string) error{
- "features.profiles": validate.Optional(validate.IsBool),
- "features.images": validate.Optional(validate.IsBool),
- "features.storage.volumes": validate.Optional(validate.IsBool),
- "features.networks": validate.Optional(validate.IsBool),
- "limits.containers": validate.Optional(validate.IsUint32),
- "limits.virtual-machines": validate.Optional(validate.IsUint32),
- "limits.memory": validate.Optional(validate.IsSize),
- "limits.processes": validate.Optional(validate.IsUint32),
- "limits.cpu": validate.Optional(validate.IsUint32),
- "limits.disk": validate.Optional(validate.IsSize),
- "limits.networks": validate.Optional(validate.IsUint32),
- "restricted": validate.Optional(validate.IsBool),
- "restricted.containers.nesting": isEitherAllowOrBlock,
- "restricted.containers.lowlevel": isEitherAllowOrBlock,
- "restricted.containers.privilege": func(value string) error {
- return validate.IsOneOf(value, []string{"allow", "unprivileged", "isolated"})
- },
- "restricted.virtual-machines.lowlevel": isEitherAllowOrBlock,
- "restricted.devices.unix-char": isEitherAllowOrBlock,
- "restricted.devices.unix-block": isEitherAllowOrBlock,
- "restricted.devices.unix-hotplug": isEitherAllowOrBlock,
- "restricted.devices.infiniband": isEitherAllowOrBlock,
- "restricted.devices.gpu": isEitherAllowOrBlock,
- "restricted.devices.usb": isEitherAllowOrBlock,
- "restricted.devices.nic": isEitherAllowOrBlockOrManaged,
- "restricted.devices.disk": isEitherAllowOrBlockOrManaged,
- "restricted.networks.uplinks": validate.IsAny,
- "restricted.networks.subnets": validate.Optional(validate.IsNetworkList),
-}
+func projectValidateConfig(s *state.State, config map[string]string) error {
+ // Validate the project configuration.
+ projectConfigKeys := map[string]func(value string) error{
+ "features.profiles": validate.Optional(validate.IsBool),
+ "features.images": validate.Optional(validate.IsBool),
+ "features.storage.volumes": validate.Optional(validate.IsBool),
+ "features.networks": validate.Optional(validate.IsBool),
+ "limits.containers": validate.Optional(validate.IsUint32),
+ "limits.virtual-machines": validate.Optional(validate.IsUint32),
+ "limits.memory": validate.Optional(validate.IsSize),
+ "limits.processes": validate.Optional(validate.IsUint32),
+ "limits.cpu": validate.Optional(validate.IsUint32),
+ "limits.disk": validate.Optional(validate.IsSize),
+ "limits.networks": validate.Optional(validate.IsUint32),
+ "restricted": validate.Optional(validate.IsBool),
+ "restricted.containers.nesting": isEitherAllowOrBlock,
+ "restricted.containers.lowlevel": isEitherAllowOrBlock,
+ "restricted.containers.privilege": func(value string) error {
+ return validate.IsOneOf(value, []string{"allow", "unprivileged", "isolated"})
+ },
+ "restricted.virtual-machines.lowlevel": isEitherAllowOrBlock,
+ "restricted.devices.unix-char": isEitherAllowOrBlock,
+ "restricted.devices.unix-block": isEitherAllowOrBlock,
+ "restricted.devices.unix-hotplug": isEitherAllowOrBlock,
+ "restricted.devices.infiniband": isEitherAllowOrBlock,
+ "restricted.devices.gpu": isEitherAllowOrBlock,
+ "restricted.devices.usb": isEitherAllowOrBlock,
+ "restricted.devices.nic": isEitherAllowOrBlockOrManaged,
+ "restricted.devices.disk": isEitherAllowOrBlockOrManaged,
+ "restricted.networks.uplinks": validate.IsAny,
+ }
-func projectValidateConfig(config map[string]string) error {
for k, v := range config {
key := k
- // User keys are free for all
+ // User keys are free for all.
if strings.HasPrefix(key, "user.") {
continue
}
- // Then validate
+ // Then validate.
validator, ok := projectConfigKeys[key]
if !ok {
- return fmt.Errorf("Invalid project configuration key: %s", k)
+ return fmt.Errorf("Invalid project configuration key %q", k)
}
err := validator(v)
if err != nil {
- return err
+ return errors.Wrapf(err, "Invalid project configuration key %q value", k)
}
}
From a39b04aefd94be34b202a5e5b9ea20318103d7f1 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 09:41:05 +0100
Subject: [PATCH 06/30] lxd/api/project: projectValidateConfig usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/api_project.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/api_project.go b/lxd/api_project.go
index cc46a61f31..5843d31173 100644
--- a/lxd/api_project.go
+++ b/lxd/api_project.go
@@ -126,7 +126,7 @@ func projectsPost(d *Daemon, r *http.Request) response.Response {
}
// Validate the configuration
- err = projectValidateConfig(project.Config)
+ err = projectValidateConfig(d.State(), project.Config)
if err != nil {
return response.BadRequest(err)
}
@@ -354,7 +354,7 @@ func projectChange(d *Daemon, project *api.Project, req api.ProjectPut) response
}
// Validate the configuration.
- err := projectValidateConfig(req.Config)
+ err := projectValidateConfig(d.State(), req.Config)
if err != nil {
return response.BadRequest(err)
}
From 25d5a972c8b54b0abc3ca524149de8f1f1069293 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 14:33:27 +0100
Subject: [PATCH 07/30] lxd/api/project: Adds projectValidateRestrictedSubnets
function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/api_project.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/lxd/api_project.go b/lxd/api_project.go
index 5843d31173..4ff05ad8a3 100644
--- a/lxd/api_project.go
+++ b/lxd/api_project.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
+ "net"
"net/http"
"strings"
@@ -12,7 +13,9 @@ import (
"github.com/pkg/errors"
"github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/lxd/network"
"github.com/lxc/lxd/lxd/operations"
+ "github.com/lxc/lxd/lxd/project"
projecthelpers "github.com/lxc/lxd/lxd/project"
"github.com/lxc/lxd/lxd/response"
"github.com/lxc/lxd/lxd/state"
@@ -606,3 +609,65 @@ func projectValidateName(name string) error {
return nil
}
+
+// projectValidateRestrictedSubnets checks that the project's restricted.networks.subnets are properly formatted
+// and are within the specified uplink network's routes.
+func projectValidateRestrictedSubnets(s *state.State, value string) error {
+ for _, subnetRaw := range strings.Split(value, ",") {
+ subnetParts := strings.SplitN(strings.TrimSpace(subnetRaw), ":", 2)
+ if len(subnetParts) != 2 {
+ return fmt.Errorf(`Subnet %q invalid, must be in the format of "<uplink network>:<subnet>"`, subnetRaw)
+ }
+
+ uplinkName := subnetParts[0]
+ subnetStr := subnetParts[1]
+
+ restrictedSubnetIP, restrictedSubnet, err := net.ParseCIDR(subnetStr)
+ if err != nil {
+ return err
+ }
+
+ if restrictedSubnetIP.String() != restrictedSubnet.IP.String() {
+ return fmt.Errorf("Not an IP network address %q", value)
+ }
+
+ // Check uplink exists and load config to compare subnets.
+ _, uplink, err := s.Cluster.GetNetworkInAnyState(project.Default, uplinkName)
+ if err != nil {
+ return errors.Wrapf(err, "Invalid uplink network %q", uplinkName)
+ }
+
+ // Parse uplink route subnets.
+ var uplinkRoutes []*net.IPNet
+ for _, k := range []string{"ipv4.routes", "ipv6.routes"} {
+ if uplink.Config[k] == "" {
+ continue
+ }
+
+ routes := strings.Split(uplink.Config[k], ",")
+ for _, route := range routes {
+ _, uplinkRoute, err := net.ParseCIDR(strings.TrimSpace(route))
+ if err != nil {
+ return err
+ }
+
+ uplinkRoutes = append(uplinkRoutes, uplinkRoute)
+ }
+ }
+
+ foundMatch := false
+ // Check that the restricted subnet is within one of the uplink's routes.
+ for _, uplinkRoute := range uplinkRoutes {
+ if network.SubnetContains(uplinkRoute, restrictedSubnet) {
+ foundMatch = true
+ break
+ }
+ }
+
+ if !foundMatch {
+ return fmt.Errorf("Uplink network %q doesn't contain %q in its routes", uplinkName, restrictedSubnet.String())
+ }
+ }
+
+ return nil
+}
From 1813e23ea4fb569f47559f3d18273be23eb307ce Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 10:22:41 +0100
Subject: [PATCH 08/30] lxd/api/project: Adds restricted.networks.subnets
validation to projectValidateConfig
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/api_project.go | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lxd/api_project.go b/lxd/api_project.go
index 4ff05ad8a3..6711064cab 100644
--- a/lxd/api_project.go
+++ b/lxd/api_project.go
@@ -561,6 +561,9 @@ func projectValidateConfig(s *state.State, config map[string]string) error {
"restricted.devices.nic": isEitherAllowOrBlockOrManaged,
"restricted.devices.disk": isEitherAllowOrBlockOrManaged,
"restricted.networks.uplinks": validate.IsAny,
+ "restricted.networks.subnets": validate.Optional(func(value string) error {
+ return projectValidateRestrictedSubnets(s, value)
+ }),
}
for k, v := range config {
From 4b16765bef6fa20a65005b071a50810e13546053 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 14:45:54 +0100
Subject: [PATCH 09/30] doc/projects: Removes trailing full stop
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/projects.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/projects.md b/doc/projects.md
index b7fa5d8961..5b49c8ccab 100644
--- a/doc/projects.md
+++ b/doc/projects.md
@@ -40,7 +40,7 @@ restricted.devices.unix-block | string | - | block
restricted.devices.unix-char | string | - | block | Prevents use of devices of type "unix-char"
restricted.devices.unix-hotplug | string | - | block | Prevents use of devices of type "unix-hotplug"
restricted.devices.usb | string | - | block | Prevents use of devices of type "usb"
-restricted.networks.uplinks | string | - | block | Comma delimited list of network names that can be used as uplinks for networks in this project.
+restricted.networks.uplinks | string | - | block | Comma delimited list of network names that can be used as uplinks for networks in this project
restricted.virtual-machines.lowlevel | string | - | block | Prevents use of low-level virtual-machine options like raw.qemu, volatile, etc.
Those keys can be set using the lxc tool with:
From f267cae779112f52070e64ab8857f7612ad98dc0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 14:46:07 +0100
Subject: [PATCH 10/30] doc/projects: Adds restricted.networks.subnets
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/projects.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/projects.md b/doc/projects.md
index 5b49c8ccab..462864fabb 100644
--- a/doc/projects.md
+++ b/doc/projects.md
@@ -41,6 +41,7 @@ restricted.devices.unix-char | string | - | block
restricted.devices.unix-hotplug | string | - | block | Prevents use of devices of type "unix-hotplug"
restricted.devices.usb | string | - | block | Prevents use of devices of type "usb"
restricted.networks.uplinks | string | - | block | Comma delimited list of network names that can be used as uplinks for networks in this project
+restricted.networks.subnets | string | - | block | Comma delimited list of network subnets from the uplink networks (in the form `<uplink>:<subnet>`) that are allocated for use in this project
restricted.virtual-machines.lowlevel | string | - | block | Prevents use of low-level virtual-machine options like raw.qemu, volatile, etc.
Those keys can be set using the lxc tool with:
From 758867661f4fe667e3054e29da7248dd27201ccf Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 14:49:33 +0100
Subject: [PATCH 11/30] api: Adds network_ovn_external_subnets extension
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/api-extensions.md | 9 +++++++++
shared/version/api.go | 1 +
2 files changed, 10 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index a2688afac2..06d84d4570 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1191,3 +1191,12 @@ to disable compression in rsync while migrating storage pools.
Adds support for additional network type `physical` that can be used as an uplink for `ovn` networks.
The interface specified by `parent` on the `physical` network will be connected to the `ovn` network's gateway.
+
+## network\_ovn\_external\_subnets
+Adds support for `ovn` networks to use external subnets from uplink networks.
+
+Introduces the `ipv4.routes` and `ipv6.routes` setting on `physical` networks that defines the external routes
+allowed to be used in child OVN networks in their `ipv4.routes.external` and `ipv6.routes.external` settings.
+
+Introduces the `restricted.networks.subnets` project setting that specifies which external subnets are allowed to
+be used by OVN networks inside the project (if not set then all routes defined on the uplink network are allowed).
diff --git a/shared/version/api.go b/shared/version/api.go
index d5035f7161..9d9da206d9 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -230,6 +230,7 @@ var APIExtensions = []string{
"backup_override_name",
"storage_rsync_compression",
"network_type_physical",
+ "network_ovn_external_subnets",
}
// APIExtensionsCount returns the number of available API extensions.
From 70050bcb9d52a1a2cfb646b21b213602718efdf8 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 14:58:35 +0100
Subject: [PATCH 12/30] doc/networks: Adds ipv4.routes and ipv6.routes settings
to physical network
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/networks.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/doc/networks.md b/doc/networks.md
index cc63de9c11..9573e19130 100644
--- a/doc/networks.md
+++ b/doc/networks.md
@@ -316,6 +316,8 @@ parent | string | - | -
vlan | integer | - | - | The VLAN ID to attach to
ipv4.gateway | string | standard mode | - | IPv4 address for the gateway and network (CIDR notation)
ipv4.ovn.ranges | string | - | none | Comma separate list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format)
+ipv4.routes | string | ipv4 address | - | Comma separated list of additional IPv4 CIDR subnets that can be used with child OVN networks ipv4.routes.external setting
ipv6.gateway | string | standard mode | - | IPv6 address for the gateway and network (CIDR notation)
ipv6.ovn.ranges | string | - | none | Comma separate list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format)
+ipv6.routes | string | ipv6 address | - | Comma separated list of additional IPv6 CIDR subnets that can be used with child OVN networks ipv6.routes.external setting
dns.nameservers | string | standard mode | - | List of DNS server IPs on physical network
From 9de757896e2a47fa6251b3f20dcb26d7f540c93c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 15:01:22 +0100
Subject: [PATCH 13/30] doc/networks: Adds ipv4.routes.external and
ipv6.routes.external to ovn networks
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/networks.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/doc/networks.md b/doc/networks.md
index 9573e19130..3d972ddf72 100644
--- a/doc/networks.md
+++ b/doc/networks.md
@@ -297,8 +297,10 @@ bridge.mtu | integer | - | 1442
dns.domain | string | - | lxd | Domain to advertise to DHCP clients and use for DNS resolution
dns.search | string | - | - | Full comma separated domain search list, defaulting to `dns.domain` value
ipv4.address | string | standard mode | random unused subnet | IPv4 address for the bridge (CIDR notation). Use "none" to turn off IPv4 or "auto" to generate a new one
+ipv4.routes.external | string | ipv4 address | - | Comma separated list of additional external IPv4 CIDR subnets that are allowed for OVN NICs ipv4.routes.external setting
ipv6.address | string | standard mode | random unused subnet | IPv6 address for the bridge (CIDR notation). Use "none" to turn off IPv6 or "auto" to generate a new one
ipv6.dhcp.stateful | boolean | ipv6 dhcp | false | Whether to allocate addresses using DHCP
+ipv6.routes.external | string | ipv6 address | - | Comma separated list of additional external IPv6 CIDR subnets that are allowed for OVN NICs ipv6.routes.external setting
network | string | - | - | Uplink network to use for external network access
## network: physical
From f656a1c51a561049b86e3132e1e8c37a4119aad5 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 17:30:14 +0100
Subject: [PATCH 14/30] lxd/network/driver/ovn: Updates Validate to check
network exists and checks external IP routes
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 124 ++++++++++++++++++++++++++++++++++++--
1 file changed, 118 insertions(+), 6 deletions(-)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index 972ea31828..98f09594a8 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -80,8 +80,13 @@ func (n *ovn) Info() Info {
// Validate network config.
func (n *ovn) Validate(config map[string]string) error {
+ // Cache the uplink network for validating "network", "ipv4.routes.external" and "ipv6.routes.external".
+ _, uplink, uplinkErr := n.state.Cluster.GetNetworkInAnyState(project.Default, config["network"])
+
rules := map[string]func(value string) error{
- "network": validate.IsAny, // Is validated during setup.
+ "network": func(value string) error {
+ return uplinkErr // Check the pre-lookup of uplink network succeeded.
+ },
"bridge.hwaddr": validate.Optional(validate.IsNetworkMAC),
"bridge.mtu": validate.Optional(validate.IsNetworkMTU),
"ipv4.address": func(value string) error {
@@ -98,11 +103,11 @@ func (n *ovn) Validate(config map[string]string) error {
return validate.Optional(validate.IsNetworkAddressCIDRV6)(value)
},
- "ipv6.dhcp.stateful": validate.Optional(validate.IsBool),
- "ipv4.routes": validate.Optional(validate.IsNetworkV4List),
- "ipv6.routes": validate.Optional(validate.IsNetworkV6List),
- "dns.domain": validate.IsAny,
- "dns.search": validate.IsAny,
+ "ipv6.dhcp.stateful": validate.Optional(validate.IsBool),
+ "ipv4.routes.external": validate.Optional(validate.IsNetworkV4List),
+ "ipv6.routes.external": validate.Optional(validate.IsNetworkV6List),
+ "dns.domain": validate.IsAny,
+ "dns.search": validate.IsAny,
// Volatile keys populated automatically as needed.
ovnVolatileUplinkIPv4: validate.Optional(validate.IsNetworkAddressV4),
@@ -114,6 +119,113 @@ func (n *ovn) Validate(config map[string]string) error {
return err
}
+ // Composite checks.
+
+ // Check IP routes are within the uplink network's routes and project's subnet restrictions.
+ if config["ipv4.routes.external"] != "" || config["ipv6.routes.external"] != "" {
+ // Load the project to get uplink network restrictions.
+ var project *api.Project
+ err = n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
+ project, err = tx.GetProject(n.project)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ })
+ if err != nil {
+ return errors.Wrapf(err, "Failed to load IP route restrictions for project %q", n.project)
+ }
+
+ // Parse uplink route subnets.
+ var uplinkRoutes []*net.IPNet
+ for _, k := range []string{"ipv4.routes", "ipv6.routes"} {
+ if uplink.Config[k] == "" {
+ continue
+ }
+
+ routes := strings.Split(uplink.Config[k], ",")
+ for _, route := range routes {
+ _, uplinkRoute, err := net.ParseCIDR(strings.TrimSpace(route))
+ if err != nil {
+ return err
+ }
+
+ uplinkRoutes = append(uplinkRoutes, uplinkRoute)
+ }
+ }
+
+ // Parse project's restricted subnets.
+ var projectRestrictedSubnets []*net.IPNet // Nil value indicates not restricted.
+ if shared.IsTrue(project.Config["restricted"]) && project.Config["restricted.networks.subnets"] != "" {
+ projectRestrictedSubnets = []*net.IPNet{} // Empty slice indicates no allowed subnets.
+
+ for _, subnetRaw := range strings.Split(project.Config["restricted.networks.subnets"], ",") {
+ subnetParts := strings.SplitN(strings.TrimSpace(subnetRaw), ":", 2)
+ if len(subnetParts) != 2 {
+ return fmt.Errorf(`Project subnet %q invalid, must be in the format of "<uplink network>:<subnet>"`, subnetRaw)
+ }
+
+ uplinkName := subnetParts[0]
+ subnetStr := subnetParts[1]
+
+ if uplinkName != uplink.Name {
+ continue // Only include subnets for our uplink.
+ }
+
+ _, restrictedSubnet, err := net.ParseCIDR(subnetStr)
+ if err != nil {
+ return err
+ }
+
+ projectRestrictedSubnets = append(projectRestrictedSubnets, restrictedSubnet)
+ }
+ }
+
+ // Parse and validate our routes.
+ for _, k := range []string{"ipv4.routes.external", "ipv6.routes.external"} {
+ if config[k] == "" {
+ continue
+ }
+
+ for _, route := range strings.Split(config[k], ",") {
+ route = strings.TrimSpace(route)
+ _, routeSubnet, err := net.ParseCIDR(route)
+ if err != nil {
+ return err
+ }
+
+ // Check that the route is within the project's restricted subnets if restricted.
+ if projectRestrictedSubnets != nil {
+ foundMatch := false
+ for _, projectRestrictedSubnet := range projectRestrictedSubnets {
+ if SubnetContains(projectRestrictedSubnet, routeSubnet) {
+ foundMatch = true
+ break
+ }
+ }
+
+ if !foundMatch {
+ return fmt.Errorf("Project %q doesn't contain %q in its restricted uplink subnets", project.Name, routeSubnet.String())
+ }
+ }
+
+ // Check that the route is within the uplink network's routes.
+ foundMatch := false
+ for _, uplinkRoute := range uplinkRoutes {
+ if SubnetContains(uplinkRoute, routeSubnet) {
+ foundMatch = true
+ break
+ }
+ }
+
+ if !foundMatch {
+ return fmt.Errorf("Uplink network %q doesn't contain %q in its routes", uplink.Name, routeSubnet.String())
+ }
+ }
+ }
+ }
+
return nil
}
From 23c2e7bd15df4682241520a0150a930efc4ef0f9 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 13 Oct 2020 17:55:00 +0100
Subject: [PATCH 15/30] doc/instances: Adds ovn NIC documentation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/instances.md | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/doc/instances.md b/doc/instances.md
index 042195c732..8ceb0cb911 100644
--- a/doc/instances.md
+++ b/doc/instances.md
@@ -258,6 +258,7 @@ LXD supports different kind of network devices:
- [bridged](#nictype-bridged): Uses an existing bridge on the host and creates a virtual device pair to connect the host bridge to the instance.
- [macvlan](#nictype-macvlan): Sets up a new network device based on an existing one but using a different MAC address.
- [ipvlan](#nictype-ipvlan): Sets up a new network device based on an existing one using the same MAC address but a different IP.
+ - [ovn](#nictype-ovn): Uses an existing OVN network and creates a virtual device pair to connect the instance to it.
- [p2p](#nictype-p2p): Creates a virtual device pair, putting one side in the instance and leaving the other side on the host.
- [sriov](#nictype-sriov): Passes a virtual function of an SR-IOV enabled physical network device into the instance.
- [routed](#nictype-routed): Creates a virtual device pair to connect the host to the instance and sets up static routes and proxy ARP/NDP entries to allow the instance to join the network of a designated parent interface.
@@ -402,6 +403,26 @@ ipv4.routes | string | - | no | Comma deli
ipv6.routes | string | - | no | Comma delimited list of IPv6 static routes to add on host to nic
boot.priority | integer | - | no | Boot priority for VMs (higher boots first)
+#### nictype: ovn
+
+Supported instance types: container, VM
+
+Uses an existing OVN network and creates a virtual device pair to connect the instance to it.
+
+Device configuration properties:
+
+Key | Type | Default | Required | Description
+:-- | :-- | :-- | :-- | :--
+network | string | - | yes | The LXD network to link device to
+name | string | kernel assigned | no | The name of the interface inside the instance
+host\_name | string | randomly assigned | no | The name of the interface inside the host
+hwaddr | string | randomly assigned | no | The MAC address of the new interface
+ipv4.address | string | - | no | An IPv4 address to assign to the instance through DHCP
+ipv6.address | string | - | no | An IPv6 address to assign to the instance through DHCP
+ipv4.routes.external | string | - | no | Comma delimited list of IPv4 static routes to route to the NIC and publish on uplink network
+ipv6.routes.external | string | - | no | Comma delimited list of IPv6 static routes to route to the NIC and publish on uplink network
+boot.priority | integer | - | no | Boot priority for VMs (higher boots first)
+
#### nictype: sriov
Supported instance types: container, VM
From 41888aa1a76bf3868b3e27aed5dbfd6f92898046 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 10:47:04 +0100
Subject: [PATCH 16/30] network ovn external routes validation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index 98f09594a8..10ae4050e9 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -121,7 +121,7 @@ func (n *ovn) Validate(config map[string]string) error {
// Composite checks.
- // Check IP routes are within the uplink network's routes and project's subnet restrictions.
+ // Check IP external routes are within the uplink network's routes and project's subnet restrictions.
if config["ipv4.routes.external"] != "" || config["ipv6.routes.external"] != "" {
// Load the project to get uplink network restrictions.
var project *api.Project
@@ -182,7 +182,7 @@ func (n *ovn) Validate(config map[string]string) error {
}
}
- // Parse and validate our routes.
+ // Parse and validate our external routes.
for _, k := range []string{"ipv4.routes.external", "ipv6.routes.external"} {
if config[k] == "" {
continue
@@ -210,7 +210,7 @@ func (n *ovn) Validate(config map[string]string) error {
}
}
- // Check that the route is within the uplink network's routes.
+ // Check that the external route is within the uplink network's routes.
foundMatch := false
for _, uplinkRoute := range uplinkRoutes {
if SubnetContains(uplinkRoute, routeSubnet) {
From 04014f5463e3cb09dfb454eb21fe392d69cb05e8 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 10:57:48 +0100
Subject: [PATCH 17/30] lxd/network/driver/ovn: Adds DNS revert to
instanceDevicePortAdd
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index 10ae4050e9..5bd86a11af 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -1877,6 +1877,8 @@ func (n *ovn) instanceDevicePortAdd(instanceID int, instanceName string, deviceN
return "", err
}
+ revert.Add(func() { client.LogicalSwitchPortDeleteDNS(n.getIntSwitchName(), instancePortName) })
+
revert.Success()
return instancePortName, nil
}
From 2a52cec2cb83c70adbcf24092a7769ecebf8f178 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 11:23:58 +0100
Subject: [PATCH 18/30] lxd/network/openvswitch/ovn: Adds
LogicalRouterRouteDelete function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/openvswitch/ovn.go | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/lxd/network/openvswitch/ovn.go b/lxd/network/openvswitch/ovn.go
index e0a0d0dc44..ffc2af8d72 100644
--- a/lxd/network/openvswitch/ovn.go
+++ b/lxd/network/openvswitch/ovn.go
@@ -161,6 +161,23 @@ func (o *OVN) LogicalRouterRouteAdd(routerName OVNRouter, destination *net.IPNet
return nil
}
+// LogicalRouterRouteDelete deletes a static route from the logical router.
+// If nextHop is specified as nil, then any route matching the destination is removed.
+func (o *OVN) LogicalRouterRouteDelete(routerName OVNRouter, destination *net.IPNet, nextHop net.IP) error {
+ args := []string{"--if-exists", "lr-route-del", string(routerName), destination.String()}
+
+ if nextHop != nil {
+ args = append(args, nextHop.String())
+ }
+
+ _, err := o.nbctl(args...)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
// LogicalRouterPortAdd adds a named logical router port to a logical router.
func (o *OVN) LogicalRouterPortAdd(routerName OVNRouter, portName OVNRouterPort, mac net.HardwareAddr, ipAddr ...*net.IPNet) error {
args := []string{"lrp-add", string(routerName), string(portName), mac.String()}
From 68777ceab75b59370fdbdda3cda2c525111abe92 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 11:24:14 +0100
Subject: [PATCH 19/30] lxd/network/openvswitch/ovn: Updates
LogicalSwitchPortSetDNS to return IPs used for DNS records
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/openvswitch/ovn.go | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/lxd/network/openvswitch/ovn.go b/lxd/network/openvswitch/ovn.go
index ffc2af8d72..23ff565950 100644
--- a/lxd/network/openvswitch/ovn.go
+++ b/lxd/network/openvswitch/ovn.go
@@ -644,7 +644,8 @@ func (o *OVN) LogicalSwitchPortDynamicIPs(portName OVNSwitchPort) ([]net.IP, err
// LogicalSwitchPortSetDNS sets up the switch DNS records for the DNS name resolving to the IPs of the switch port.
// Attempts to find at most one IP for each IP protocol, preferring static addresses over dynamic.
-func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPort, dnsName string) error {
+// Returns the IPv4 and IPv6 addresses used for DNS records.
+func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPort, dnsName string) (net.IP, net.IP, error) {
var dnsIPv4, dnsIPv6 net.IP
// checkAndStoreIP checks if the supplied IP is valid and can be used for a missing DNS IP variable.
@@ -667,7 +668,7 @@ func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPo
// Get static and dynamic IPs for switch port.
staticAddressesRaw, err := o.nbctl("lsp-get-addresses", string(portName))
if err != nil {
- return err
+ return nil, nil, err
}
staticAddresses := strings.Split(strings.TrimSpace(staticAddressesRaw), " ")
@@ -691,7 +692,7 @@ func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPo
if hasDynamic && (dnsIPv4 == nil || dnsIPv6 == nil) {
dynamicIPs, err := o.LogicalSwitchPortDynamicIPs(portName)
if err != nil {
- return err
+ return nil, nil, err
}
for _, dynamicIP := range dynamicIPs {
@@ -719,7 +720,7 @@ func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPo
fmt.Sprintf("external_ids:lxd_switch_port=%s", string(portName)),
)
if err != nil {
- return err
+ return nil, nil, err
}
cmdArgs := []string{
@@ -733,13 +734,13 @@ func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPo
// Update existing record if exists.
_, err = o.nbctl(append([]string{"set", "dns", dnsUUID}, cmdArgs...)...)
if err != nil {
- return err
+ return nil, nil, err
}
} else {
// Create new record if needed.
dnsUUID, err = o.nbctl(append([]string{"create", "dns"}, cmdArgs...)...)
if err != nil {
- return err
+ return nil, nil, err
}
dnsUUID = strings.TrimSpace(dnsUUID)
}
@@ -747,10 +748,10 @@ func (o *OVN) LogicalSwitchPortSetDNS(switchName OVNSwitch, portName OVNSwitchPo
// Add DNS record to switch DNS records.
_, err = o.nbctl("add", "logical_switch", string(switchName), "dns_records", dnsUUID)
if err != nil {
- return err
+ return nil, nil, err
}
- return nil
+ return dnsIPv4, dnsIPv6, nil
}
// LogicalSwitchPortDeleteDNS removes DNS records for a switch port.
From 5c12ba529d10af42853d9794a6eb817ed899b37d Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Oct 2020 09:25:31 +0100
Subject: [PATCH 20/30] lxd/network/openvswitch/ovn: Adds
LogicalRouterDNATSNATAdd function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/openvswitch/ovn.go | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/lxd/network/openvswitch/ovn.go b/lxd/network/openvswitch/ovn.go
index 23ff565950..be71c1e854 100644
--- a/lxd/network/openvswitch/ovn.go
+++ b/lxd/network/openvswitch/ovn.go
@@ -151,6 +151,26 @@ func (o *OVN) LogicalRouterSNATAdd(routerName OVNRouter, intNet *net.IPNet, extI
return nil
}
+// LogicalRouterDNATSNATAdd adds a DNAT and SNAT rule to a logical router to translate packets from extIP to intIP.
+func (o *OVN) LogicalRouterDNATSNATAdd(routerName OVNRouter, extIP net.IP, intIP net.IP, stateless bool, mayExist bool) error {
+ args := []string{}
+
+ if mayExist {
+ args = append(args, "--may-exist")
+ }
+
+ if stateless {
+ args = append(args, "--stateless")
+ }
+
+ _, err := o.nbctl(append(args, "lr-nat-add", string(routerName), "dnat_and_snat", extIP.String(), intIP.String())...)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
// LogicalRouterRouteAdd adds a static route to the logical router.
func (o *OVN) LogicalRouterRouteAdd(routerName OVNRouter, destination *net.IPNet, nextHop net.IP) error {
_, err := o.nbctl("lr-route-add", string(routerName), destination.String(), nextHop.String())
From d7218955942b3d5f5a8b4966a88dac3f9e5a9436 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 13:16:23 +0100
Subject: [PATCH 21/30] lxd/network/openvswitch/ovn: Adds
LogicalRouterDNATSNATDelete function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/openvswitch/ovn.go | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/lxd/network/openvswitch/ovn.go b/lxd/network/openvswitch/ovn.go
index be71c1e854..5a474973f1 100644
--- a/lxd/network/openvswitch/ovn.go
+++ b/lxd/network/openvswitch/ovn.go
@@ -171,6 +171,16 @@ func (o *OVN) LogicalRouterDNATSNATAdd(routerName OVNRouter, extIP net.IP, intIP
return nil
}
+// LogicalRouterDNATSNATDelete deletes a DNAT and SNAT rule from a logical router.
+func (o *OVN) LogicalRouterDNATSNATDelete(routerName OVNRouter, extIP net.IP) error {
+ _, err := o.nbctl("--if-exists", "lr-nat-del", string(routerName), "dnat_and_snat", extIP.String())
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
// LogicalRouterRouteAdd adds a static route to the logical router.
func (o *OVN) LogicalRouterRouteAdd(routerName OVNRouter, destination *net.IPNet, nextHop net.IP) error {
_, err := o.nbctl("lr-route-add", string(routerName), destination.String(), nextHop.String())
From 20c6b125e84cb74c6f3f26653f0dda196dedcea9 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 13:16:38 +0100
Subject: [PATCH 22/30] lxd/network/openvswitch/ovn: Updates
LogicalRouterRouteAdd to support mayExist argument
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/openvswitch/ovn.go | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lxd/network/openvswitch/ovn.go b/lxd/network/openvswitch/ovn.go
index 5a474973f1..bb08c505af 100644
--- a/lxd/network/openvswitch/ovn.go
+++ b/lxd/network/openvswitch/ovn.go
@@ -182,8 +182,15 @@ func (o *OVN) LogicalRouterDNATSNATDelete(routerName OVNRouter, extIP net.IP) er
}
// LogicalRouterRouteAdd adds a static route to the logical router.
-func (o *OVN) LogicalRouterRouteAdd(routerName OVNRouter, destination *net.IPNet, nextHop net.IP) error {
- _, err := o.nbctl("lr-route-add", string(routerName), destination.String(), nextHop.String())
+func (o *OVN) LogicalRouterRouteAdd(routerName OVNRouter, destination *net.IPNet, nextHop net.IP, mayExist bool) error {
+ args := []string{}
+
+ if mayExist {
+ args = append(args, "--may-exist")
+ }
+
+ args = append(args, "lr-route-add", string(routerName), destination.String(), nextHop.String())
+ _, err := o.nbctl(args...)
if err != nil {
return err
}
From 94f4c06c0db2d372ba114758241d811d74edb7e3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 13:17:13 +0100
Subject: [PATCH 23/30] lxd/network/driver/ovn: client.LogicalRouterRouteAdd
usage
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index 5bd86a11af..d077c7ffa4 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -1343,14 +1343,14 @@ func (n *ovn) setup(update bool) error {
// Add default routes.
if uplinkNet.routerExtGwIPv4 != nil {
- err = client.LogicalRouterRouteAdd(n.getRouterName(), &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, uplinkNet.routerExtGwIPv4)
+ err = client.LogicalRouterRouteAdd(n.getRouterName(), &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, uplinkNet.routerExtGwIPv4, false)
if err != nil {
return errors.Wrapf(err, "Failed adding IPv4 default route")
}
}
if uplinkNet.routerExtGwIPv6 != nil {
- err = client.LogicalRouterRouteAdd(n.getRouterName(), &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}, uplinkNet.routerExtGwIPv6)
+ err = client.LogicalRouterRouteAdd(n.getRouterName(), &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}, uplinkNet.routerExtGwIPv6, false)
if err != nil {
return errors.Wrapf(err, "Failed adding IPv6 default route")
}
From 3843a667eb10b388b72beaef9ecb0773eaaab1de Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 11:24:38 +0100
Subject: [PATCH 24/30] lxd/network/network/utils/ovn: Updates
OVNInstanceDevicePortAdd to take an externalRoutes argument
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/network_utils_ovn.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/network/network_utils_ovn.go b/lxd/network/network_utils_ovn.go
index df30df80a2..30a2e02b9a 100644
--- a/lxd/network/network_utils_ovn.go
+++ b/lxd/network/network_utils_ovn.go
@@ -9,14 +9,14 @@ import (
// OVNInstanceDevicePortAdd adds a logical port to the OVN network's internal switch and returns the logical
// port name for use linking an OVS port on the integration bridge to the logical switch port.
-func OVNInstanceDevicePortAdd(network Network, instanceID int, instanceName string, deviceName string, mac net.HardwareAddr, ips []net.IP) (openvswitch.OVNSwitchPort, error) {
+func OVNInstanceDevicePortAdd(network Network, instanceID int, instanceName string, deviceName string, mac net.HardwareAddr, ips []net.IP, externalRoutes []*net.IPNet) (openvswitch.OVNSwitchPort, error) {
// Check network is of type OVN.
n, ok := network.(*ovn)
if !ok {
return "", fmt.Errorf("Network is not OVN type")
}
- return n.instanceDevicePortAdd(instanceID, instanceName, deviceName, mac, ips)
+ return n.instanceDevicePortAdd(instanceID, instanceName, deviceName, mac, ips, externalRoutes)
}
// OVNInstanceDevicePortDynamicIPs gets a logical port's dynamic IPs stored in the OVN network's internal switch.
From a0203c8132af90831b024379c68e2baa0005eaa3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 12:01:22 +0100
Subject: [PATCH 25/30] lxd/network/network/utils/ovn: Updates
OVNInstanceDevicePortDelete to accept an externalRoutes argument
So we can clean up external routes on port delete.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/network_utils_ovn.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/network/network_utils_ovn.go b/lxd/network/network_utils_ovn.go
index 30a2e02b9a..948ca6cb4d 100644
--- a/lxd/network/network_utils_ovn.go
+++ b/lxd/network/network_utils_ovn.go
@@ -31,12 +31,12 @@ func OVNInstanceDevicePortDynamicIPs(network Network, instanceID int, deviceName
}
// OVNInstanceDevicePortDelete deletes a logical port from the OVN network's internal switch.
-func OVNInstanceDevicePortDelete(network Network, instanceID int, deviceName string) error {
+func OVNInstanceDevicePortDelete(network Network, instanceID int, deviceName string, externalRoutes []*net.IPNet) error {
// Check network is of type OVN.
n, ok := network.(*ovn)
if !ok {
return fmt.Errorf("Network is not OVN type")
}
- return n.instanceDevicePortDelete(instanceID, deviceName)
+ return n.instanceDevicePortDelete(instanceID, deviceName, externalRoutes)
}
From aa594ec8eea223b20d6325324bd7e2d4d3df84bb Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 12:02:05 +0100
Subject: [PATCH 26/30] lxd/network/driver/ovn: Adds externalRoutes support to
instanceDevicePortAdd
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 41 +++++++++++++++++++++++++++++++++++++--
1 file changed, 39 insertions(+), 2 deletions(-)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index d077c7ffa4..b2c5f18128 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -1813,7 +1813,7 @@ func (n *ovn) getInstanceDevicePortName(instanceID int, deviceName string) openv
}
// instanceDevicePortAdd adds an instance device port to the internal logical switch and returns the port name.
-func (n *ovn) instanceDevicePortAdd(instanceID int, instanceName string, deviceName string, mac net.HardwareAddr, ips []net.IP) (openvswitch.OVNSwitchPort, error) {
+func (n *ovn) instanceDevicePortAdd(instanceID int, instanceName string, deviceName string, mac net.HardwareAddr, ips []net.IP, externalRoutes []*net.IPNet) (openvswitch.OVNSwitchPort, error) {
var dhcpV4ID, dhcpv6ID string
revert := revert.New()
@@ -1872,13 +1872,50 @@ func (n *ovn) instanceDevicePortAdd(instanceID int, instanceName string, deviceN
return "", err
}
- err = client.LogicalSwitchPortSetDNS(n.getIntSwitchName(), instancePortName, fmt.Sprintf("%s.%s", instanceName, n.getDomainName()))
+ dnsIPv4, dnsIPv6, err := client.LogicalSwitchPortSetDNS(n.getIntSwitchName(), instancePortName, fmt.Sprintf("%s.%s", instanceName, n.getDomainName()))
if err != nil {
return "", err
}
revert.Add(func() { client.LogicalSwitchPortDeleteDNS(n.getIntSwitchName(), instancePortName) })
+ // Add each external route (using the IPs set for DNS as target).
+ for _, externalRoute := range externalRoutes {
+ targetIP := dnsIPv4
+ if externalRoute.IP.To4() == nil {
+ targetIP = dnsIPv6
+ }
+
+ if targetIP == nil {
+ return "", fmt.Errorf("Cannot add static route for %q as target IP is not set", externalRoute.String())
+ }
+
+ err = client.LogicalRouterRouteAdd(n.getRouterName(), externalRoute, targetIP, true)
+ if err != nil {
+ return "", err
+ }
+
+ revert.Add(func() { client.LogicalRouterRouteDelete(n.getRouterName(), externalRoute, targetIP) })
+
+ // In order to advertise the external route to the uplink network using proxy ARP/NDP we need to
+ // add a stateless dnat_and_snat rule (as to my knowledge this is the only way to get the OVN
+ // router to respond to ARP/NDP requests for IPs that it doesn't actually have). However we have
+ // to add each IP in the external route individually as DNAT doesn't support whole subnets.
+ err = SubnetIterate(externalRoute, func(ip net.IP) error {
+ err = client.LogicalRouterDNATSNATAdd(n.getRouterName(), ip, ip, true, true)
+ if err != nil {
+ return err
+ }
+
+ revert.Add(func() { client.LogicalRouterDNATSNATDelete(n.getRouterName(), ip) })
+
+ return nil
+ })
+ if err != nil {
+ return "", err
+ }
+ }
+
revert.Success()
return instancePortName, nil
}
From 03df2f2ecd639be4ba06382f3b84898df2e820b0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 12:02:20 +0100
Subject: [PATCH 27/30] lxd/network/driver/ovn: Delete externalRoutes in
instanceDevicePortDelete
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/driver_ovn.go | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go
index b2c5f18128..e6e5b88644 100644
--- a/lxd/network/driver_ovn.go
+++ b/lxd/network/driver_ovn.go
@@ -1933,7 +1933,7 @@ func (n *ovn) instanceDevicePortDynamicIPs(instanceID int, deviceName string) ([
}
// instanceDevicePortDelete deletes an instance device port from the internal logical switch.
-func (n *ovn) instanceDevicePortDelete(instanceID int, deviceName string) error {
+func (n *ovn) instanceDevicePortDelete(instanceID int, deviceName string, externalRoutes []*net.IPNet) error {
instancePortName := n.getInstanceDevicePortName(instanceID, deviceName)
client, err := n.getClient()
@@ -1951,6 +1951,27 @@ func (n *ovn) instanceDevicePortDelete(instanceID int, deviceName string) error
return err
}
+ // Delete each external route.
+ for _, externalRoute := range externalRoutes {
+ err = client.LogicalRouterRouteDelete(n.getRouterName(), externalRoute, nil)
+ if err != nil {
+ return err
+ }
+
+ // Remove the DNAT rules.
+ err = SubnetIterate(externalRoute, func(ip net.IP) error {
+ err = client.LogicalRouterDNATSNATDelete(n.getRouterName(), ip)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ }
+
return nil
}
From dc8fa863b9c1b3319975a5366a29be1b325e562a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 11:25:24 +0100
Subject: [PATCH 28/30] lxd/device/nic: Adds ipv4.routes.external and
ipv6.routes.external to nicValidationRules
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/device/nic.go b/lxd/device/nic.go
index 8bd313b93d..3aa3d164c2 100644
--- a/lxd/device/nic.go
+++ b/lxd/device/nic.go
@@ -34,6 +34,8 @@ func nicValidationRules(requiredFields []string, optionalFields []string) map[st
"ipv6.host_address": validate.Optional(validate.IsNetworkAddressV6),
"ipv4.host_table": validate.Optional(validate.IsUint32),
"ipv6.host_table": validate.Optional(validate.IsUint32),
+ "ipv4.routes.external": validate.Optional(validate.IsNetworkV4List),
+ "ipv6.routes.external": validate.Optional(validate.IsNetworkV6List),
}
validators := map[string]func(value string) error{}
From 74d99b4d722800a66fbd2e0f632033653496854a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 12:02:51 +0100
Subject: [PATCH 29/30] lxd/device/nic/ovn: Adds support for
ipv4.routes.external and ipv6.routes.external
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_ovn.go | 98 +++++++++++++++++++++++++++++++++++++++----
1 file changed, 89 insertions(+), 9 deletions(-)
diff --git a/lxd/device/nic_ovn.go b/lxd/device/nic_ovn.go
index e499c7cf90..d12fa9dfb2 100644
--- a/lxd/device/nic_ovn.go
+++ b/lxd/device/nic_ovn.go
@@ -4,6 +4,7 @@ import (
"fmt"
"net"
"os"
+ "strings"
"github.com/mdlayher/netx/eui64"
"github.com/pkg/errors"
@@ -56,6 +57,8 @@ func (d *nicOVN) validateConfig(instConf instance.ConfigReader) error {
"mtu",
"ipv4.address",
"ipv6.address",
+ "ipv4.routes.external",
+ "ipv6.routes.external",
"boot.priority",
}
@@ -125,6 +128,55 @@ func (d *nicOVN) validateConfig(instConf instance.ConfigReader) error {
}
}
+ // Check IP external routes are within the network's external routes.
+ if d.config["ipv4.routes.external"] != "" || d.config["ipv6.routes.external"] != "" {
+ // Parse network external route subnets.
+ var networkRoutes []*net.IPNet
+ for _, k := range []string{"ipv4.routes.external", "ipv6.routes.external"} {
+ if netConfig[k] == "" {
+ continue
+ }
+
+ routes := strings.Split(netConfig[k], ",")
+ for _, route := range routes {
+ _, networkRoute, err := net.ParseCIDR(strings.TrimSpace(route))
+ if err != nil {
+ return err
+ }
+
+ networkRoutes = append(networkRoutes, networkRoute)
+ }
+ }
+
+ // Parse and validate our external routes.
+ for _, k := range []string{"ipv4.routes.external", "ipv6.routes.external"} {
+ if d.config[k] == "" {
+ continue
+ }
+
+ for _, route := range strings.Split(d.config[k], ",") {
+ route = strings.TrimSpace(route)
+ _, routeSubnet, err := net.ParseCIDR(route)
+ if err != nil {
+ return err
+ }
+
+ // Check that the external route is within the network's routes.
+ foundMatch := false
+ for _, networkRoute := range networkRoutes {
+ if network.SubnetContains(networkRoute, routeSubnet) {
+ foundMatch = true
+ break
+ }
+ }
+
+ if !foundMatch {
+ return fmt.Errorf("Network %q doesn't contain %q in its external routes", n.Name(), routeSubnet.String())
+ }
+ }
+ }
+ }
+
// Apply network level config options to device config before validation.
d.config["mtu"] = fmt.Sprintf("%s", netConfig["bridge.mtu"])
@@ -226,22 +278,37 @@ func (d *nicOVN) Start() (*deviceConfig.RunConfig, error) {
ips := []net.IP{}
for _, key := range []string{"ipv4.address", "ipv6.address"} {
- if d.config[key] != "" {
- ip := net.ParseIP(d.config[key])
- if ip == nil {
- return nil, fmt.Errorf("Invalid %s value %q", key, d.config[key])
- }
- ips = append(ips, ip)
+ if d.config[key] == "" {
+ continue
+ }
+
+ ip := net.ParseIP(d.config[key])
+ if ip == nil {
+ return nil, fmt.Errorf("Invalid %s value %q", key, d.config[key])
+ }
+ ips = append(ips, ip)
+ }
+
+ externalRoutes := []*net.IPNet{}
+ for _, key := range []string{"ipv4.routes.external", "ipv6.routes.external"} {
+ if d.config[key] == "" {
+ continue
+ }
+
+ _, externalRoute, err := net.ParseCIDR(d.config[key])
+ if err != nil {
+ return nil, errors.Wrapf(err, "Invalid %s value %q", key, d.config[key])
}
+ externalRoutes = append(externalRoutes, externalRoute)
}
// Add new OVN logical switch port for instance.
- logicalPortName, err := network.OVNInstanceDevicePortAdd(d.network, d.inst.ID(), d.inst.Name(), d.name, mac, ips)
+ logicalPortName, err := network.OVNInstanceDevicePortAdd(d.network, d.inst.ID(), d.inst.Name(), d.name, mac, ips, externalRoutes)
if err != nil {
return nil, errors.Wrapf(err, "Failed adding OVN port")
}
- revert.Add(func() { network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name) })
+ revert.Add(func() { network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name, externalRoutes) })
// Attach host side veth interface to bridge.
integrationBridge, err := d.getIntegrationBridgeName()
@@ -347,7 +414,20 @@ func (d *nicOVN) Stop() (*deviceConfig.RunConfig, error) {
PostHooks: []func() error{d.postStop},
}
- err := network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name)
+ externalRoutes := []*net.IPNet{}
+ for _, key := range []string{"ipv4.routes.external", "ipv6.routes.external"} {
+ if d.config[key] == "" {
+ continue
+ }
+
+ _, externalRoute, err := net.ParseCIDR(d.config[key])
+ if err != nil {
+ return nil, errors.Wrapf(err, "Invalid %s value %q", key, d.config[key])
+ }
+ externalRoutes = append(externalRoutes, externalRoute)
+ }
+
+ err := network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name, externalRoutes)
if err != nil {
// Don't fail here as we still want the postStop hook to run to clean up the local veth pair.
d.logger.Error("Failed to remove OVN device port", log.Ctx{"err": err})
From daecef34cf83bcab283103be23f3313fdacbe54e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Oct 2020 15:16:44 +0100
Subject: [PATCH 30/30] lxd/network/network/utils: Adds SubnetIterate function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/network/network_utils.go | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
diff --git a/lxd/network/network_utils.go b/lxd/network/network_utils.go
index b4d5811898..7adc0e2fd0 100644
--- a/lxd/network/network_utils.go
+++ b/lxd/network/network_utils.go
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"fmt"
"io/ioutil"
+ "math/big"
"math/rand"
"net"
"os"
@@ -1098,3 +1099,35 @@ func SubnetContains(outerSubnet *net.IPNet, innerSubnet *net.IPNet) bool {
return true
}
+
+// SubnetIterate iterates through each IP in a subnet calling a function for each IP.
+// If the ipFunc returns a non-nil error then the iteration stops and the error is returned.
+func SubnetIterate(subnet *net.IPNet, ipFunc func(ip net.IP) error) error {
+ inc := big.NewInt(1)
+
+ // Convert route start IP to native representations to allow incrementing.
+ startIP := subnet.IP.To4()
+ if startIP == nil {
+ startIP = subnet.IP.To16()
+ }
+
+ startBig := big.NewInt(0)
+ startBig.SetBytes(startIP)
+
+ // Iterate through IPs in subnet, calling ipFunc for each one.
+ for {
+ ip := net.IP(startBig.Bytes())
+ if !subnet.Contains(ip) {
+ break
+ }
+
+ err := ipFunc(ip)
+ if err != nil {
+ return err
+ }
+
+ startBig.Add(startBig, inc)
+ }
+
+ return nil
+}
More information about the lxc-devel
mailing list