[lxc-devel] [lxd/master] NIC IPVLAN: Add custom policy routing table support
tomponline on Github
lxc-bot at linuxcontainers.org
Wed Apr 15 10:55:11 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 766 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200415/cd310134/attachment.bin>
-------------- next part --------------
From f1f3cc1692fe4c98cdbee13b76f14d0fbe945bd9 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 10:11:50 +0100
Subject: [PATCH 1/9] lxd/device/nic: Adds host_table setting validation rule
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/device/nic.go b/lxd/device/nic.go
index 2b0841c3f2..4af56b01a5 100644
--- a/lxd/device/nic.go
+++ b/lxd/device/nic.go
@@ -36,6 +36,7 @@ func nicValidationRules(requiredFields []string, optionalFields []string) map[st
"vlan": shared.IsAny,
"hwaddr": networkValidMAC,
"host_name": shared.IsAny,
+ "host_table": shared.IsUint32,
"limits.ingress": shared.IsAny,
"limits.egress": shared.IsAny,
"limits.max": shared.IsAny,
From 84cbdec1a1f9f90050a4d80fb4f1057f8ff89d5c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 10:13:12 +0100
Subject: [PATCH 2/9] lxd/device/nic/routed: Fix sysctl command suggestion when
using vlans
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_routed.go | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/lxd/device/nic_routed.go b/lxd/device/nic_routed.go
index 66372f1233..e0166e70d0 100644
--- a/lxd/device/nic_routed.go
+++ b/lxd/device/nic_routed.go
@@ -131,7 +131,8 @@ func (d *nicRouted) validateEnvironment() error {
return fmt.Errorf("Error reading net sysctl %s: %v", ipv4FwdPath, err)
}
if sysctlVal != "1\n" {
- return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", effectiveParentName)
+ // Replace . in parent name with / for sysctl formatting.
+ return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/"))
}
}
@@ -143,7 +144,8 @@ func (d *nicRouted) validateEnvironment() error {
return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
}
if sysctlVal != "1\n" {
- return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", effectiveParentName)
+ // Replace . in parent name with / for sysctl formatting.
+ return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/"))
}
ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", effectiveParentName)
@@ -152,7 +154,8 @@ func (d *nicRouted) validateEnvironment() error {
return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
}
if sysctlVal != "1\n" {
- return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", effectiveParentName)
+ // Replace . in parent name with / for sysctl formatting.
+ return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", strings.ReplaceAll(effectiveParentName, ".", "/"))
}
}
From de45bd8ef3fc45fa2e5eeeaa519b05e8f0bdf698 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 10:13:37 +0100
Subject: [PATCH 3/9] lxd/device/nic/routed: Add host_table support
Allows adding static routes for instance IPs to custom policy routing tables.
Fixes #7152
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_routed.go | 40 ++++++++++++++++++++++++++++++++++++----
1 file changed, 36 insertions(+), 4 deletions(-)
diff --git a/lxd/device/nic_routed.go b/lxd/device/nic_routed.go
index e0166e70d0..5dbea050c2 100644
--- a/lxd/device/nic_routed.go
+++ b/lxd/device/nic_routed.go
@@ -39,6 +39,7 @@ func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error {
"mtu",
"hwaddr",
"host_name",
+ "host_table",
"vlan",
"ipv4.gateway",
"ipv6.gateway",
@@ -294,10 +295,7 @@ func (d *nicRouted) setupParentSysctls(parentName string) error {
func (d *nicRouted) postStart() error {
v := d.volatileGet()
- // If host_name is defined (and it should be), then we add the dummy link-local gateway IPs
- // to the host end of the veth pair. This ensures that liveness detection of the gateways
- // inside the instance work and ensure that traffic doesn't periodically halt whilst ARP/NDP
- // is re-detected.
+ // If volatile host_name is defined (and it should be), then configure the host-side interface.
if v["host_name"] != "" {
// Attempt to disable IPv6 router advertisement acceptance.
err := util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", v["host_name"]), "0")
@@ -318,17 +316,51 @@ func (d *nicRouted) postStart() error {
}
if d.config["ipv4.address"] != "" {
+ // Add dummy link-local gateway IPs to the host end of the veth pair. This ensures that
+ // liveness detection of the gateways inside the instance work and ensure that traffic
+ // doesn't periodically halt whilst ARP is re-detected.
_, err := shared.RunCommand("ip", "-4", "addr", "add", fmt.Sprintf("%s/32", d.ipv4HostAddress()), "dev", v["host_name"])
if err != nil {
return err
}
+
+ // Add static routes to instance IPs to custom routing tables if specified.
+ // This is in addition to the static route added by liblxc to the main routing table, which
+ // is still critical to ensure that reverse path filtering doesn't kick in blocking traffic
+ // from the instance.
+ if d.config["host_table"] != "" {
+ for _, addr := range strings.Split(d.config["ipv4.address"], ",") {
+ addr = strings.TrimSpace(addr)
+ _, err := shared.RunCommand("ip", "-4", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/32", addr), "dev", v["host_name"])
+ if err != nil {
+ return err
+ }
+ }
+ }
}
if d.config["ipv6.address"] != "" {
+ // Add dummy link-local gateway IPs to the host end of the veth pair. This ensures that
+ // liveness detection of the gateways inside the instance work and ensure that traffic
+ // doesn't periodically halt whilst NDP is re-detected.
_, err := shared.RunCommand("ip", "-6", "addr", "add", fmt.Sprintf("%s/128", d.ipv6HostAddress()), "dev", v["host_name"])
if err != nil {
return err
}
+
+ // Add static routes to instance IPs to custom routing tables if specified.
+ // This is in addition to the static route added by liblxc to the main routing table, which
+ // is still critical to ensure that reverse path filtering doesn't kick in blocking traffic
+ // from the instance.
+ if d.config["host_table"] != "" {
+ for _, addr := range strings.Split(d.config["ipv6.address"], ",") {
+ addr = strings.TrimSpace(addr)
+ _, err := shared.RunCommand("ip", "-6", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/128", addr), "dev", v["host_name"])
+ if err != nil {
+ return err
+ }
+ }
+ }
}
}
From ac391675781727029b4f2e5adb457e561a83c4fe Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 10:16:50 +0100
Subject: [PATCH 4/9] api: Adds container_nic_routed_host_table extension
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/api-extensions.md | 4 ++++
shared/version/api.go | 1 +
2 files changed, 5 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 82fb120b81..7e1587551e 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1013,3 +1013,7 @@ Exposes the die\_id information on each core.
This introduces two new fields in `/1.0`, `os` and `os\_version`.
Those are taken from the os-release data on the system.
+
+## container\_nic\_routed\_host\_table
+This introduces the `host_table` NIC config key that can be used to add static routes for the instance's IPs to a
+custom policy routing table by ID.
diff --git a/shared/version/api.go b/shared/version/api.go
index 87e58fd56b..6c914d1aeb 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -205,6 +205,7 @@ var APIExtensions = []string{
"resources_cpu_threads_numa",
"resources_cpu_core_die",
"api_os",
+ "container_nic_routed_host_table",
}
// APIExtensionsCount returns the number of available API extensions.
From f98d3fd73cd9b96b82895e11dc7c63eddd83eead Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 10:18:53 +0100
Subject: [PATCH 5/9] doc: Adds documentation for routed NIC host_table setting
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/instances.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/instances.md b/doc/instances.md
index c212149f0c..e84c4ce2b8 100644
--- a/doc/instances.md
+++ b/doc/instances.md
@@ -469,6 +469,7 @@ Key | Type | Default | Required | Descriptio
parent | string | - | no | The name of the host device to join the instance to
name | string | kernel assigned | no | The name of the interface inside the instance
host\_name | string | randomly assigned | no | The name of the interface inside the host
+host\_table | integer | - | no | The custom policy routing table ID to add IP static routes to (in addition to main routing table).
mtu | integer | parent MTU | no | The MTU of the new interface
hwaddr | string | randomly assigned | no | The MAC address of the new interface
ipv4.address | string | - | no | Comma delimited list of IPv4 static addresses to add to the instance
From 8f7977dbe1ef323dd0930d8a396b9f6ebd27a450 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 10:45:35 +0100
Subject: [PATCH 6/9] lxd/device/nic/vlan: Improve validation of sysctl
settings when vlan setting used
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_ipvlan.go | 38 +++++++++++++++++++++++++++-----------
1 file changed, 27 insertions(+), 11 deletions(-)
diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go
index e51af7d96f..4247e4545a 100644
--- a/lxd/device/nic_ipvlan.go
+++ b/lxd/device/nic_ipvlan.go
@@ -66,45 +66,61 @@ func (d *nicIPVLAN) validateEnvironment() error {
return fmt.Errorf("Requires name property to start")
}
+ extensions := d.state.OS.LXCFeatures
+ if !extensions["network_ipvlan"] || !extensions["network_l2proxy"] || !extensions["network_gateway_device_route"] {
+ return fmt.Errorf("Requires liblxc has following API extensions: network_ipvlan, network_l2proxy, network_gateway_device_route")
+ }
+
if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) {
return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"])
}
- extensions := d.state.OS.LXCFeatures
- if !extensions["network_ipvlan"] || !extensions["network_l2proxy"] || !extensions["network_gateway_device_route"] {
- return fmt.Errorf("Requires liblxc has following API extensions: network_ipvlan, network_l2proxy, network_gateway_device_route")
+ if d.config["parent"] == "" && d.config["vlan"] != "" {
+ return fmt.Errorf("The vlan setting can only be used when combined with a parent interface")
+ }
+
+ // Generate effective parent name, including the VLAN part if option used.
+ effectiveParentName := network.GetHostDevice(d.config["parent"], d.config["vlan"])
+
+ // If the effective parent doesn't exist and the vlan option is specified, it means we are going to create
+ // the VLAN parent at start, and we will configure the needed sysctls so don't need to check them yet.
+ if d.config["vlan"] != "" && !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", effectiveParentName)) {
+ return nil
}
if d.config["ipv4.address"] != "" {
// Check necessary sysctls are configured for use with l2proxy parent in IPVLAN l3s mode.
- ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", d.config["parent"])
+ ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", effectiveParentName)
sysctlVal, err := util.SysctlGet(ipv4FwdPath)
if err != nil || sysctlVal != "1\n" {
return fmt.Errorf("Error reading net sysctl %s: %v", ipv4FwdPath, err)
}
if sysctlVal != "1\n" {
- return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv4.conf.%s.forwarding=1", d.config["parent"])
+ // Replace . in parent name with / for sysctl formatting.
+ return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv4.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/"))
}
}
if d.config["ipv6.address"] != "" {
// Check necessary sysctls are configured for use with l2proxy parent in IPVLAN l3s mode.
- ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", d.config["parent"])
+ ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", effectiveParentName)
sysctlVal, err := util.SysctlGet(ipv6FwdPath)
if err != nil {
return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
}
if sysctlVal != "1\n" {
- return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.forwarding=1", d.config["parent"])
+ // Replace . in parent name with / for sysctl formatting.
+ return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/"))
}
- ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", d.config["parent"])
+ ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", effectiveParentName)
sysctlVal, err = util.SysctlGet(ipv6ProxyNdpPath)
if err != nil {
return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
}
if sysctlVal != "1\n" {
- return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", d.config["parent"])
+ // Replace . in parent name with / for sysctl formatting.
+ return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", strings.ReplaceAll(effectiveParentName, ".", "/"))
}
}
@@ -207,13 +223,13 @@ func (d *nicIPVLAN) setupParentSysctls(parentName string) error {
ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", parentName)
err := util.SysctlSet(ipv6FwdPath, "1")
if err != nil {
- return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
+ return fmt.Errorf("Error setting net sysctl %s: %v", ipv6FwdPath, err)
}
ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", parentName)
err = util.SysctlSet(ipv6ProxyNdpPath, "1")
if err != nil {
- return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
+ return fmt.Errorf("Error setting net sysctl %s: %v", ipv6ProxyNdpPath, err)
}
}
From 73a92ebe0e9f16bc5b832020a4e72747ddd29507 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 11:48:22 +0100
Subject: [PATCH 7/9] lxd/device/nic/ipvlan: Adds host_table setting support
Fixes #7123
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_ipvlan.go | 61 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 61 insertions(+)
diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go
index 4247e4545a..c28386a764 100644
--- a/lxd/device/nic_ipvlan.go
+++ b/lxd/device/nic_ipvlan.go
@@ -31,6 +31,7 @@ func (d *nicIPVLAN) validateConfig(instConf instance.ConfigReader) error {
"name",
"mtu",
"hwaddr",
+ "host_table",
"vlan",
"ipv4.gateway",
"ipv6.gateway",
@@ -202,6 +203,7 @@ func (d *nicIPVLAN) Start() (*deviceConfig.RunConfig, error) {
}
runConf.NetworkInterface = nic
+ runConf.PostHooks = append(runConf.PostHooks, d.postStart)
return &runConf, nil
}
@@ -236,6 +238,39 @@ func (d *nicIPVLAN) setupParentSysctls(parentName string) error {
return nil
}
+// postStart is run after the instance is started.
+func (d *nicIPVLAN) postStart() error {
+ if d.config["ipv4.address"] != "" {
+ // Add static routes to instance IPs to custom routing tables if specified.
+ // This is in addition to the static route added by liblxc to the main routing table.
+ if d.config["host_table"] != "" {
+ for _, addr := range strings.Split(d.config["ipv4.address"], ",") {
+ addr = strings.TrimSpace(addr)
+ _, err := shared.RunCommand("ip", "-4", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/32", addr), "dev", "lo")
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+
+ if d.config["ipv6.address"] != "" {
+ // Add static routes to instance IPs to custom routing tables if specified.
+ // This is in addition to the static route added by liblxc to the main routing table.
+ if d.config["host_table"] != "" {
+ for _, addr := range strings.Split(d.config["ipv6.address"], ",") {
+ addr = strings.TrimSpace(addr)
+ _, err := shared.RunCommand("ip", "-6", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/128", addr), "dev", "lo")
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
// Stop is run when the device is removed from the instance.
func (d *nicIPVLAN) Stop() (*deviceConfig.RunConfig, error) {
runConf := deviceConfig.RunConfig{
@@ -253,6 +288,32 @@ func (d *nicIPVLAN) postStop() error {
v := d.volatileGet()
+ if d.config["ipv4.address"] != "" {
+ // Remove static routes to instance IPs to custom routing tables if specified.
+ if d.config["host_table"] != "" {
+ for _, addr := range strings.Split(d.config["ipv4.address"], ",") {
+ addr = strings.TrimSpace(addr)
+ _, err := shared.RunCommand("ip", "-4", "route", "delete", "table", d.config["host_table"], fmt.Sprintf("%s/32", addr), "dev", "lo")
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+
+ if d.config["ipv6.address"] != "" {
+ // Remove static routes to instance IPs to custom routing tables if specified.
+ if d.config["host_table"] != "" {
+ for _, addr := range strings.Split(d.config["ipv6.address"], ",") {
+ addr = strings.TrimSpace(addr)
+ _, err := shared.RunCommand("ip", "-6", "route", "delete", "table", d.config["host_table"], fmt.Sprintf("%s/128", addr), "dev", "lo")
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+
// This will delete the parent interface if we created it for VLAN parent.
if shared.IsTrue(v["last_state.created"]) {
parentName := network.GetHostDevice(d.config["parent"], d.config["vlan"])
From 498c15842cc37c598f41bc0b2ba0e4c854450ddd Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 11:50:44 +0100
Subject: [PATCH 8/9] api: Adds container_nic_ipvlan_host_table extension
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/api-extensions.md | 4 ++++
shared/version/api.go | 1 +
2 files changed, 5 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 7e1587551e..8b55a04693 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1017,3 +1017,7 @@ Those are taken from the os-release data on the system.
## container\_nic\_routed\_host\_table
This introduces the `host_table` NIC config key that can be used to add static routes for the instance's IPs to a
custom policy routing table by ID.
+
+## container\_nic\_ipvlan\_host\_table
+This introduces the `host_table` NIC config key that can be used to add static routes for the instance's IPs to a
+custom policy routing table by ID.
diff --git a/shared/version/api.go b/shared/version/api.go
index 6c914d1aeb..9673f9cce5 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -206,6 +206,7 @@ var APIExtensions = []string{
"resources_cpu_core_die",
"api_os",
"container_nic_routed_host_table",
+ "container_nic_ipvlan_host_table",
}
// APIExtensionsCount returns the number of available API extensions.
From ca7b4a4fd1953ce117d0c92fbfc6356c93b04c48 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 15 Apr 2020 11:51:36 +0100
Subject: [PATCH 9/9] doc: Adds documentation for ipvlan NIC host_table setting
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
doc/instances.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/instances.md b/doc/instances.md
index e84c4ce2b8..0b6eb78504 100644
--- a/doc/instances.md
+++ b/doc/instances.md
@@ -329,6 +329,7 @@ parent | string | - | yes | The name o
name | string | kernel assigned | no | The name of the interface inside the instance
mtu | integer | parent MTU | no | The MTU of the new interface
hwaddr | string | randomly assigned | no | The MAC address of the new interface
+host\_table | integer | - | no | The custom policy routing table ID to add IP static routes to (in addition to main routing table).
vlan | integer | - | no | The VLAN ID to attach to
maas.subnet.ipv4 | string | - | no | MAAS IPv4 subnet to register the instance in
maas.subnet.ipv6 | string | - | no | MAAS IPv6 subnet to register the instance in
More information about the lxc-devel
mailing list