[lxc-devel] [lxd/master] Proxy: Adds hairpin NAT rules
tomponline on Github
lxc-bot at linuxcontainers.org
Mon Apr 20 13:16:10 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 377 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200420/be019d70/attachment.bin>
-------------- next part --------------
From 6ec542e41c23e2446a21d91ecec5ed1c616c7edd Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 12:03:47 +0100
Subject: [PATCH 1/7] lxd/device/proxy: Check for br_netfilter enabled and log
warning if not
br_netfilter is be required in order to allow other instances to connect to instance proxy listen addresses in nat mode.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/proxy.go | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/lxd/device/proxy.go b/lxd/device/proxy.go
index 6b509aa38d..ca7ea7146d 100644
--- a/lxd/device/proxy.go
+++ b/lxd/device/proxy.go
@@ -12,6 +12,7 @@ import (
"strings"
"time"
+ "github.com/pkg/errors"
"golang.org/x/sys/unix"
liblxc "gopkg.in/lxc/go-lxc.v2"
@@ -19,6 +20,7 @@ import (
"github.com/lxc/lxd/lxd/instance"
"github.com/lxc/lxd/lxd/instance/instancetype"
"github.com/lxc/lxd/lxd/project"
+ "github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logger"
)
@@ -312,6 +314,11 @@ func (d *proxy) setupNAT() error {
}
}
+ err = d.checkBridgeNetfilterEnabled(ipFamily)
+ if err != nil {
+ logger.Warnf("Proxy bridge netfilter not enabled: %v. Instances using the bridge will not be able to connect to the proxy's listen IP", err)
+ }
+
err = d.state.Firewall.InstanceSetupProxyNAT(d.inst.Project(), d.inst.Name(), d.name, listenAddr, connectAddr)
if err != nil {
return err
@@ -320,6 +327,30 @@ func (d *proxy) setupNAT() error {
return nil
}
+// checkBridgeNetfilterEnabled checks whether the bridge netfilter feature is loaded and enabled.
+// If it is not an error is returned. This is needed in order for instances connected to a bridge to access the
+// proxy's listen IP on the LXD host, as otherwise the packets from the bridge do not go through the netfilter
+// NAT SNAT/MASQUERADE rules.
+func (d *proxy) checkBridgeNetfilterEnabled(ipFamily string) error {
+ sysctlName := "iptables"
+ if ipFamily == "ipv6" {
+ sysctlName = "ip6tables"
+ }
+
+ sysctlPath := fmt.Sprintf("net/bridge/bridge-nf-call-%s", sysctlName)
+ sysctlVal, err := util.SysctlGet(sysctlPath)
+ if err != nil {
+ return errors.Wrap(err, "br_netfilter not loaded")
+ }
+
+ sysctlVal = strings.TrimSpace(sysctlVal)
+ if sysctlVal != "1" {
+ return fmt.Errorf("br_netfilter sysctl net.bridge.bridge-nf-call-%s=%s", sysctlName, sysctlVal)
+ }
+
+ return nil
+}
+
func (d *proxy) rewriteHostAddr(addr string) string {
fields := strings.SplitN(addr, ":", 2)
proto := fields[0]
From 1da3c8cc718100f8e13f216015c269269875d3fa Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 12:07:24 +0100
Subject: [PATCH 2/7] lxd/firewall/drivers/driver/xtables: Adds MASQUERADE
hairpin proxy NAT rule
Allows instance that has proxy device to connect to its own proxy listen IP.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/firewall/drivers/drivers_xtables.go | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/lxd/firewall/drivers/drivers_xtables.go b/lxd/firewall/drivers/drivers_xtables.go
index 8aea7b38da..2f251c8cf0 100644
--- a/lxd/firewall/drivers/drivers_xtables.go
+++ b/lxd/firewall/drivers/drivers_xtables.go
@@ -397,6 +397,13 @@ func (d Xtables) InstanceSetupProxyNAT(projectName string, instanceName string,
if err != nil {
return err
}
+
+ // instance <-> instance.
+ // Requires instance's bridge port has hairpin mode enabled when br_netfilter is loaded.
+ err = d.iptablesPrepend(ipVersion, comment, "nat", "POSTROUTING", "-p", listen.ConnType, "--source", connectHost, "--destination", connectHost, "--dport", connectPort, "-j", "MASQUERADE")
+ if err != nil {
+ return err
+ }
}
revert.Success()
From d649a71598cc61971ae1b20311ae1535b7819d1e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 12:08:33 +0100
Subject: [PATCH 3/7] lxd/firewall/drivers/drivers/xtables: comments
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/firewall/drivers/drivers_xtables.go | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lxd/firewall/drivers/drivers_xtables.go b/lxd/firewall/drivers/drivers_xtables.go
index 2f251c8cf0..7565bf7f65 100644
--- a/lxd/firewall/drivers/drivers_xtables.go
+++ b/lxd/firewall/drivers/drivers_xtables.go
@@ -377,7 +377,7 @@ func (d Xtables) InstanceSetupProxyNAT(projectName string, instanceName string,
return err
}
- // Figure out if we are using iptables or ip6tables and format the destination host/port as appropriate.
+ // Decide if we are using iptables/ip6tables and format the destination host/port as appropriate.
ipVersion := uint(4)
toDest := fmt.Sprintf("%s:%s", connectHost, connectPort)
connectIP := net.ParseIP(connectHost)
@@ -386,13 +386,13 @@ func (d Xtables) InstanceSetupProxyNAT(projectName string, instanceName string,
toDest = fmt.Sprintf("[%s]:%s", connectHost, connectPort)
}
- // outbound <-> container.
+ // outbound <-> instance.
err = d.iptablesPrepend(ipVersion, comment, "nat", "PREROUTING", "-p", listen.ConnType, "--destination", listenHost, "--dport", listenPort, "-j", "DNAT", "--to-destination", toDest)
if err != nil {
return err
}
- // host <-> container.
+ // host <-> instance.
err = d.iptablesPrepend(ipVersion, comment, "nat", "OUTPUT", "-p", listen.ConnType, "--destination", listenHost, "--dport", listenPort, "-j", "DNAT", "--to-destination", toDest)
if err != nil {
return err
@@ -433,7 +433,7 @@ func (d Xtables) InstanceClearProxyNAT(projectName string, instanceName string,
// generateFilterEbtablesRules returns a customised set of ebtables filter rules based on the device.
func (d Xtables) generateFilterEbtablesRules(hostName string, hwAddr string, IPv4 net.IP, IPv6 net.IP) [][]string {
- // MAC source filtering rules. Blocks any packet coming from instance with an incorrect Ethernet source MAC.
+ // MAC source filtering rules. Block any packet coming from instance with an incorrect Ethernet source MAC.
// This is required for IP filtering too.
rules := [][]string{
{"ebtables", "-t", "filter", "-A", "INPUT", "-s", "!", hwAddr, "-i", hostName, "-j", "DROP"},
From d5260209e0f56b1565467351cb24a95a698be14a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 13:16:47 +0100
Subject: [PATCH 4/7] lxd/device/proxy: Sets bridge port hairpin mode on when
br_netfilter loaded
This allows the proxy instance and the other instances on the bridge to connect to the proxy's listen IP.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/proxy.go | 25 +++++++++++++++++++++----
1 file changed, 21 insertions(+), 4 deletions(-)
diff --git a/lxd/device/proxy.go b/lxd/device/proxy.go
index ca7ea7146d..4e25213b58 100644
--- a/lxd/device/proxy.go
+++ b/lxd/device/proxy.go
@@ -273,8 +273,9 @@ func (d *proxy) setupNAT() error {
}
var connectIP net.IP
+ var hostName string
- for _, devConfig := range d.inst.ExpandedDevices() {
+ for devName, devConfig := range d.inst.ExpandedDevices() {
if devConfig["type"] != "nic" || (devConfig["type"] == "nic" && devConfig.NICType() != "bridged") {
continue
}
@@ -285,18 +286,22 @@ func (d *proxy) setupNAT() error {
if ipFamily == "ipv4" && devConfig["ipv4.address"] != "" {
if connectHost == devConfig["ipv4.address"] || connectHost == "0.0.0.0" {
connectIP = net.ParseIP(devConfig["ipv4.address"])
- break
}
} else if ipFamily == "ipv6" && devConfig["ipv6.address"] != "" {
if connectHost == devConfig["ipv6.address"] || connectHost == "::" {
connectIP = net.ParseIP(devConfig["ipv6.address"])
- break
}
}
+
+ if connectIP != nil {
+ // Get host_name of device so we can enable hairpin mode on bridge port.
+ hostName = d.inst.ExpandedConfig()[fmt.Sprintf("volatile.%s.host_name", devName)]
+ break // Found a match, stop searching.
+ }
}
if connectIP == nil {
- return fmt.Errorf("Proxy connect IP cannot be used with any NIC static IPs")
+ return fmt.Errorf("Proxy connect IP cannot be used with any of the instance NICs static IPs")
}
// Override the host part of the connectAddr.Addr to the chosen connect IP.
@@ -317,6 +322,18 @@ func (d *proxy) setupNAT() error {
err = d.checkBridgeNetfilterEnabled(ipFamily)
if err != nil {
logger.Warnf("Proxy bridge netfilter not enabled: %v. Instances using the bridge will not be able to connect to the proxy's listen IP", err)
+ } else {
+ if hostName == "" {
+ return fmt.Errorf("Proxy cannot find bridge port host_name to enable hairpin mode")
+ }
+
+ // br_netfilter is enabled, so we need to enable hairpin mode on instance's bridge port otherwise
+ // the instances on the bridge will not be able to connect to the proxy device's listn IP and the
+ // NAT rule added by the firewall below to allow instance <-> instance traffic will also not work.
+ _, err = shared.RunCommand("bridge", "link", "set", "dev", hostName, "hairpin", "on")
+ if err != nil {
+ return errors.Wrapf(err, "Error enabling hairpin mode on bridge port %q", hostName)
+ }
}
err = d.state.Firewall.InstanceSetupProxyNAT(d.inst.Project(), d.inst.Name(), d.name, listenAddr, connectAddr)
From 7f78e819455c75858534b288a894072f0cabadbb Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 13:25:39 +0100
Subject: [PATCH 5/7] lxd/firewall/drivers/drivers/xtables: Renames toDest to
connectDest
For consistency with connectHost and connectPort vars.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/firewall/drivers/drivers_xtables.go | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lxd/firewall/drivers/drivers_xtables.go b/lxd/firewall/drivers/drivers_xtables.go
index 7565bf7f65..836dfce32e 100644
--- a/lxd/firewall/drivers/drivers_xtables.go
+++ b/lxd/firewall/drivers/drivers_xtables.go
@@ -379,21 +379,21 @@ func (d Xtables) InstanceSetupProxyNAT(projectName string, instanceName string,
// Decide if we are using iptables/ip6tables and format the destination host/port as appropriate.
ipVersion := uint(4)
- toDest := fmt.Sprintf("%s:%s", connectHost, connectPort)
+ connectDest := fmt.Sprintf("%s:%s", connectHost, connectPort)
connectIP := net.ParseIP(connectHost)
if connectIP.To4() == nil {
ipVersion = 6
- toDest = fmt.Sprintf("[%s]:%s", connectHost, connectPort)
+ connectDest = fmt.Sprintf("[%s]:%s", connectHost, connectPort)
}
// outbound <-> instance.
- err = d.iptablesPrepend(ipVersion, comment, "nat", "PREROUTING", "-p", listen.ConnType, "--destination", listenHost, "--dport", listenPort, "-j", "DNAT", "--to-destination", toDest)
+ err = d.iptablesPrepend(ipVersion, comment, "nat", "PREROUTING", "-p", listen.ConnType, "--destination", listenHost, "--dport", listenPort, "-j", "DNAT", "--to-destination", connectDest)
if err != nil {
return err
}
// host <-> instance.
- err = d.iptablesPrepend(ipVersion, comment, "nat", "OUTPUT", "-p", listen.ConnType, "--destination", listenHost, "--dport", listenPort, "-j", "DNAT", "--to-destination", toDest)
+ err = d.iptablesPrepend(ipVersion, comment, "nat", "OUTPUT", "-p", listen.ConnType, "--destination", listenHost, "--dport", listenPort, "-j", "DNAT", "--to-destination", connectDest)
if err != nil {
return err
}
From 68e5f9aa1c895fc7f9247436360c54b6da5fcb02 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 13:27:02 +0100
Subject: [PATCH 6/7] lxd/firewall/drivers/drivers/nftables: Renames toDest to
connectDest
For consistency with connectHost and connectPort vars.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/firewall/drivers/drivers_nftables.go | 14 +++++++-------
lxd/firewall/drivers/drivers_nftables_templates.go | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/lxd/firewall/drivers/drivers_nftables.go b/lxd/firewall/drivers/drivers_nftables.go
index be42a16751..916e357fea 100644
--- a/lxd/firewall/drivers/drivers_nftables.go
+++ b/lxd/firewall/drivers/drivers_nftables.go
@@ -363,19 +363,19 @@ func (d Nftables) InstanceSetupProxyNAT(projectName string, instanceName string,
// Figure out which IP family we are using and format the destination host/port as appropriate.
family := "ip"
- toDest := fmt.Sprintf("%s:%s", connectHost, connectPort)
+ connectDest := fmt.Sprintf("%s:%s", connectHost, connectPort)
connectIP := net.ParseIP(connectHost)
if connectIP.To4() == nil {
family = "ip6"
- toDest = fmt.Sprintf("[%s]:%s", connectHost, connectPort)
+ connectDest = fmt.Sprintf("[%s]:%s", connectHost, connectPort)
}
rules = append(rules, map[string]interface{}{
- "family": family,
- "connType": listen.ConnType,
- "listenHost": listenHost,
- "listenPort": listenPort,
- "toDest": toDest,
+ "family": family,
+ "connType": listen.ConnType,
+ "listenHost": listenHost,
+ "listenPort": listenPort,
+ "connectDest": connectDest,
})
}
diff --git a/lxd/firewall/drivers/drivers_nftables_templates.go b/lxd/firewall/drivers/drivers_nftables_templates.go
index 49fcf35e67..6aae6006cb 100644
--- a/lxd/firewall/drivers/drivers_nftables_templates.go
+++ b/lxd/firewall/drivers/drivers_nftables_templates.go
@@ -57,7 +57,7 @@ var nftablesNetProxyNAT = template.Must(template.New("nftablesNetProxyNAT").Pars
chain prert{{.chainSeparator}}{{.deviceLabel}} {
type nat hook prerouting priority -100; policy accept;
{{- range .rules}}
- {{.family}} daddr {{.listenHost}} {{.connType}} dport {{.listenPort}} dnat to {{.toDest}}
+ {{.family}} daddr {{.listenHost}} {{.connType}} dport {{.listenPort}} dnat to {{.connectDest}}
{{- end}}
}
From b6664b8cc7a65f91f425dafc66ce5d1d9e471677 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 20 Apr 2020 14:14:08 +0100
Subject: [PATCH 7/7] lxd/firewall/drivers/drivers/nftables: Adds MASQUERADE
hairpin proxy NAT rule
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/firewall/drivers/drivers_nftables.go | 4 +++-
lxd/firewall/drivers/drivers_nftables_templates.go | 11 +++++++++--
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/lxd/firewall/drivers/drivers_nftables.go b/lxd/firewall/drivers/drivers_nftables.go
index 916e357fea..d13aad89b3 100644
--- a/lxd/firewall/drivers/drivers_nftables.go
+++ b/lxd/firewall/drivers/drivers_nftables.go
@@ -376,6 +376,8 @@ func (d Nftables) InstanceSetupProxyNAT(projectName string, instanceName string,
"listenHost": listenHost,
"listenPort": listenPort,
"connectDest": connectDest,
+ "connectHost": connectHost,
+ "connectPort": connectPort,
})
}
@@ -399,7 +401,7 @@ func (d Nftables) InstanceSetupProxyNAT(projectName string, instanceName string,
// InstanceClearProxyNAT remove DNAT rules for proxy devices.
func (d Nftables) InstanceClearProxyNAT(projectName string, instanceName string, deviceName string) error {
deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName)
- err := d.removeChains([]string{"ip", "ip6"}, deviceLabel, "out", "prert")
+ err := d.removeChains([]string{"ip", "ip6"}, deviceLabel, "out", "prert", "pstrt")
if err != nil {
return errors.Wrapf(err, "Failed clearing proxy rules for instance device %q", deviceLabel)
}
diff --git a/lxd/firewall/drivers/drivers_nftables_templates.go b/lxd/firewall/drivers/drivers_nftables_templates.go
index 6aae6006cb..31551e4bcd 100644
--- a/lxd/firewall/drivers/drivers_nftables_templates.go
+++ b/lxd/firewall/drivers/drivers_nftables_templates.go
@@ -61,10 +61,17 @@ chain prert{{.chainSeparator}}{{.deviceLabel}} {
{{- end}}
}
-chain out{{.chainSeparator}}{{.deviceLabel}}{
+chain out{{.chainSeparator}}{{.deviceLabel}} {
type nat hook output priority -100; policy accept;
{{- range .rules}}
- {{.family}} daddr {{.listenHost}} {{.connType}} dport {{.listenPort}} dnat to {{.toDest}}
+ {{.family}} daddr {{.listenHost}} {{.connType}} dport {{.listenPort}} dnat to {{.connectDest}}
+ {{- end}}
+}
+
+chain pstrt{{.chainSeparator}}{{.deviceLabel}} {
+ type nat hook postrouting priority 100; policy accept;
+ {{- range .rules}}
+ {{.family}} saddr {{.connectHost}} ip daddr {{.connectHost}} {{.connType}} dport {{.connectPort}} masquerade
{{- end}}
}
`))
More information about the lxc-devel
mailing list