[lxc-users] macvlan-based networking for unprivileged containers
Fajar A. Nugraha
list at fajar.net
Tue Feb 17 06:07:04 UTC 2015
On Mon, Feb 16, 2015 at 9:52 PM, Serge Hallyn <serge.hallyn at ubuntu.com> wrote:
> Quoting overlay fs (overlayfs at gmail.com):
>> > > However veth works
>> > > just fine. And you don't have to put your public link (e.g. eth0) on
>> > > bridge mode to have a working container with veth network.
>> >
>> > FWIW what it would take is an extension to lxc-user-nic to support
>> > (accounted) unpriv macvlan. /etc/lxc/lxc-usernet would then support
>> > something like "$user macvlan eth0 10".
>> >
>> > But as Fajar says, the value of this seems dubious, and I'm not sure
>> > whether that would have the same snooping-on-same-link concerns
>> > that you'd have with a bridged eth0.
>>
>> Is there presently a way to block network traffic between unprivileged
>> containers, or between a container and the host? This could be
>> desirable when running untrusted containers.
>
> You (your administrator) could create separate bridges for each user.
It might be useful to enhance lxc-user-nic to allow:
- setting lxc.network.veth.pair
- allow veth without bridge (i.e. no lxc.network.link line on config file)
With those two capabilities you could make routed setup without any
bridge, where all containers route their traffic thru the host similar
to the way pptp works. Containers can have IPs in the same segment as
eth0, but can't see traffic meant to other IPs thru link-snooping. In
this setup you DON'T need separate bridges for each user/container,
but you DO need a config stanza (including fixed IP allocation) on
host's /etc/network/interfaces for each container.
This setup currently works on my test setup, privileged container. It
also works for have root-started unprivileged container (i.e. created
and started by root in /var/lib/lxc, but uses "lxc.include =
/usr/share/lxc/config/ubuntu.userns.conf" and lxc.id_map) since it
doesn't use lxc-user-nic. It does NOT work user-started unprivileged
container.
Assuming:
- your public link is eth0, 192.168.124.30/24 (LAN address in my test setup)
- your containers (c1 and c2) gets IP address 192.168.124.251 and
192.168.124.252
- you allocate private IP 172.16.0.1 for container's gateway (can be
any private IP of your choice)
##########
Host setup
##########
/etc/network/interfaces (if using ubuntu).
###
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet static
address 192.168.124.130
netmask 255.255.255.0
gateway 192.168.124.1
# c1's veth name on host side
auto v-c1-0
iface v-c1-0 inet static
address 172.16.0.1/32
scope link
pointopoint 192.168.124.251
# c2's veth name on host side
auto v-c2-0
iface v-c2-0 inet static
# note that this is the same IP as above, not a typo
address 172.16.0.1/32
scope link
# c2's IP
pointopoint 192.168.124.252
###
... enable proxy arp on the host (this simplifies your setup, instead
of setting up static arps manually). Activate with "start
procps-instance" (ubuntu 14.10), or simply reboot.
###
# cat /etc/sysctl.d/50-eth0-proxy_arp.conf
net.ipv4.conf.eth0.proxy_arp=1
###
##########
C1 setup
##########
lxc config file
###
lxc.network.type=veth
# comment out the next line, we don't need a bridge
#lxc.network.link=
lxc.network.veth.pair=veth-c1-0
lxc.network.flags=up
lxc.network.hwaddr = 00:16:3E:08:EB:E1
###
/etc/network/interfaces (assuming it also runs ubuntu)
###
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet static
address 192.168.124.251/32
pointopoint 172.16.0.1
gateway 172.16.0.1
###
##########
C2 setup
##########
lxc config file
###
lxc.network.type=veth
# comment out the next line, we don't need a bridge
#lxc.network.link=
lxc.network.veth.pair=veth-c2-0
lxc.network.flags=up
lxc.network.hwaddr = 00:16:3E:08:EB:E2
###
/etc/network/interfaces
###
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet static
address 192.168.124.252/32
pointopoint 172.16.0.1
gateway 172.16.0.1
###
##############################
Result from host side when both containers are started:
##############################
# lxc-ls -f
NAME STATE IPV4 IPV6 GROUPS AUTOSTART
-------------------------------------------------------
c1 RUNNING 192.168.124.251 - - NO
c2 RUNNING 192.168.124.252 - - NO
# ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
group default
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast
state UP group default qlen 1000
link/ether 08:00:27:85:b3:65 brd ff:ff:ff:ff:ff:ff
inet 192.168.124.130/24 brd 192.168.124.255 scope global eth0
valid_lft forever preferred_lft forever
19: v-c1-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
pfifo_fast state UP group default qlen 1000
link/ether fe:09:ae:43:e5:5b brd ff:ff:ff:ff:ff:ff
inet 172.16.0.1 peer 192.168.124.251/32 brd 172.16.0.1 scope link v-c1-0
valid_lft forever preferred_lft forever
21: v-c2-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
pfifo_fast state UP group default qlen 1000
link/ether fe:1d:d9:f1:49:68 brd ff:ff:ff:ff:ff:ff
inet 172.16.0.1 peer 192.168.124.252/32 brd 172.16.0.1 scope link v-c2-0
valid_lft forever preferred_lft forever
# netstat -nr
Kernel IP routing table
Destination Gateway Genmask Flags MSS Window irtt Iface
0.0.0.0 192.168.124.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 0 0 0 eth0
192.168.124.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
192.168.124.251 0.0.0.0 255.255.255.255 UH 0 0 0 v-c1-0
192.168.124.252 0.0.0.0 255.255.255.255 UH 0 0 0 v-c2-0
##############################
Result on c1
##############################
# ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
group default
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
18: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast
state UP group default qlen 1000
link/ether 00:16:3e:63:7a:a6 brd ff:ff:ff:ff:ff:ff
inet 192.168.124.251 peer 172.16.0.1/32 brd 192.168.124.251 scope
global eth0
valid_lft forever preferred_lft forever
inet6 fe80::216:3eff:fe63:7aa6/64 scope link
valid_lft forever preferred_lft forever
# netstat -nr
Kernel IP routing table
Destination Gateway Genmask Flags MSS Window irtt Iface
0.0.0.0 172.16.0.1 0.0.0.0 UG 0 0 0 eth0
172.16.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 eth0
# ping -n -c 1 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.
64 bytes from 8.8.8.8: icmp_seq=1 ttl=54 time=18.6 ms
--- 8.8.8.8 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 18.670/18.670/18.670/0.000 ms
##############################
Result from another host on the same LAN
##############################
# ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
group default
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast
state UP group default qlen 1000
link/ether 08:00:27:69:69:41 brd ff:ff:ff:ff:ff:ff
inet 192.168.124.182/24 brd 192.168.124.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fe69:6941/64 scope link
valid_lft forever preferred_lft forever
# ping -n -c 1 192.168.124.130
PING 192.168.124.130 (192.168.124.130) 56(84) bytes of data.
64 bytes from 192.168.124.130: icmp_seq=1 ttl=64 time=0.587 ms
--- 192.168.124.130 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.587/0.587/0.587/0.000 ms
# ping -n -c 1 192.168.124.251
PING 192.168.124.251 (192.168.124.251) 56(84) bytes of data.
64 bytes from 192.168.124.251: icmp_seq=1 ttl=63 time=0.810 ms
--- 192.168.124.251 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.810/0.810/0.810/0.000 ms
# ping -n -c 1 192.168.124.252
PING 192.168.124.252 (192.168.124.252) 56(84) bytes of data.
64 bytes from 192.168.124.252: icmp_seq=1 ttl=63 time=0.822 ms
--- 192.168.124.252 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.822/0.822/0.822/0.000 ms
# arp -n
Address HWtype HWaddress Flags Mask Iface
192.168.124.252 ether 08:00:27:85:b3:65 C eth0
192.168.124.251 ether 08:00:27:85:b3:65 C eth0
192.168.124.1 ether f6:9b:b5:e7:c2:08 C eth0
192.168.124.130 ether 08:00:27:85:b3:65 C eth0
Note that other hosts on the LAN sees all container IPs belong to the
same MAC (the host's eth0).
--
Fajar
More information about the lxc-users
mailing list