[lxc-devel] [lxc/master] remove network devices from network namespace

brauner on Github lxc-bot at linuxcontainers.org
Thu Oct 27 12:50:30 UTC 2016


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 362 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20161027/85c6cda1/attachment.bin>
-------------- next part --------------
From 670867cc24f87c34cbb97e28aaae07fabb6f973f Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Thu, 27 Oct 2016 14:35:26 +0200
Subject: [PATCH 1/5] start: add netnsfd to lxc_handler

Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
 src/lxc/start.c | 59 ++++++++++++++++++++++++++++++---------------------------
 src/lxc/start.h |  1 +
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/src/lxc/start.c b/src/lxc/start.c
index ecc7b08..b2d76e4 100644
--- a/src/lxc/start.c
+++ b/src/lxc/start.c
@@ -1060,6 +1060,26 @@ void resolve_clone_flags(struct lxc_handler *handler)
 	}
 }
 
+int get_netns_fd(int pid)
+{
+	char path[MAXPATHLEN];
+	int ret, fd;
+
+	ret = snprintf(path, MAXPATHLEN, "/proc/%d/ns/net", pid);
+	if (ret < 0 || ret >= MAXPATHLEN) {
+		WARN("Failed to pin netns file for pid %d", pid);
+		return -1;
+	}
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0) {
+		WARN("Failed to pin netns file %s for pid %d: %s",
+				path, pid, strerror(errno));
+		return -1;
+	}
+	return fd;
+}
+
 static int lxc_spawn(struct lxc_handler *handler)
 {
 	int failed_before_rename = 0;
@@ -1285,6 +1305,7 @@ static int lxc_spawn(struct lxc_handler *handler)
 	}
 
 	lxc_sync_fini(handler);
+	handler->netnsfd = get_netns_fd(handler->pid);
 
 	return 0;
 
@@ -1304,26 +1325,6 @@ static int lxc_spawn(struct lxc_handler *handler)
 	return -1;
 }
 
-int get_netns_fd(int pid)
-{
-	char path[MAXPATHLEN];
-	int ret, fd;
-
-	ret = snprintf(path, MAXPATHLEN, "/proc/%d/ns/net", pid);
-	if (ret < 0 || ret >= MAXPATHLEN) {
-		WARN("Failed to pin netns file for pid %d", pid);
-		return -1;
-	}
-
-	fd = open(path, O_RDONLY);
-	if (fd < 0) {
-		WARN("Failed to pin netns file %s for pid %d: %s",
-				path, pid, strerror(errno));
-		return -1;
-	}
-	return fd;
-}
-
 int __lxc_start(const char *name, struct lxc_conf *conf,
 		struct lxc_operations* ops, void *data, const char *lxcpath,
 		bool backgrounded)
@@ -1331,7 +1332,6 @@ int __lxc_start(const char *name, struct lxc_conf *conf,
 	struct lxc_handler *handler;
 	int err = -1;
 	int status;
-	int netnsfd = -1;
 
 	handler = lxc_init(name, conf, lxcpath);
 	if (!handler) {
@@ -1341,6 +1341,7 @@ int __lxc_start(const char *name, struct lxc_conf *conf,
 	handler->ops = ops;
 	handler->data = data;
 	handler->backgrounded = backgrounded;
+	handler->netnsfd = -1;
 
 	if (must_drop_cap_sys_boot(handler->conf)) {
 		#if HAVE_SYS_CAPABILITY_H
@@ -1382,13 +1383,13 @@ int __lxc_start(const char *name, struct lxc_conf *conf,
 
 	handler->conf->reboot = 0;
 
-	netnsfd = get_netns_fd(handler->pid);
-
 	err = lxc_poll(name, handler);
 	if (err) {
 		ERROR("mainloop exited with an error");
-		if (netnsfd >= 0)
-			close(netnsfd);
+		if (handler->netnsfd >= 0) {
+			close(handler->netnsfd);
+			handler->netnsfd = -1;
+		}
 		goto out_abort;
 	}
 
@@ -1420,13 +1421,15 @@ int __lxc_start(const char *name, struct lxc_conf *conf,
 	}
 
 	DEBUG("Pushing physical nics back to host namespace");
-	lxc_rename_phys_nics_on_shutdown(netnsfd, handler->conf);
+	lxc_rename_phys_nics_on_shutdown(handler->netnsfd, handler->conf);
 
 	DEBUG("Tearing down virtual network devices used by container");
 	lxc_delete_network(handler);
 
-	if (netnsfd >= 0)
-		close(netnsfd);
+	if (handler->netnsfd >= 0) {
+		close(handler->netnsfd);
+		handler->netnsfd = -1;
+	}
 
 	if (handler->pinfd >= 0) {
 		close(handler->pinfd);
diff --git a/src/lxc/start.h b/src/lxc/start.h
index fe47ab9..65d553b 100644
--- a/src/lxc/start.h
+++ b/src/lxc/start.h
@@ -77,6 +77,7 @@ struct lxc_handler {
 	int ttysock[2]; // socketpair for child->parent tty fd passing
 	bool backgrounded; // indicates whether should we close std{in,out,err} on start
 	int nsfd[LXC_NS_MAX];
+	int netnsfd;
 };
 
 

From 5b65a082be9f9096610d9f8361a3f9bbe467d85c Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Thu, 27 Oct 2016 14:37:24 +0200
Subject: [PATCH 2/5] utils: add lxc_preserve_ns()

This allows to retrieve a file descriptor referring to a namespace.

Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
 src/lxc/utils.c | 13 +++++++++++++
 src/lxc/utils.h |  1 +
 2 files changed, 14 insertions(+)

diff --git a/src/lxc/utils.c b/src/lxc/utils.c
index c912fe8..9d66b71 100644
--- a/src/lxc/utils.c
+++ b/src/lxc/utils.c
@@ -1930,3 +1930,16 @@ bool task_blocking_signal(pid_t pid, int signal)
 	fclose(f);
 	return bret;
 }
+
+int lxc_preserve_ns(const int pid, const char *ns)
+{
+	int ret;
+	size_t len = 5 /* /proc */ + 21 /* /int_as_str */ + 3 /* /ns */ + 20 /* /NS_NAME */ + 1 /* \0 */;
+	char path[len];
+
+	ret = snprintf(path, len, "/proc/%d/ns/%s", pid, ns);
+	if (ret < 0 || (size_t)ret >= len)
+		return -1;
+
+	return open(path, O_RDONLY | O_CLOEXEC);
+}
diff --git a/src/lxc/utils.h b/src/lxc/utils.h
index a0fa0e2..25f19a5 100644
--- a/src/lxc/utils.h
+++ b/src/lxc/utils.h
@@ -310,6 +310,7 @@ int open_devnull(void);
 int set_stdfds(int fd);
 int null_stdfds(void);
 int lxc_count_file_lines(const char *fn);
+int lxc_preserve_ns(const int pid, const char *ns);
 
 /* Check whether a signal is blocked by a process. */
 bool task_blocking_signal(pid_t pid, int signal);

From 8a63dd64dd0d337862e866f4a0295fbdb2c2c78d Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Thu, 27 Oct 2016 14:39:45 +0200
Subject: [PATCH 3/5] conf: use lxc_preserve_ns()

Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
 src/lxc/conf.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/lxc/conf.c b/src/lxc/conf.c
index f688f94..9563d8a 100644
--- a/src/lxc/conf.c
+++ b/src/lxc/conf.c
@@ -2399,22 +2399,18 @@ static int setup_network(struct lxc_list *network)
 /* try to move physical nics to the init netns */
 void restore_phys_nics_to_netns(int netnsfd, struct lxc_conf *conf)
 {
-	int i, ret, oldfd;
-	char path[MAXPATHLEN];
+	int i, oldfd;
 	char ifname[IFNAMSIZ];
 
 	if (netnsfd < 0)
 		return;
 
-	ret = snprintf(path, MAXPATHLEN, "/proc/self/ns/net");
-	if (ret < 0 || ret >= MAXPATHLEN) {
-		WARN("Failed to open monitor netns fd");
-		return;
-	}
-	if ((oldfd = open(path, O_RDONLY)) < 0) {
+	oldfd = lxc_preserve_ns(getpid(), "net");
+	if (oldfd < 0) {
 		SYSERROR("Failed to open monitor netns fd");
 		return;
 	}
+
 	if (setns(netnsfd, 0) != 0) {
 		SYSERROR("Failed to enter container netns to reset nics");
 		close(oldfd);

From d1b82821d691676e095de03ad95c3b16a940430f Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Thu, 27 Oct 2016 14:40:56 +0200
Subject: [PATCH 4/5] conf: indicate whether netdev was moved to a netns

Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
 src/lxc/conf.c | 2 ++
 src/lxc/conf.h | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/lxc/conf.c b/src/lxc/conf.c
index 9563d8a..6394160 100644
--- a/src/lxc/conf.c
+++ b/src/lxc/conf.c
@@ -3039,6 +3039,7 @@ int lxc_assign_network(const char *lxcpath, char *lxcname,
 			// lxc-user-nic has moved the nic to the new ns.
 			// unpriv_assign_nic() fills in netdev->name.
 			// netdev->ifindex will be filed in at setup_netdev.
+			netdev->moved_to_netns = true;
 			continue;
 		}
 
@@ -3059,6 +3060,7 @@ int lxc_assign_network(const char *lxcpath, char *lxcname,
 			return -1;
 		}
 
+		netdev->moved_to_netns = true;
 		DEBUG("move '%s' to '%d'", netdev->name, pid);
 	}
 
diff --git a/src/lxc/conf.h b/src/lxc/conf.h
index e484667..76a08cf 100644
--- a/src/lxc/conf.h
+++ b/src/lxc/conf.h
@@ -136,6 +136,7 @@ struct lxc_netdev {
 	bool ipv6_gateway_auto;
 	char *upscript;
 	char *downscript;
+	bool moved_to_netns;
 };
 
 /*

From 33fa2751add9c46f279649c17651c31d58557fa8 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Thu, 27 Oct 2016 14:41:49 +0200
Subject: [PATCH 5/5] conf: explicitly remove netdev from netns

The kernel sometimes seems a little lazy when cleaning up network namespaces
and leaves network devices behind. This causes issues when restarting
containers rapidly. Let's try to help the kernel along by trying to remove the
network device explicitly from the network namespace before its destruction.

Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
 src/lxc/conf.c | 40 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/src/lxc/conf.c b/src/lxc/conf.c
index 6394160..5239e7b 100644
--- a/src/lxc/conf.c
+++ b/src/lxc/conf.c
@@ -2894,13 +2894,34 @@ int lxc_create_network(struct lxc_handler *handler)
 
 void lxc_delete_network(struct lxc_handler *handler)
 {
+	int oldnetns, ret;
 	struct lxc_list *network = &handler->conf->network;
 	struct lxc_list *iterator;
 	struct lxc_netdev *netdev;
+	bool switched_ns = false;
+
+	oldnetns = lxc_preserve_ns(getpid(), "net");
+	if (oldnetns < 0) {
+		SYSERROR("Failed to open monitor netns fd");
+	}
 
 	lxc_list_for_each(iterator, network) {
 		netdev = iterator->elem;
 
+		if (netdev->moved_to_netns && oldnetns >= 0) {
+			if (!switched_ns && (handler->clone_flags & CLONE_NEWNET) && handler->netnsfd >= 0) {
+				ret = setns(handler->netnsfd, 0);
+				if (ret < 0)
+					WARN("Failed attaching to new network namespace.");
+				switched_ns = true;
+			}
+		} else if (switched_ns && oldnetns >= 0) {
+			ret = setns(oldnetns, 0);
+			if (ret < 0)
+				WARN("Failed attaching to old network namespace.");
+			switched_ns = false;
+		}
+
 		if (netdev->ifindex != 0 && netdev->type == LXC_NET_PHYS) {
 			if (lxc_netdev_rename_by_index(netdev->ifindex, netdev->link))
 				WARN("failed to rename to the initial name the " \
@@ -2916,12 +2937,21 @@ void lxc_delete_network(struct lxc_handler *handler)
 		 * namespace is destroyed but in case we did not moved the
 		 * interface to the network namespace, we have to destroy it
 		 */
-		if (netdev->ifindex != 0 &&
-		    lxc_netdev_delete_by_index(netdev->ifindex))
-			WARN("failed to remove interface %d '%s'",
-				netdev->ifindex,
-				netdev->name ? netdev->name : "(null)");
+		if (netdev->ifindex != 0) {
+			ret = lxc_netdev_delete_by_index(netdev->ifindex);
+			if (ret < 0)
+				WARN("failed to remove interface %d '%s': %s",
+				     netdev->ifindex,
+				     netdev->name ? netdev->name : "(null)", strerror(-ret));
+			else
+				INFO("Removed interface %d '%s'",
+				     netdev->ifindex,
+				     netdev->name ? netdev->name : "(null)");
+		}
 	}
+
+	if (oldnetns >= 0)
+		close(oldnetns);
 }
 
 #define LXC_USERNIC_PATH LIBEXECDIR "/lxc/lxc-user-nic"


More information about the lxc-devel mailing list