[lxc-devel] [PATCH 2/2] RFC: cgroup: start introducing support for cgroup manager
Serge Hallyn
serge.hallyn at ubuntu.com
Mon Jan 13 21:33:16 UTC 2014
With this patch all regular cgroup functinality still seems to
work. The cgmanager support however doesn't even feign
completeness and is completely untested. The patch is to show
the direction I'm going. You can see how lxc interacts with cgmanager
at the lxc_cgmanager_enter() and lxc_cgmanager_create() calls.
Signed-off-by: Serge Hallyn <serge.hallyn at ubuntu.com>
---
configure.ac | 9 +++
src/lxc/Makefile.am | 9 +++
src/lxc/cgroup.c | 214 ++++++++++++++++++++++++++++++++++++++++++++--------
src/lxc/cgroup.h | 1 +
4 files changed, 201 insertions(+), 32 deletions(-)
diff --git a/configure.ac b/configure.ac
index e153693..1c74ee3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -31,6 +31,15 @@ AC_CANONICAL_HOST
AM_PROG_CC_C_O
AC_GNU_SOURCE
+PKG_CHECK_MODULES([CGMANAGER], [libcgmanager], [have_cgmanager=yes], [have_cgmanager=no])
+
+if test "$have_cgmanager" = yes; then
+ AC_DEFINE([HAVE_CGMANAGER], [1], [Build with cgmanager support])
+ PKG_CHECK_MODULES([NIH], [libnih >= 1.0.2])
+ PKG_CHECK_MODULES([NIH_DBUS], [libnih-dbus >= 1.0.0])
+ PKG_CHECK_MODULES([DBUS], [dbus-1 >= 1.2.16])
+fi
+
# Detect the distribution. This is used for the default configuration and
# for some distro-specific build options.
AC_MSG_CHECKING([host distribution])
diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
index 1e0232b..01f6026 100644
--- a/src/lxc/Makefile.am
+++ b/src/lxc/Makefile.am
@@ -144,6 +144,11 @@ liblxc_so_LDFLAGS = \
liblxc_so_LDADD = $(CAP_LIBS) $(APPARMOR_LIBS) $(SECCOMP_LIBS)
+#if HAVE_CGMANAGER
+liblxc_so_LDADD += $(CGMANAGER_LIBS) $(DBUS_LIBS) $(NIH_LIBS) $(NIH_DBUS_LIBS)
+liblxc_so_CFLAGS += $(CGMANAGER_CFLAGS) $(DBUS_CFLAGS) $(NIH_CFLAGS) $(NIH_DBUS_CFLAGS)
+#endif
+
bin_SCRIPTS = \
lxc-ps \
lxc-netstat \
@@ -245,6 +250,10 @@ LDADD=liblxc.so @CAP_LIBS@ @APPARMOR_LIBS@ @SECCOMP_LIBS@
lxc_attach_SOURCES = lxc_attach.c
lxc_autostart_SOURCES = lxc_autostart.c
lxc_cgroup_SOURCES = lxc_cgroup.c
+#if HAVE_CGMANAGER
+lxc_cgroup_LDADD = $(CGMANAGER_LIBS) $(DBUS_LIBS) $(NIH_LIBS) $(NIH_DBUS_LIBS) $(LDADD)
+lxc_cgroup_CFLAGS = $(CGMANAGER_CFLAGS) $(DBUS_CFLAGS) $(NIH_CFLAGS) $(NIH_DBUS_CFLAGS)
+#endif
lxc_checkpoint_SOURCES = lxc_checkpoint.c
lxc_config_SOURCES = lxc_config.c
lxc_console_SOURCES = lxc_console.c
diff --git a/src/lxc/cgroup.c b/src/lxc/cgroup.c
index ff237fe..8a81a12 100644
--- a/src/lxc/cgroup.c
+++ b/src/lxc/cgroup.c
@@ -82,6 +82,103 @@ static int cgroup_recursive_task_count(const char *cgroup_path);
static int count_lines(const char *fn);
static int handle_cgroup_settings(struct cgroup_mount_point *mp, char *cgroup_path);
+#ifdef HAVE_CGMANAGER
+#include <nih-dbus/dbus_connection.h>
+#include <cgmanager-client/cgmanager-client.h>
+/* this needs to be mutexed for api use */
+static bool cgmanager_initialized = false;
+static bool use_cgmanager = true;
+/* move this stuff into cgmanager.c */
+NihDBusProxy *cgroup_manager = NULL;
+
+static bool lxc_init_cgmanager(void);
+static void cgmanager_disconnected(DBusConnection *connection)
+{
+ WARN("Cgroup manager connection was terminated");
+ cgroup_manager = NULL;
+ cgmanager_initialized = false;
+ if (lxc_init_cgmanager()) {
+ cgmanager_initialized = true;
+ INFO("New cgroup manager connection was opened");
+ }
+}
+
+#define CGMANAGER_DBUS_SOCK "unix:path=/sys/fs/cgroup/cgmanager/sock"
+static bool lxc_init_cgmanager(void)
+{
+ DBusError dbus_error;
+ DBusConnection *connection;
+ dbus_error_init(&dbus_error);
+
+ connection = nih_dbus_connect(CGMANAGER_DBUS_SOCK, cgmanager_disconnected);
+ if (!connection) {
+ ERROR("Error opening cgmanager connection at %s", CGMANAGER_DBUS_SOCK);
+ return false;
+ }
+ dbus_connection_set_exit_on_disconnect(connection, FALSE);
+ dbus_error_free(&dbus_error);
+ cgroup_manager = nih_dbus_proxy_new(NULL, connection,
+ NULL /* p2p */,
+ "/org/linuxcontainers/cgmanager", NULL, NULL);
+ dbus_connection_unref(connection);
+ if (!cgroup_manager) {
+ return false;
+ }
+ return true;
+}
+
+/*
+ * Use the cgmanager to move a task into a cgroup for a particular
+ * hierarchy.
+ * All the subsystems in this hierarchy are co-mounted, so we only
+ * need to transition the task into one of the cgroups
+ */
+static bool lxc_cgmanager_enter(pid_t pid, struct cgroup_hierarchy *h, char *cgroup_path)
+{
+ char *controller = h->subsystems[0];
+
+ if (!cgmanager_initialized) {
+ if (!lxc_init_cgmanager()) {
+ ERROR("%s: could not reach cgmanager", __func__);
+ return -1;
+ }
+ }
+ return cgmanager_move_pid_sync(NULL, cgroup_manager, controller,
+ cgroup_path, pid) == 0;
+}
+
+static bool lxc_cgmanager_create(struct cgroup_mount_point *mp, const char *cgroup_path)
+{
+ char *controller = mp->hierarchy->subsystems[0];
+
+ if (!cgmanager_initialized) {
+ if (!lxc_init_cgmanager()) {
+ ERROR("%s: could not reach cgmanager", __func__);
+ return -1;
+ }
+ }
+ if ( cgmanager_create_sync(NULL, cgroup_manager, controller,
+ cgroup_path) != 0) {
+ ERROR("Failed to create %s:%s", controller, cgroup_path);
+ return -1;
+ }
+
+ // TODO - try to chown the cgroup to the container root
+ return 0;
+}
+
+#else
+static inline bool lxc_cgmanager_enter(pid_t pid, struct cgroup_hierarchy *h, char *cgroup_path)
+{
+ return false;
+}
+static inline bool lxc_cgmanager_create(struct cgroup_mount_point *mp, const char *cgroup_path)
+{
+ return false;
+}
+static bool use_cgmanager = false;
+#endif
+
static int cgroup_rmdir(char *dirname)
{
struct dirent dirent, *direntp;
@@ -498,15 +595,20 @@ struct cgroup_meta_data *lxc_cgroup_load_meta2(const char **subsystem_whitelist)
all_named_subsystems, subsystem_whitelist))
goto out_error;
- if (!find_hierarchy_mountpts(meta_data, kernel_subsystems))
- goto out_error;
-
- /* oops, we couldn't find anything */
- if (!meta_data->hierarchies || !meta_data->mount_points) {
+ if (!meta_data->hierarchies) {
errno = EINVAL;
goto out_error;
}
+ if (!use_cgmanager) {
+ if (!find_hierarchy_mountpts(meta_data, kernel_subsystems))
+ goto out_error;
+ if (!meta_data->mount_points) {
+ errno = EINVAL;
+ goto out_error;
+ }
+ }
+
lxc_free_array((void **)kernel_subsystems, free);
return meta_data;
@@ -723,7 +825,7 @@ extern struct cgroup_process_info *lxc_cgroup_create(const char *name, const cha
char *path_so_far = NULL;
char **new_cgroup_paths = NULL;
char **new_cgroup_paths_sub = NULL;
- struct cgroup_mount_point *mp;
+ struct cgroup_mount_point *mp = NULL;
struct cgroup_hierarchy *h;
struct cgroup_process_info *base_info = NULL;
struct cgroup_process_info *info_ptr;
@@ -763,20 +865,22 @@ extern struct cgroup_process_info *lxc_cgroup_create(const char *name, const cha
goto out_initial_error;
/* find mount points we can use */
- for (info_ptr = base_info; info_ptr; info_ptr = info_ptr->next) {
- h = info_ptr->hierarchy;
- mp = lxc_cgroup_find_mount_point(h, info_ptr->cgroup_path, true);
- if (!mp) {
- ERROR("Could not find writable mount point for cgroup hierarchy %d while trying to create cgroup.", h->index);
- goto out_initial_error;
- }
- info_ptr->designated_mount_point = mp;
+ if (!use_cgmanager) {
+ for (info_ptr = base_info; info_ptr; info_ptr = info_ptr->next) {
+ h = info_ptr->hierarchy;
+ mp = lxc_cgroup_find_mount_point(h, info_ptr->cgroup_path, true);
+ if (!mp) {
+ ERROR("Could not find writable mount point for cgroup hierarchy %d while trying to create cgroup.", h->index);
+ goto out_initial_error;
+ }
+ info_ptr->designated_mount_point = mp;
- if (lxc_string_in_array("ns", (const char **)h->subsystems))
- continue;
- if (handle_cgroup_settings(mp, info_ptr->cgroup_path) < 0) {
- ERROR("Could not set clone_children to 1 for cpuset hierarchy in parent cgroup.");
- goto out_initial_error;
+ if (lxc_string_in_array("ns", (const char **)h->subsystems))
+ continue;
+ if (handle_cgroup_settings(mp, info_ptr->cgroup_path) < 0) {
+ ERROR("Could not set clone_children to 1 for cpuset hierarchy in parent cgroup.");
+ goto out_initial_error;
+ }
}
}
@@ -974,6 +1078,8 @@ int lxc_cgroup_create_legacy(struct cgroup_process_info *base_info, const char *
struct cgroup_process_info *info_ptr;
int r;
+ if (use_cgmanager)
+ return 0;
for (info_ptr = base_info; info_ptr; info_ptr = info_ptr->next) {
if (!lxc_string_in_array("ns", (const char **)info_ptr->hierarchy->subsystems))
continue;
@@ -1030,7 +1136,10 @@ struct cgroup_process_info *lxc_cgroup_get_container_info(const char *name, cons
/* it is not an error if we don't find anything here,
* it is up to the caller to decide what to do in that
* case */
- entry->designated_mount_point = lxc_cgroup_find_mount_point(h, entry->cgroup_path, true);
+ if (use_cgmanager)
+ entry->designated_mount_point = NULL;
+ else
+ entry->designated_mount_point = lxc_cgroup_find_mount_point(h, entry->cgroup_path, true);
*cptr = entry;
cptr = &entry->next;
@@ -1061,6 +1170,14 @@ int lxc_cgroup_enter(struct cgroup_process_info *info, pid_t pid, bool enter_sub
info_ptr->cgroup_path_sub :
info_ptr->cgroup_path;
+ if (use_cgmanager) {
+ if (!lxc_cgmanager_enter(pid, info_ptr->hierarchy, cgroup_path)) {
+ ERROR("Could not add %lu to cgroup %s",
+ (unsigned long)pid, cgroup_path);
+ return -1;
+ }
+ continue;
+ }
if (!info_ptr->designated_mount_point) {
info_ptr->designated_mount_point = lxc_cgroup_find_mount_point(info_ptr->hierarchy, cgroup_path, true);
if (!info_ptr->designated_mount_point) {
@@ -1110,15 +1227,19 @@ void lxc_cgroup_process_info_free_and_remove(struct cgroup_process_info *info)
return;
next = info->next;
{
- struct cgroup_mount_point *mp = info->designated_mount_point;
- if (!mp)
- mp = lxc_cgroup_find_mount_point(info->hierarchy, info->cgroup_path, true);
- if (mp)
- /* ignore return value here, perhaps we created the
- * '/lxc' cgroup in this container but another container
- * is still running (for example)
- */
- (void)remove_cgroup(mp, info->cgroup_path, true);
+ if (use_cgmanager) {
+ (void)remove_cgroup(NULL, info->cgroup_path, true);
+ } else {
+ struct cgroup_mount_point *mp = info->designated_mount_point;
+ if (!mp)
+ mp = lxc_cgroup_find_mount_point(info->hierarchy, info->cgroup_path, true);
+ if (mp)
+ /* ignore return value here, perhaps we created the
+ * '/lxc' cgroup in this container but another container
+ * is still running (for example)
+ */
+ (void)remove_cgroup(mp, info->cgroup_path, true);
+ }
}
for (pp = info->created_paths; pp && *pp; pp++);
for ((void)(pp && --pp); info->created_paths && pp >= info->created_paths; --pp) {
@@ -1151,6 +1272,10 @@ char *lxc_cgroup_get_hierarchy_abs_path_handler(const char *subsystem, struct lx
struct cgroup_process_info *info = find_info_for_subsystem(handler->cgroup, subsystem);
if (!info)
return NULL;
+
+ if (use_cgmanager)
+ return cgroup_to_absolute_path(NULL, info->cgroup_path, NULL);
+
if (info->designated_mount_point) {
mp = info->designated_mount_point;
} else {
@@ -1342,6 +1467,12 @@ int lxc_setup_mount_cgroup(const char *root, struct cgroup_process_info *base_in
return -1;
}
+ /* TODO if using cgmanager, then cgroup_auto should mount /sys/fs/cgroup/cgmanager/ */
+ if (use_cgmanager) {
+ INFO("Using cgmanager, so not mounting sysfs paths");
+ goto remount_ro;
+ }
+
/* now mount all the hierarchies we care about */
for (info = base_info; info; info = info->next) {
size_t subsystem_count, i;
@@ -1469,6 +1600,7 @@ int lxc_setup_mount_cgroup(const char *root, struct cgroup_process_info *base_in
parts = NULL;
}
+remount_ro:
/* try to remount the tmpfs readonly, since the container shouldn't
* change anything (this will also make sure that trying to create
* new cgroups outside the allowed area fails with an error instead
@@ -1681,9 +1813,17 @@ bool is_valid_cgroup(const char *name)
return strcmp(name, ".") != 0 && strcmp(name, "..") != 0;
}
+/* TODO - if in a userns, try to chown the new cgroup to the container root */
int create_or_remove_cgroup(bool do_remove, struct cgroup_mount_point *mp, const char *path, int recurse)
{
int r, saved_errno = 0;
+
+ if (use_cgmanager) {
+ if (do_remove) // not yet implemented in cgmanager
+ return 0;
+ return lxc_cgmanager_create(mp, path) ? 0 : -1;
+ }
+
char *buf = cgroup_to_absolute_path(mp, path, NULL);
if (!buf)
return -1;
@@ -1696,6 +1836,7 @@ int create_or_remove_cgroup(bool do_remove, struct cgroup_mount_point *mp, const
r = rmdir(buf);
} else
r = mkdir(buf, 0777);
+
saved_errno = errno;
free(buf);
errno = saved_errno;
@@ -1715,10 +1856,12 @@ int remove_cgroup(struct cgroup_mount_point *mp, const char *path, bool recurse)
char *cgroup_to_absolute_path(struct cgroup_mount_point *mp, const char *path, const char *suffix)
{
/* first we have to make sure we subtract the mount point's prefix */
- char *prefix = mp->mount_prefix;
+ char *prefix = NULL;
char *buf;
ssize_t len, rv;
+ if (mp)
+ prefix = mp->mount_prefix;
/* we want to make sure only absolute paths to cgroups are passed to us */
if (path[0] != '/') {
errno = EINVAL;
@@ -1742,11 +1885,14 @@ char *cgroup_to_absolute_path(struct cgroup_mount_point *mp, const char *path, c
/* remove prefix from path */
path += prefix ? strlen(prefix) : 0;
- len = strlen(mp->mount_point) + strlen(path) + (suffix ? strlen(suffix) : 0);
+ len = strlen(path) + (suffix ? strlen(suffix) : 0);
+ if (mp)
+ len += strlen(mp->mount_point);
buf = calloc(len + 1, 1);
if (!buf)
return NULL;
- rv = snprintf(buf, len + 1, "%s%s%s", mp->mount_point, path, suffix ? suffix : "");
+ rv = snprintf(buf, len + 1, "%s%s%s", mp->mount_point ? mp->mount_point : "",
+ path, suffix ? suffix : "");
if (rv > len) {
free(buf);
errno = ENOMEM;
@@ -1986,6 +2132,10 @@ int handle_cgroup_settings(struct cgroup_mount_point *mp, char *cgroup_path)
int r, saved_errno = 0;
char buf[2];
+ // If using the cgmanager, it will have set these for us
+ if (!mp)
+ return 0;
+
/* If this is the memory cgroup, we want to enforce hierarchy.
* But don't fail if for some reason we can't.
*/
diff --git a/src/lxc/cgroup.h b/src/lxc/cgroup.h
index 3aab12d..6ba83ac 100644
--- a/src/lxc/cgroup.h
+++ b/src/lxc/cgroup.h
@@ -41,6 +41,7 @@ struct cgroup_meta_data {
struct cgroup_hierarchy **hierarchies;
struct cgroup_mount_point **mount_points;
int maximum_hierarchy;
+ bool use_cgmanager;
};
/*
--
1.8.5.2
More information about the lxc-devel
mailing list