[lxc-devel] [lxcfs/master] Per-container CPU usage in /proc/stat
aither64 on Github
lxc-bot at linuxcontainers.org
Fri Jun 15 06:48:34 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 1298 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180615/3d83d5dc/attachment.bin>
-------------- next part --------------
From de5efbb3cec16c2df3940cfe7486582794f1271a Mon Sep 17 00:00:00 2001
From: Jakub Skokan <jakub.skokan at havefun.cz>
Date: Thu, 14 Jun 2018 11:51:58 +0200
Subject: [PATCH] Per-container CPU usage in /proc/stat
Containers can see utilization of all available CPUs, even if the CPU
is utilized by other containers or by the host. The contents
of `/proc/stat` is shared across the system, except for hiding CPUs
excluded by cpuset. This commit attempts to fix that, but at a cost.
CPU usage is read from cpuacct cgroup, but that accounts only for
`user` and `system` fields from `/proc/stat`. Idle time can be
calculated, but other fields cannot, thus are always set to 0.
Signed-off-by: Jakub Skokan <jakub.skokan at havefun.cz>
---
bindings.c | 173 ++++++++++++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 136 insertions(+), 37 deletions(-)
diff --git a/bindings.c b/bindings.c
index 6ab1bdb..1ce0784 100644
--- a/bindings.c
+++ b/bindings.c
@@ -80,6 +80,11 @@ struct file_info {
int cached;
};
+struct cpuacct_usage {
+ uint64_t user;
+ uint64_t system;
+};
+
/* The function of hash table.*/
#define LOAD_SIZE 100 /*the size of hash_table */
#define FLUSH_TIME 5 /*the flush rate */
@@ -3794,6 +3799,90 @@ static uint64_t get_reaper_age(pid_t pid)
return procage;
}
+/*
+ * Returns 0 on success.
+ * It is the caller's responsibility to free `return_usage`, unless this
+ * function returns an error.
+ */
+static int read_cpuacct_usage_all(char *cg, char *cpuset, struct cpuacct_usage **return_usage)
+{
+
+ int cpucount = get_nprocs();
+ struct cpuacct_usage *cpu_usage;
+ int rv = 0, i, j, ret, read_pos = 0, read_cnt;
+ int cg_cpu;
+ uint64_t cg_user, cg_system;
+ int64_t ticks_per_sec;
+ char *usage_str = NULL;
+
+ ticks_per_sec = sysconf(_SC_CLK_TCK);
+
+ if (ticks_per_sec < 0 && errno == EINVAL) {
+ lxcfs_debug(
+ "%s\n",
+ "read_cpuacct_usage_all failed to determine number of clock ticks "
+ "in a second");
+ return -1;
+ }
+
+ cpu_usage = malloc(sizeof(struct cpuacct_usage) * cpucount);
+ if (!cpu_usage)
+ return -ENOMEM;
+
+ if (!cgfs_get_value("cpuacct", cg, "cpuacct.usage_all", &usage_str)) {
+ rv = -1;
+ goto err;
+ }
+
+ if (sscanf(usage_str, "cpu user system\n%n", &read_cnt) != 0) {
+ lxcfs_error("read_cpuacct_usage_all reading first line from "
+ "%s/cpuacct.usage_all failed.\n", cg);
+ rv = -1;
+ goto err;
+ }
+
+ read_pos += read_cnt;
+
+ for (i = 0, j = 0; i < cpucount; i++) {
+ ret = sscanf(usage_str + read_pos, "%d %lu %lu\n%n", &cg_cpu, &cg_user,
+ &cg_system, &read_cnt);
+
+ if (ret == EOF) {
+ break;
+
+ } else if (ret != 3) {
+ lxcfs_error("read_cpuacct_usage_all reading from %s/cpuacct.usage_all "
+ "failed.\n", cg);
+ rv = -1;
+ goto err;
+ }
+
+ read_pos += read_cnt;
+
+ if (!cpu_in_cpuset(i, cpuset))
+ continue;
+
+ /* Convert the time from nanoseconds to USER_HZ */
+ cpu_usage[j].user = cg_user / 1000.0 / 1000 / 1000 * ticks_per_sec;
+ cpu_usage[j].system = cg_system / 1000.0 / 1000 / 1000 * ticks_per_sec;
+ j++;
+ }
+
+ rv = 0;
+ *return_usage = cpu_usage;
+
+err:
+ if (usage_str)
+ free(usage_str);
+
+ if (rv != 0) {
+ free(cpu_usage);
+ *return_usage = NULL;
+ }
+
+ return rv;
+}
+
#define CPUALL_MAX_SIZE (BUF_RESERVE_SIZE / 2)
static int proc_stat_read(char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
@@ -3806,13 +3895,13 @@ static int proc_stat_read(char *buf, size_t size, off_t offset,
size_t linelen = 0, total_len = 0, rv = 0;
int curcpu = -1; /* cpu numbering starts at 0 */
unsigned long user = 0, nice = 0, system = 0, idle = 0, iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guest_nice = 0;
- unsigned long user_sum = 0, nice_sum = 0, system_sum = 0, idle_sum = 0, iowait_sum = 0,
- irq_sum = 0, softirq_sum = 0, steal_sum = 0, guest_sum = 0, guest_nice_sum = 0;
+ unsigned long user_sum = 0, system_sum = 0, idle_sum = 0;
char cpuall[CPUALL_MAX_SIZE];
/* reserve for cpu all */
char *cache = d->buf + CPUALL_MAX_SIZE;
size_t cache_size = d->buflen - CPUALL_MAX_SIZE;
FILE *f = NULL;
+ struct cpuacct_usage *cg_cpu_usage = NULL;
if (offset){
if (offset > d->size)
@@ -3837,6 +3926,12 @@ static int proc_stat_read(char *buf, size_t size, off_t offset,
if (!cpuset)
goto err;
+ /* Read cpuacct.usage_all for all CPUs */
+ if (read_cpuacct_usage_all(cg, cpuset, &cg_cpu_usage) < 0) {
+ lxcfs_error("%s\n", "proc_stat_read failed to read from cpuacct.");
+ goto err;
+ }
+
f = fopen("/proc/stat", "r");
if (!f)
goto err;
@@ -3851,7 +3946,7 @@ static int proc_stat_read(char *buf, size_t size, off_t offset,
ssize_t l;
int cpu;
char cpu_char[10]; /* That's a lot of cores */
- char *c;
+ uint64_t all_used, cg_used, new_idle;
if (strlen(line) == 0)
continue;
@@ -3880,10 +3975,36 @@ static int proc_stat_read(char *buf, size_t size, off_t offset,
continue;
curcpu ++;
- c = strchr(line, ' ');
- if (!c)
+ if (sscanf(line, "%*s %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu",
+ &user,
+ &nice,
+ &system,
+ &idle,
+ &iowait,
+ &irq,
+ &softirq,
+ &steal,
+ &guest,
+ &guest_nice) != 10)
continue;
- l = snprintf(cache, cache_size, "cpu%d%s", curcpu, c);
+
+ all_used = user + nice + system + iowait + irq + softirq + steal + guest + guest_nice;
+ cg_used = cg_cpu_usage[curcpu].user + cg_cpu_usage[curcpu].system;
+
+ if (all_used >= cg_used) {
+ new_idle = idle + (all_used - cg_used);
+
+ } else {
+ lxcfs_error("cpu%d from %s has unexpected cpu time: %lu in /proc/stat, "
+ "%lu in cpuacct.usage_all; unable to determine idle time\n",
+ curcpu, cg, all_used, cg_used);
+ new_idle = idle;
+ }
+
+ l = snprintf(cache, cache_size, "cpu%d %lu 0 %lu %lu 0 0 0 0 0 0\n",
+ curcpu, cg_cpu_usage[curcpu].user, cg_cpu_usage[curcpu].system,
+ new_idle);
+
if (l < 0) {
perror("Error writing to cache");
rv = 0;
@@ -3900,43 +4021,17 @@ static int proc_stat_read(char *buf, size_t size, off_t offset,
cache_size -= l;
total_len += l;
- if (sscanf(line, "%*s %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu",
- &user,
- &nice,
- &system,
- &idle,
- &iowait,
- &irq,
- &softirq,
- &steal,
- &guest,
- &guest_nice) != 10)
- continue;
- user_sum += user;
- nice_sum += nice;
- system_sum += system;
- idle_sum += idle;
- iowait_sum += iowait;
- irq_sum += irq;
- softirq_sum += softirq;
- steal_sum += steal;
- guest_sum += guest;
- guest_nice_sum += guest_nice;
+ user_sum += cg_cpu_usage[curcpu].user;
+ system_sum += cg_cpu_usage[curcpu].system;
+ idle_sum += new_idle;
}
cache = d->buf;
- int cpuall_len = snprintf(cpuall, CPUALL_MAX_SIZE, "cpu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu\n",
+ int cpuall_len = snprintf(cpuall, CPUALL_MAX_SIZE, "cpu %lu 0 %lu %lu 0 0 0 0 0 0\n",
user_sum,
- nice_sum,
system_sum,
- idle_sum,
- iowait_sum,
- irq_sum,
- softirq_sum,
- steal_sum,
- guest_sum,
- guest_nice_sum);
+ idle_sum);
if (cpuall_len > 0 && cpuall_len < CPUALL_MAX_SIZE) {
memcpy(cache, cpuall, cpuall_len);
cache += cpuall_len;
@@ -3959,6 +4054,10 @@ static int proc_stat_read(char *buf, size_t size, off_t offset,
err:
if (f)
fclose(f);
+
+ if (cg_cpu_usage)
+ free(cg_cpu_usage);
+
free(line);
free(cpuset);
free(cg);
More information about the lxc-devel
mailing list