[lxc-users] sshd-keygen fails during container boot

Wed Dec 9 19:46:51 UTC 2015

On 12/09/2015 10:18 AM, Serge Hallyn wrote:
> This is the kind of thing I'd expect when using cgmanager or lxcfs, 
> but not with straight lxc+cgfs. Can you show what /sys/fs/cgroup tree 
> and /proc/1/cgroup looks like in a working container? 
As requested:

# ll /sys/fs/cgroup(top level only)
total 0
drwxr-xr-x 3 root root 60 Dec  9 10:12 blkio
lrwxrwxrwx 1 root root 11 Dec  9 10:12 cpu -> cpu,cpuacct
drwxr-xr-x 3 root root 60 Dec  9 10:12 cpu,cpuacct
lrwxrwxrwx 1 root root 11 Dec  9 10:12 cpuacct -> cpu,cpuacct
drwxr-xr-x 3 root root 60 Dec  9 10:12 cpuset
drwxr-xr-x 3 root root 60 Dec  9 10:12 devices
drwxr-xr-x 3 root root 60 Dec  9 10:12 freezer
drwxr-xr-x 3 root root 60 Dec  9 10:12 hugetlb
drwxr-xr-x 3 root root 60 Dec  9 10:12 memory
lrwxrwxrwx 1 root root 16 Dec  9 10:12 net_cls -> net_cls,net_prio
drwxr-xr-x 3 root root 60 Dec  9 10:12 net_cls,net_prio
lrwxrwxrwx 1 root root 16 Dec  9 10:12 net_prio -> net_cls,net_prio
drwxr-xr-x 3 root root 60 Dec  9 10:12 perf_event
dr-xr-xr-x 4 root root  0 Dec  9 10:28 systemd

# cat /proc/1/cgroup
10:hugetlb:/lxc/vm-00
9:perf_event:/lxc/vm-00
8:net_cls,net_prio:/lxc/vm-00
7:freezer:/lxc/vm-00
6:devices:/lxc/vm-00
5:memory:/lxc/vm-00
4:blkio:/lxc/vm-00
3:cpu,cpuacct:/lxc/vm-00
2:cpuset:/lxc/vm-00
1:name=systemd:/system.slice/supervisord.service

And for a bonus:

# mount
/dev/md1 on / type ext4 (rw,relatime,stripe=256,data=ordered)
none on /dev type tmpfs (rw,relatime,size=100k,mode=755)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
proc on /proc/sys/net type proc (rw,nosuid,nodev,noexec,relatime)
proc on /proc/sys type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/sysrq-trigger type proc (ro,nosuid,nodev,noexec,relatime)
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
sysfs on /sys type sysfs (ro,nosuid,nodev,noexec,relatime)
sysfs on /sys/devices/virtual/net type sysfs (rw,relatime)
sysfs on /sys/devices/virtual/net type sysfs 
(rw,nosuid,nodev,noexec,relatime)
sysfs on /sys/fs/fuse/connections type sysfs 
(rw,nosuid,nodev,noexec,relatime)
cgroup_root on /sys/fs/cgroup type tmpfs 
(rw,nosuid,nodev,noexec,relatime,size=10240k,mode=755)
cgroup_root on /sys/fs/cgroup/hugetlb type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/hugetlb/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup_root on /sys/fs/cgroup/perf_event type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/perf_event/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup_root on /sys/fs/cgroup/net_cls,net_prio type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/net_cls,net_prio/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup_root on /sys/fs/cgroup/freezer type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/freezer/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,freezer)
cgroup_root on /sys/fs/cgroup/devices type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/devices/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,devices)
cgroup_root on /sys/fs/cgroup/memory type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/memory/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,memory)
cgroup_root on /sys/fs/cgroup/blkio type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/blkio/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,blkio)
cgroup_root on /sys/fs/cgroup/cpu,cpuacct type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/cpu,cpuacct/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup_root on /sys/fs/cgroup/cpuset type tmpfs 
(ro,relatime,size=10240k,mode=755)
cgroup on /sys/fs/cgroup/cpuset/lxc/vm-00 type cgroup 
(rw,nosuid,nodev,noexec,relatime,cpuset,clone_children)
devpts on /dev/lxc/console type devpts 
(rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
devpts on /dev/pts type devpts (rw,relatime,gid=5,mode=620,ptmxmode=666)
devpts on /dev/lxc/tty1 type devpts 
(rw,relatime,gid=5,mode=620,ptmxmode=666)
devpts on /dev/lxc/tty2 type devpts 
(rw,relatime,gid=5,mode=620,ptmxmode=666)
devpts on /dev/lxc/tty3 type devpts 
(rw,relatime,gid=5,mode=620,ptmxmode=666)
devpts on /dev/lxc/tty4 type devpts 
(rw,relatime,gid=5,mode=620,ptmxmode=666)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup 
(rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
configfs on /sys/kernel/config type configfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)

> Interesting.
>
> I suppose just looking at the 'capsh --print' output difference for the
> bounding set between the custom containers spawned by lxc and libvirt-lxc could
> be enlightening.
Here's the diff:

# sdiff lxc libvirt
cap_chown cap_chown
cap_dac_override cap_dac_override
cap_dac_read_search cap_dac_read_search
cap_fowner cap_fowner
cap_fsetid cap_fsetid
cap_kill cap_kill
cap_setgid cap_setgid
cap_setuid cap_setuid
cap_setpcap cap_setpcap
cap_linux_immutable cap_linux_immutable
cap_net_bind_service cap_net_bind_service
cap_net_broadcast cap_net_broadcast
cap_net_admin cap_net_admin
cap_net_raw cap_net_raw
cap_ipc_lock cap_ipc_lock
cap_ipc_owner cap_ipc_owner
                                                               > 
cap_sys_rawio
cap_sys_chroot cap_sys_chroot
cap_sys_ptrace cap_sys_ptrace
                                                               > 
cap_sys_pacct
cap_sys_admin cap_sys_admin
cap_sys_boot cap_sys_boot
                                                               > 
cap_sys_nice
cap_sys_resource cap_sys_resource
cap_sys_tty_config cap_sys_tty_config
cap_mknod                                                     <
cap_lease cap_lease
cap_audit_write cap_audit_write
cap_audit_control                                             | cap_setfcap
cap_setfcap,cap_syslog                                        | 
cap_mac_override
                                                               > cap_syslog

I've tried another config as well that is more similar, but the systemd 
errors still occur:

# sdiff lxc libvirt
cap_chown cap_chown
cap_dac_override cap_dac_override
cap_dac_read_search cap_dac_read_search
cap_fowner cap_fowner
cap_fsetid cap_fsetid
cap_kill cap_kill
cap_setgid cap_setgid
cap_setuid cap_setuid
cap_setpcap cap_setpcap
cap_linux_immutable cap_linux_immutable
cap_net_bind_service cap_net_bind_service
cap_net_broadcast cap_net_broadcast
cap_net_admin cap_net_admin
cap_net_raw cap_net_raw
cap_ipc_lock cap_ipc_lock
cap_ipc_owner cap_ipc_owner
cap_sys_rawio cap_sys_rawio
cap_sys_chroot cap_sys_chroot
cap_sys_ptrace cap_sys_ptrace
cap_sys_pacct cap_sys_pacct
cap_sys_admin cap_sys_admin
cap_sys_boot cap_sys_boot
cap_sys_nice cap_sys_nice
cap_sys_resource cap_sys_resource
cap_sys_tty_config cap_sys_tty_config
cap_mknod <
cap_lease cap_lease
cap_audit_write cap_audit_write
cap_audit_control <
cap_setfcap cap_setfcap
 > cap_mac_override
cap_syslog cap_syslog