[lxc-users] filesystem full -> lxd won't start after freeing space

[Tomasz Chmielewski]

I have a Ubuntu 16.04 server with LXD 3.0 installed from snap.

I've filled the disk to 100% to get "No space left on device".

A while later, it was not possible to use "lxc shell container" or "lxc 
list", because of:

Error: Get http://unix.socket/1.0: dial unix 
/var/snap/lxd/common/lxd/unix.socket: connect: connection refused


The same error shows up after freeing space and restarting the server.

How to debug this?

No scary messages in /var/snap/lxd/common/lxd/logs/lxd.log"

# cat /var/snap/lxd/common/lxd/logs/lxd.log
lvl=info msg="LXD 3.0.0 is starting in normal mode" 
path=/var/snap/lxd/common/lxd t=2018-05-06T08:00:36+0000
lvl=info msg="Kernel uid/gid map:" t=2018-05-06T08:00:36+0000
lvl=info msg=" - u 0 0 4294967295" t=2018-05-06T08:00:36+0000
lvl=info msg=" - g 0 0 4294967295" t=2018-05-06T08:00:36+0000
lvl=info msg="Configured LXD uid/gid map:" t=2018-05-06T08:00:36+0000
lvl=info msg=" - u 0 1000000 1000000000" t=2018-05-06T08:00:36+0000
lvl=info msg=" - g 0 1000000 1000000000" t=2018-05-06T08:00:36+0000
lvl=warn msg="CGroup memory swap accounting is disabled, swap limits 
will be ignored." t=2018-05-06T08:00:36+0000
lvl=info msg="Initializing local database" t=2018-05-06T08:00:36+0000
lvl=info msg="Initializing database gateway" t=2018-05-06T08:00:36+0000
address= id=1 lvl=info msg="Start database node" 
t=2018-05-06T08:00:36+0000
lvl=info msg="Raft: Restored from snapshot 1-1101-1525519473278" 
t=2018-05-06T08:00:36+0000
lvl=info msg="Raft: Initial configuration (index=1): [{Suffrage:Voter 
ID:1 Address:0}]" t=2018-05-06T08:00:36+0000
lvl=info msg="Raft: Node at 0 [Leader] entering Leader state" 
t=2018-05-06T08:00:36+0000
lvl=info msg="LXD isn't socket activated" t=2018-05-06T08:00:36+0000
lvl=info msg="Starting /dev/lxd handler:" t=2018-05-06T08:00:36+0000
lvl=info msg=" - binding devlxd socket" 
socket=/var/snap/lxd/common/lxd/devlxd/sock t=2018-05-06T08:00:36+0000
lvl=info msg="REST API daemon:" t=2018-05-06T08:00:36+0000
lvl=info msg=" - binding Unix socket" 
socket=/var/snap/lxd/common/lxd/unix.socket t=2018-05-06T08:00:36+0000
lvl=info msg=" - binding TCP socket" socket=[::]:8443 
t=2018-05-06T08:00:36+0000
lvl=info msg="Initializing global database" t=2018-05-06T08:00:36+0000


# ps aux|grep lxd
root       3336  0.0  0.0   4504  1656 ?        Ss   10:00   0:00 
/bin/sh /snap/lxd/6954/commands/daemon.start
root       3586  0.0  0.0  95384  1380 ?        Sl   10:00   0:00 lxcfs 
/var/snap/lxd/common/var/lib/lxcfs -p /var/snap/lxd/common/lxcfs.pid
root       3595  0.1  0.0 512472 22660 ?        Sl   10:00   0:00 lxd 
waitready --timeout=600


Tomasz Chmielewski
https://lxadm.com