[lxc-users] ?==?utf-8?q? ?==?utf-8?q? ?= lxc commands failing randomly:?(null)

Wed Jul 12 21:29:39 UTC 2017

Here it is:

lvl=info msg="Connecting to a remote LXD over HTTPs" t=2017-07-12T21:24:20+0000
etag= lvl=info method=GET msg="Sending request to LXD" t=2017-07-12T21:24:20+0000 url=https://lxd-server:8443/1.0
lvl=dbug msg="Got response struct from LXD" t=2017-07-12T21:24:20+0000
lvl=dbug msg="\n\t{\n\t\t\"config\": {\n\t\t\t\"core.https_address\": \"[::]\",\n\t\t\t\"core.trust_password\": true\n\t\t},\n\t\t\"api_extensions\": [\n\t\t\t\"storage_zfs_remove_snapshots\",\n\t\t\t\"container_host_shutdown_timeout\",\n\t\t\t\"container_syscall_filtering\",\n\t\t\t\"auth_pki\",\n\t\t\t\"container_last_used_at\",\n\t\t\t\"etag\",\n\t\t\t\"patch\",\n\t\t\t\"usb_devices\",\n\t\t\t\"https_allowed_credentials\",\n\t\t\t\"image_compression_algorithm\",\n\t\t\t\"directory_manipulation\",\n\t\t\t\"container_cpu_time\",\n\t\t\t\"storage_zfs_use_refquota\",\n\t\t\t\"storage_lvm_mount_options\",\n\t\t\t\"network\",\n\t\t\t\"profile_usedby\",\n\t\t\t\"container_push\",\n\t\t\t\"container_exec_recording\",\n\t\t\t\"certificate_update\",\n\t\t\t\"container_exec_signal_handling\",\n\t\t\t\"gpu_devices\",\n\t\t\t\"container_image_properties\",\n\t\t\t\"migration_progress\",\n\t\t\t\"id_map\",\n\t\t\t\"network_firewall_filtering\",\n\t\t\t\"network_routes\",\n\t\t\t\"storage\",\n\t\t\t\"file_delete\",\n\t\t\t\"file_append\",\n\t\t\t\"network_dhcp_expiry\",\n\t\t\t\"storage_lvm_vg_rename\",\n\t\t\t\"storage_lvm_thinpool_rename\",\n\t\t\t\"network_vlan\",\n\t\t\t\"image_create_aliases\",\n\t\t\t\"container_stateless_copy\",\n\t\t\t\"container_only_migration\",\n\t\t\t\"storage_zfs_clone_copy\",\n\t\t\t\"unix_device_rename\",\n\t\t\t\"storage_lvm_use_thinpool\",\n\t\t\t\"storage_rsync_bwlimit\",\n\t\t\t\"network_vxlan_interface\",\n\t\t\t\"storage_btrfs_mount_options\",\n\t\t\t\"entity_description\",\n\t\t\t\"image_force_refresh\",\n\t\t\t\"storage_lvm_lv_resizing\"\n\t\t],\n\t\t\"api_status\": \"stable\",\n\t\t\"api_version\": \"1.0\",\n\t\t\"auth\": \"trusted\",\n\t\t\"public\": false,\n\t\t\"environment\": {\n\t\t\t\"addresses\": [\n\t\t\t\t\"10.3.156.227:8443\",\n\t\t\t\t\"10.190.1.1:8443\",\n\t\t\t\t\"10.191.0.1:8443\",\n\t\t\t\t\"10.191.1.1:8443\",\n\t\t\t\t\"10.192.0.2:8443\",\n\t\t\t\t\"10.205.161.1:8443\",\n\t\t\t\t\"[fd11:02f:336e:d0e1::1]:8443\",\n\t\t\t\t\"10.190.0.1:8443\"\n\t\t\t],\n\t\t\t\"architectures\": [\n\t\t\t\t\"x86_64\",\n\t\t\t\t\"i686\"\n\t\t\t],\n\t\t\t\"certificate\": \"-----BEGIN CERTIFICATE-----...lots...of...certificate...here...-----END CERTIFICATE-----\\n\",\n\t\t\t\"certificate_fingerprint\": \"7381b19ae7563676857393701b4ec45313a3d84981bd665d12daf2de9a0a0dfa\",\n\t\t\t\"driver\": \"lxc\",\n\t\t\t\"driver_version\": \"2.0.8\",\n\t\t\t\"kernel\": \"Linux\",\n\t\t\t\"kernel_architecture\": \"x86_64\",\n\t\t\t\"kernel_version\": \"4.11.3-041103-generic\",\n\t\t\t\"server\": \"lxd\",\n\t\t\t\"server_pid\": 13675,\n\t\t\t\"server_version\": \"2.15\",\n\t\t\t\"storage\": \"btrfs\",\n\t\t\t\"storage_version\": \"4.10.2\"\n\t\t}\n\t}" t=2017-07-12T21:24:20+0000
lvl=dbug msg="Connected to the websocket" t=2017-07-12T21:24:20+0000
etag= lvl=info method=POST msg="Sending request to LXD" t=2017-07-12T21:24:20+0000 url=https://lxd-server:8443/1.0/containers/vpn-hz1/exec
lvl=dbug msg="\n\t{\n\t\t\"command\": [\n\t\t\t\"date\"\n\t\t],\n\t\t\"wait-for-websocket\": true,\n\t\t\"interactive\": false,\n\t\t\"environment\": {\n\t\t\t\"HOME\": \"/root\",\n\t\t\t\"TERM\": \"xterm-256color\",\n\t\t\t\"USER\": \"root\"\n\t\t},\n\t\t\"width\": 0,\n\t\t\"height\": 0,\n\t\t\"record-output\": false\n\t}" t=2017-07-12T21:24:20+0000
lvl=dbug msg="Got operation from LXD" t=2017-07-12T21:24:21+0000
lvl=dbug msg="\n\t{\n\t\t\"id\": \"c2148e6d-bb82-4d8f-aafa-0a2a5b7c9fa0\",\n\t\t\"class\": \"websocket\",\n\t\t\"created_at\": \"2017-07-12T21:24:21.01927786Z\",\n\t\t\"updated_at\": \"2017-07-12T21:24:21.01927786Z\",\n\t\t\"status\": \"Running\",\n\t\t\"status_code\": 103,\n\t\t\"resources\": {\n\t\t\t\"containers\": [\n\t\t\t\t\"/1.0/containers/vpn-hz1\"\n\t\t\t]\n\t\t},\n\t\t\"metadata\": {\n\t\t\t\"fds\": {\n\t\t\t\t\"0\": \"05693ff9baedf120f482bf513e13956b214536d47629e5b40c635d0037e8bd27\",\n\t\t\t\t\"1\": \"a59868283645f18a144eb671d15768681dc577b4d2ca1163cb4c65454ce6192e\",\n\t\t\t\t\"2\": \"707ef10a6025752a4e89dcf24c26ac1ef2962492660b71e4ab1e51fb073c4939\",\n\t\t\t\t\"control\": \"14b79db9b5ccd0224201c0e22cd35b2420a236ffb080992ec46c2488b36a3314\"\n\t\t\t}\n\t\t},\n\t\t\"may_cancel\": false,\n\t\t\"err\": \"\"\n\t}" t=2017-07-12T21:24:21+0000
lvl=dbug msg="Connected to the websocket" t=2017-07-12T21:24:21+0000
lvl=dbug msg="Connected to the websocket" t=2017-07-12T21:24:21+0000
lvl=dbug msg="Connected to the websocket" t=2017-07-12T21:24:21+0000
etag= lvl=info method=GET msg="Sending request to LXD" t=2017-07-12T21:24:21+0000 url=https://lxd-server:8443/1.0/operations/c2148e6d-bb82-4d8f-aafa-0a2a5b7c9fa0
lvl=dbug msg="got message barrier" t=2017-07-12T21:24:21+0000
lvl=dbug msg="got message barrier" t=2017-07-12T21:24:21+0000
error: not found

Wed Jul 12 21:24:21 UTC 2017 1

On Thursday, July 13, 2017 04:55 JST, Ivan Kurnosov <zerkms at zerkms.ru> wrote: 

> Please run it with `--debug` for more details.
> 
> On 13 July 2017 at 03:42, Tomasz Chmielewski <mangoo at wpkg.org> wrote:
> 
> > On Thursday, July 13, 2017 00:35 JST, "Tomasz Chmielewski" <
> > mangoo at wpkg.org> wrote:
> >
> > > On Wednesday, July 12, 2017 20:52 JST, "Tomasz Chmielewski" <
> > mangoo at wpkg.org> wrote:
> > >
> > > > It only fails with "error: not found" on the first, second or third
> > "lxc config" line.
> > > >
> > > > It started to fail in the last 2 weeks I think (lxd updates?) -
> > before, it was rock solid.
> > >
> > > Also "lxc exec" fails.
> > >
> > > Here is a reproducer:
> > >
> > > # lxc copy base-uni-web01 ztest
> > > # lxc start ztest ; while true ; do OUT=$(lxc exec ztest date ; echo $?)
> > ; echo $OUT ; done
> > > Wed Jul 12 15:32:43 UTC 2017 0
> > > Wed Jul 12 15:32:54 UTC 2017 0
> > > Wed Jul 12 15:32:54 UTC 2017 0
> > > Wed Jul 12 15:32:55 UTC 2017 0
> > > Wed Jul 12 15:32:55 UTC 2017 0
> > > Wed Jul 12 15:32:55 UTC 2017 0
> > > Wed Jul 12 15:32:56 UTC 2017 0
> > > Wed Jul 12 15:32:56 UTC 2017 0
> > > Wed Jul 12 15:32:57 UTC 2017 0
> > > Wed Jul 12 15:32:57 UTC 2017 0
> > > Wed Jul 12 15:32:57 UTC 2017 0
> > > Wed Jul 12 15:32:58 UTC 2017 0
> > > error: not found
> > > Wed Jul 12 15:32:58 UTC 2017 1
> > > Wed Jul 12 15:33:09 UTC 2017 0
> > > Wed Jul 12 15:33:09 UTC 2017 0
> > > Wed Jul 12 15:33:19 UTC 2017 0
> > > Wed Jul 12 15:33:25 UTC 2017 0
> > > Wed Jul 12 15:33:40 UTC 2017 0
> > > Wed Jul 12 15:33:46 UTC 2017 0
> > >
> > >
> > > It seems to be easier to reproduce if the host server is slightly
> > overloaded.
> >
> > Also - it only happens when lxc remote is https://.
> >
> > It doesn't happen when lxc remote is unix://
> >
> >
> > Tomasz Chmielewski
> > https://lxadm.com
> > _______________________________________________
> > lxc-users mailing list
> > lxc-users at lists.linuxcontainers.org
> > http://lists.linuxcontainers.org/listinfo/lxc-users
> 
> 
> 
> 
> -- 
> With best regards, Ivan Kurnosov

-- 
Tomasz Chmielewski