Bug 1198244

FWIW, here's what gnome-boxes calls the Troubleshooting Log for this VM:

Broker URL: qemu+unix:///session
Domain: fedora21-2
UUID: b5de28de-1f83-44dd-bce3-402f780dcc0f
Persistent: yes
Cpu time: 0
Memory: 0 KiB
Max memory: 11987920 KiB
CPUs: 4
State: GVIR_DOMAIN_STATE_SHUTOFF

Domain config:
------------------------------------------------------------
<domain type="kvm">
  <name>fedora21-2</name>
  <uuid>b5de28de-1f83-44dd-bce3-402f780dcc0f</uuid>
  <title>Fedora 21</title>
  <metadata>
    <boxes:gnome-boxes xmlns:boxes="http://live.gnome.org/Boxes/">
      <os-state>installed</os-state>
      <os-id>http://fedoraproject.org/fedora/21</os-id>
      <media-id>http://fedoraproject.org/fedora/21:3</media-id>
      <media>/mnt/work/Archive/Fedora-Live-Workstation-x86_64-21/Fedora-Live-Workstation-x86_64-21-5.iso</media>
    </boxes:gnome-boxes>
  </metadata>
  <memory unit="KiB">1048576</memory>
  <currentMemory unit="KiB">1048576</currentMemory>
  <vcpu placement="static">4</vcpu>
  <os>
    <type arch="x86_64" machine="pc-i440fx-2.1">hvm</type>
    <boot dev="hd"/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode="custom" match="exact">
    <model fallback="allow">SandyBridge</model>
    <topology sockets="1" cores="2" threads="2"/>
  </cpu>
  <clock offset="utc">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-kvm</emulator>
    <disk type="file" device="disk">
      <driver name="qemu" type="qcow2" cache="none"/>
      <source file="/home/twaugh/.local/share/gnome-boxes/images/fedora21-2"/>
      <target dev="vda" bus="virtio"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x07" function="0x0"/>
    </disk>
    <disk type="file" device="cdrom">
      <driver name="qemu" type="raw"/>
      <source file="/mnt/work/Archive/Fedora-Live-Workstation-x86_64-21/Fedora-Live-Workstation-x86_64-21-5.iso" startupPolicy="optional"/>
      <target dev="hdc" bus="ide"/>
      <readonly/>
      <address type="drive" controller="0" bus="1" target="0" unit="0"/>
    </disk>
    <controller type="usb" index="0" model="ich9-ehci1">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x7"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci1">
      <master startport="0"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x0" multifunction="on"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci2">
      <master startport="2"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x1"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci3">
      <master startport="4"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pci-root"/>
    <controller type="ide" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
    </controller>
    <controller type="virtio-serial" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x06" function="0x0"/>
    </controller>
    <controller type="ccid" index="0"/>
    <interface type="bridge">
      <mac address="52:54:00:5f:2d:3e"/>
      <source bridge="virbr0"/>
      <model type="rtl8139"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0"/>
    </interface>
    <smartcard mode="passthrough" type="spicevmc">
      <address type="ccid" controller="0" slot="0"/>
    </smartcard>
    <serial type="pty">
      <target port="0"/>
    </serial>
    <console type="pty">
      <target type="serial" port="0"/>
    </console>
    <channel type="spicevmc">
      <target type="virtio" name="com.redhat.spice.0"/>
      <address type="virtio-serial" controller="0" bus="0" port="1"/>
    </channel>
    <input type="tablet" bus="usb"/>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <graphics type="spice" autoport="yes" listen="127.0.0.1">
      <listen type="address" address="127.0.0.1"/>
      <image compression="off"/>
    </graphics>
    <sound model="ac97">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x04" function="0x0"/>
    </sound>
    <video>
      <model type="qxl" ram="65536" vram="65536" vgamem="16384" heads="1"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0"/>
    </video>
    <redirdev bus="usb" type="spicevmc">
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
    </redirdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x08" function="0x0"/>
    </memballoon>
  </devices>
  <seclabel type="dynamic" model="selinux" relabel="yes"/>
</domain>
------------------------------------------------------------

QEMU log:
------------------------------------------------------------
2015-03-04 16:19:40.405+0000: shutting down
------------------------------------------------------------

I get the same error when trying to set up a new virtual machine in gnome-boxes as well.

The error is from virNetDevGetLinkInfo(), which tries to read the status of an network device from /sys/class/net/$name/operstate. There are multiple oddities here:

1) That function is only called from the node device driver, or from the udev backend for the interface driver. I can only hope that the udev backend of the interface driver isn't being called, as it should never be built on Fedora (the netcf backend should always be preferred).

2) If (1) is correct (i.e. it's the node device driver that is being called), then it would require something *outside of libvirt* to be calling virNodeDeviceGetXMLDesc() for the tap device being created for the guest (because libvirt itself certainly never does that). Normally that API is only used for host devices, not for tap devices created for a guest.

3) the file in question "operstate" is normally world-readable, and at the time it is being read (if the NetworkManager log messages are to be believed) the device (and thus the sysfs node) actually does exist.

My only rawhide image is a virtual machine that is currently broken. I'm trying to update it so that I can do a local build of rawhide libvirt and see what build options were used. That should hopefully rule out (1). Then we're left with (2) and (3).

btw, I'm talking about this error:

  unable to read: /sys/class/net/tap0/operstate: No such file or directory

not the one in the summary.

Created attachment 999667 [details]
libvirtd.log

libvirtd logging where the problem is happening for me.

Note I'm using non-root (qemu:///session).

It's not SELinux, as the problem is reproducible even when
SELinux == Permissive.

I don't understand why the 'operstate' file is being fingered here.

I believe the code that's failing is this:

int virNetDevGetIndex(const char *ifname, int *ifindex)
{
    int ret = -1;
    struct ifreq ifreq;
    int fd = socket(VIR_NETDEV_FAMILY, SOCK_DGRAM, 0);

    if (fd < 0) {
        virReportSystemError(errno, "%s",
                             _("Unable to open control socket"));
        return -1;
    }

Since VIR_NETDEV_FAMILY == AF_PACKET, that comes down to
a failure of the simple system call `socket (AF_PACKET, SOCK_DGRAM, 0)'

Now as far as I understand, only processes with CAP_NET_RAW are
able to make this system call.  libvirtd is not setuid and doesn't
have any capabilities (according to 'getcap').  No idea if it should
have that.  I'm unclear why libvirtd thinks it can make this
system call, and why it usually works.

Here's a simple reproducer (at least, when the machine gets stuck in
this state, which is not always).  As NON-ROOT do:

$ for i in `seq 1 5`; do guestfish -a /dev/null --network run >&/tmp/log$i & done
[2] 22498
[3] 22499
[4] 22500
[5] 22501
[6] 22502

Wait a few seconds, and hit return:

$
[2]   Exit 1                  guestfish -a /dev/null --network run &> /tmp/log$i
[3]   Exit 1                  guestfish -a /dev/null --network run &> /tmp/log$i
[4]   Exit 1                  guestfish -a /dev/null --network run &> /tmp/log$i
[5]-  Exit 1                  guestfish -a /dev/null --network run &> /tmp/log$i
[6]+  Exit 1                  guestfish -a /dev/null --network run &> /tmp/log$i

Examine the /tmp/log* files and you should see:

Original error from libvirt: Unable to open control socket: Operation not permitted [code=38 domain=0]

Notice that removing the --network flag makes it work, because I believe
this bug is caused when libvirtd tries to construct the qemu command
line and has trouble constructing the qemu network device options.

Stack trace on failure:

#0  virNetDevGetIndex (ifname=0x7f98e0006fb0 "tap0", 
    ifindex=ifindex@entry=0x7f98ffa4aef0) at util/virnetdev.c:875
#1  0x00007f98f77d82de in qemuBuildInterfaceCommandLine (
    nicindexes=0x7f98ffa4b1b8, nnicindexes=0x7f98ffa4b1b0, standalone=false, 
    vmop=VIR_NETDEV_VPORT_PROFILE_OP_CREATE, bootindex=0, vlan=-1, 
    qemuCaps=0x7f98e00033c0, net=0x7f98e000c710, def=0x7f98e000bac0, 
    driver=0x7f98f0158d90, cmd=0x7f98e00085a0) at qemu/qemu_command.c:7845
#2  qemuBuildCommandLine (conn=conn@entry=0x7f98d8000dd0, 
    driver=driver@entry=0x7f98f0158d90, def=0x7f98e000bac0, 
    monitor_chr=<optimized out>, monitor_json=<optimized out>, 
    qemuCaps=0x7f98e00033c0, migrateFrom=0x0, migrateFd=-1, snapshot=0x0, 
    vmop=VIR_NETDEV_VPORT_PROFILE_OP_CREATE, 
    callbacks=0x7f98f7ab57a8 <buildCommandLineCallbacks>, standalone=false, 
    enableFips=false, nodeset=0x0, nnicindexes=0x7f98ffa4b1b0, 
    nicindexes=0x7f98ffa4b1b8) at qemu/qemu_command.c:9321
#3  0x00007f98f780300d in qemuProcessStart (conn=conn@entry=0x7f98d8000dd0, 
    driver=driver@entry=0x7f98f0158d90, vm=0x7f98e0002ff0, 
    asyncJob=asyncJob@entry=0, migrateFrom=migrateFrom@entry=0x0, 
    stdin_fd=stdin_fd@entry=-1, stdin_path=0x0, snapshot=0x0, 
    vmop=VIR_NETDEV_VPORT_PROFILE_OP_CREATE, flags=5)
    at qemu/qemu_process.c:4706
#4  0x00007f98f784afd5 in qemuDomainCreateXML (conn=0x7f98d8000dd0, 
    xml=<optimized out>, flags=2) at qemu/qemu_driver.c:1723
#5  0x00007f990b6133db in virDomainCreateXML (conn=0x7f98d8000dd0, 
    xmlDesc=0x7f98e0000990 "<?xml version=\"1.0\"?>\n<domain type=\"kvm\" xmlns:qemu=\"http://libvirt.org/schemas/domain/qemu/1.0\">\n  <name>guestfs-csp4a2d0nz7wdvby</name>\n  <memory unit=\"MiB\">500</memory>\n  <currentMemory unit=\"MiB\">"..., flags=2) at libvirt-domain.c:180
#6  0x00007f990c10a52f in remoteDispatchDomainCreateXML (
    server=<optimized out>, msg=<optimized out>, ret=0x7f98e0000960, 
    args=0x7f98e00008e0, rerr=0x7f98ffa4bc30, client=0x7f990d942fd0)
    at remote_dispatch.h:3303
#7  remoteDispatchDomainCreateXMLHelper (server=<optimized out>, 
    client=0x7f990d942fd0, msg=<optimized out>, rerr=0x7f98ffa4bc30, 
    args=0x7f98e00008e0, ret=0x7f98e0000960) at remote_dispatch.h:3283
#8  0x00007f990b684379 in virNetServerProgramDispatchCall (msg=0x7f990d941e40, 
    client=0x7f990d942fd0, server=0x7f990d914bb0, prog=0x7f990d93efb0)
    at rpc/virnetserverprogram.c:437
#9  virNetServerProgramDispatch (prog=0x7f990d93efb0, 
    server=server@entry=0x7f990d914bb0, client=0x7f990d942fd0, 
    msg=0x7f990d941e40) at rpc/virnetserverprogram.c:307
#10 0x00007f990c117a18 in virNetServerProcessMsg (msg=<optimized out>, 
    prog=<optimized out>, client=<optimized out>, srv=0x7f990d914bb0)
    at rpc/virnetserver.c:172
#11 virNetServerHandleJob (jobOpaque=<optimized out>, opaque=0x7f990d914bb0)
    at rpc/virnetserver.c:193
#12 0x00007f990b582116 in virThreadPoolWorker (
    opaque=opaque@entry=0x7f990d8ffb50) at util/virthreadpool.c:144
#13 0x00007f990b581abe in virThreadHelper (data=<optimized out>)
    at util/virthread.c:197
#14 0x00007f9907cc352a in start_thread (arg=0x7f98ffa4c700)
    at pthread_create.c:310
#15 0x00007f99079ff79d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

I would say the problematic commit is:

commit 4bbe1029f2fb6cd1c102794951a944c62fdbd0e6 (tag: v1.2.13-rc2)
Author: Laine Stump <laine>
Date:   Fri Feb 20 14:52:37 2015 -0500

    qemu: fix ifindex array reported to systemd
    
    Commit f7afeddc added code to report to systemd an array of interface
    indexes for all tap devices used by a guest. Unfortunately it not only
    didn't add code to report the ifindexes for macvtap interfaces
    (interface type='direct') or the tap devices used by type='ethernet',
    it ended up sending "-1" as the ifindex for each macvtap or hostdev
    interface. This resulted in a failure to start any domain that had a
    macvtap or hostdev interface (or actually any type other than
    "network" or "bridge").
    
    This patch does the following with the nicindexes array:
    
    1) Modify qemuBuildInterfaceCommandLine() to only fill in the
    nicindexes array if given a non-NULL pointer to an array (and modifies
    the test jig calls to the function to send NULL). This is because
    there are tests in the test suite that have type='ethernet' and still
    have an ifname specified, but that device of course doesn't actually
    exist on the test system, so attempts to call virNetDevGetIndex() will
    fail.
    
    2) Even then, only add an entry to the nicindexes array for
    appropriate types, and to do so for all appropriate types ("network",
    "bridge", and "direct"), but only if the ifname is known (since that
    is required to call virNetDevGetIndex().

because it makes the assumption that libvirtd is running as root.
When using qemu:///session, libvirtd does not run as root.

Note that libguestfs no longer uses SLIRP when using the libvirt
backend, even when running as non-root (bug 1148012).

Sigh. Fixed one bug (nothing with macvtap or hostdev interfaces would start) only to replace it with another.

I just posted a patch to libvir-list that solves the problem with calling virNetDevGetIndex() in session mode libvirt:

  https://www.redhat.com/archives/libvir-list/2015-March/msg00432.html

Feel free to push it (also to the 1.2.13-maint branch) if you need it there immediately, otherwise I'll do it in a few hours when I wake up.

BTW, the attempt to read operstate of the tap device appears to be coming from some udev event callback that is set by the nodedevice driver. My guess is that the event is sent by udev when the qemu bridge helper creates the tap device, but by the time libvirtd can respond to the event, the error has already happened and the device has been deleted.

Just tested with this patch locally, using Rich's reproducer. Fixes the
issue.


Test
----

Before applying this patch, tested with version
libvirt-daemon-kvm-1.2.13-1.fc23.x86_64:

  $ whoami
  kashyapc

  $ guestfish -a /dev/null --network run
  libguestfs: error: could not create appliance through libvirt.
  . . .
  Original error from libvirt: Unable to open control socket: Operation not permitted [code=38 domain=0]


After applying this patch (applied on current git):

    $ git describe
    v1.2.13-100-gb7d027b

    $ guestfish -a /dev/null --network run
    $ echo $?
    0

libvirt-1.2.13-2.fc22 has been submitted as an update for Fedora 22.
https://admin.fedoraproject.org/updates/libvirt-1.2.13-2.fc22

*** Bug 1200521 has been marked as a duplicate of this bug. ***

CCing, and testing.  Will

Package libvirt-1.2.13-2.fc22:
* should fix your issue,
* was pushed to the Fedora 22 testing repository,
* should be available at your local mirror within two days.
Update it with:
# su -c 'yum update --enablerepo=updates-testing libvirt-1.2.13-2.fc22'
as soon as you are able to.
Please go to the following url:
https://admin.fedoraproject.org/updates/FEDORA-2015-3739/libvirt-1.2.13-2.fc22
then log in and leave karma (feedback).

The new version of libvirt fixed things for me.

However, I think the command should be

su -c 'yum install --enablerepo=updates-testing libvirt-1.2.13-2.fc22'

The other command did not work for me (it found the package but did not consider it an update).

libvirt-1.2.13-2.fc22 has been pushed to the Fedora 22 stable repository.  If problems still persist, please make note of it in this bug report.

Hi, I installed Fedora 22 and Boxes didn't works (this was the error messagge):

[mtblock@balamb ~]$ gnome-boxes 

(gnome-boxes:5169): Boxes-WARNING **: libvirt-broker.vala:98: Unable to open qemu+unix:///session: Cannot write data: Transport endpoint is not connected


After I read this page and I installed libvirt: this is the output


[mtblock@balamb ~]$ sudo dnf install libvirt
[sudo] password for mtblock: 
Last metadata expiration check performed 1:32:40 ago on Tue Oct 20 12:32:45 2015.
Dependencies resolved.
================================================================================
 Package                          Arch     Version              Repository
                                                                           Size
================================================================================
Installing:
 libvirt                          x86_64   1.2.13.1-3.fc22      updates    44 k
 libvirt-daemon-config-nwfilter   x86_64   1.2.13.1-3.fc22      updates    48 k
 libvirt-daemon-driver-libxl      x86_64   1.2.13.1-3.fc22      updates   152 k
 libvirt-daemon-driver-lxc        x86_64   1.2.13.1-3.fc22      updates   683 k
 libvirt-daemon-driver-uml        x86_64   1.2.13.1-3.fc22      updates    97 k
 libvirt-daemon-driver-vbox       x86_64   1.2.13.1-3.fc22      updates   196 k
 libvirt-daemon-driver-xen        x86_64   1.2.13.1-3.fc22      updates   134 k

Transaction Summary
================================================================================
Install  7 Packages

Total download size: 1.3 M
Installed size: 2.9 M
Is this ok [y/N]: y
Downloading Packages:
(1/7): libvirt-daemon-config-nwfilter-1.2.13.1-  30 kB/s |  48 kB     00:01    
(2/7): libvirt-1.2.13.1-3.fc22.x86_64.rpm        27 kB/s |  44 kB     00:01    
(3/7): libvirt-daemon-driver-libxl-1.2.13.1-3.f  88 kB/s | 152 kB     00:01    
(4/7): libvirt-daemon-driver-uml-1.2.13.1-3.fc2 220 kB/s |  97 kB     00:00    
(5/7): libvirt-daemon-driver-vbox-1.2.13.1-3.fc 346 kB/s | 196 kB     00:00    
(6/7): libvirt-daemon-driver-lxc-1.2.13.1-3.fc2 661 kB/s | 683 kB     00:01    
(7/7): libvirt-daemon-driver-xen-1.2.13.1-3.fc2 230 kB/s | 134 kB     00:00    
--------------------------------------------------------------------------------
Total                                           334 kB/s | 1.3 MB     00:04     
Running transaction check
Transaction check succeeded.
Running transaction test
Transaction test succeeded.
Running transaction
  Installing  : libvirt-daemon-driver-xen-1.2.13.1-3.fc22.x86_64            1/7 
  Installing  : libvirt-daemon-driver-vbox-1.2.13.1-3.fc22.x86_64           2/7 
  Installing  : libvirt-daemon-driver-uml-1.2.13.1-3.fc22.x86_64            3/7 
  Installing  : libvirt-daemon-driver-lxc-1.2.13.1-3.fc22.x86_64            4/7 
  Installing  : libvirt-daemon-driver-libxl-1.2.13.1-3.fc22.x86_64          5/7 
  Installing  : libvirt-daemon-config-nwfilter-1.2.13.1-3.fc22.x86_64       6/7 
  Installing  : libvirt-1.2.13.1-3.fc22.x86_64                              7/7 
  Verifying   : libvirt-1.2.13.1-3.fc22.x86_64                              1/7 
  Verifying   : libvirt-daemon-config-nwfilter-1.2.13.1-3.fc22.x86_64       2/7 
  Verifying   : libvirt-daemon-driver-libxl-1.2.13.1-3.fc22.x86_64          3/7 
  Verifying   : libvirt-daemon-driver-lxc-1.2.13.1-3.fc22.x86_64            4/7 
  Verifying   : libvirt-daemon-driver-uml-1.2.13.1-3.fc22.x86_64            5/7 
  Verifying   : libvirt-daemon-driver-vbox-1.2.13.1-3.fc22.x86_64           6/7 
  Verifying   : libvirt-daemon-driver-xen-1.2.13.1-3.fc22.x86_64            7/7 

Installed:
  libvirt.x86_64 1.2.13.1-3.fc22                                                
  libvirt-daemon-config-nwfilter.x86_64 1.2.13.1-3.fc22                         
  libvirt-daemon-driver-libxl.x86_64 1.2.13.1-3.fc22                            
  libvirt-daemon-driver-lxc.x86_64 1.2.13.1-3.fc22                              
  libvirt-daemon-driver-uml.x86_64 1.2.13.1-3.fc22                              
  libvirt-daemon-driver-vbox.x86_64 1.2.13.1-3.fc22                             
  libvirt-daemon-driver-xen.x86_64 1.2.13.1-3.fc22                              

Complete!



And now, Boxes works perfectly