Re: [Qemu-devel] [PATCH] virtio: Implement userspace forwarding for host notifiers

[Christian Borntraeger]

Looks like this would be ok for virtio-ccw (as it does not call 
virtio_queue_set_host_notifier_forwarding)
Question is might something like this for virtio-ccw useful as well?

We cannot use memory_region, though, we would need to handle that in our
diagnose code.

Conny, FYI.




On 11/19/2015 10:50 AM, Pavel Fedin wrote:
> If you happen to have a kernel with ioeventfd support enabled, but
> missing support for them in KVM, and you attempt to enable vhost by
> setting vhost=on, qemu aborts with error:
> 
> kvm_mem_ioeventfd_add: error adding ioeventfd: Function not implemented
> 
> This patch adds a mechanism which allows to emulate KVM binding by
> triggering the related notifiers via the userspace. The first time the
> emulation is used, a warning is displayed, so that the user knows about
> potential performance impact:
> 
> 2015-11-19T09:35:16.618380Z qemu-system-aarch64: KVM does not support eventfd 
> binding, using userspace event forwarding (slow)
> 
> This problem can be observed with libvirt, which checks for /dev/vhost-net
> availability and just inserts "vhost=on" automatically in this case; on an
> ARM64 system using stock kernel 3.18.0 with CONFIG_IOEVENTFD enabled in
> expert settings.
> 
> Signed-off-by: Pavel Fedin <address@hidden>
> ---
>  hw/virtio/virtio-mmio.c    | 15 +++++++++---
>  hw/virtio/virtio-pci.c     | 61 
> ++++++++++++++++++++++++++--------------------
>  hw/virtio/virtio.c         | 24 +++++++++++++++++-
>  include/hw/virtio/virtio.h |  1 +
>  4 files changed, 69 insertions(+), 32 deletions(-)
> 
> diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
> index 16621fa..69d4cbc 100644
> --- a/hw/virtio/virtio-mmio.c
> +++ b/hw/virtio/virtio-mmio.c
> @@ -110,11 +110,18 @@ static int 
> virtio_mmio_set_host_notifier_internal(VirtIOMMIOProxy *proxy,
>              return r;
>          }
>          virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
> -        memory_region_add_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
> -                                  true, n, notifier);
> +
> +        if (kvm_eventfds_enabled()) {
> +            memory_region_add_eventfd(&proxy->iomem, 
> VIRTIO_MMIO_QUEUENOTIFY, 4,
> +                                      true, n, notifier);
> +        } else if (!set_handler) {
> +            virtio_queue_set_host_notifier_forwarding(vq);
> +        }
>      } else {
> -        memory_region_del_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
> -                                  true, n, notifier);
> +        if (kvm_eventfds_enabled()) {
> +            memory_region_del_eventfd(&proxy->iomem, 
> VIRTIO_MMIO_QUEUENOTIFY, 4,
> +                                      true, n, notifier);
> +        }
>          virtio_queue_set_host_notifier_fd_handler(vq, false, false);
>          event_notifier_cleanup(notifier);
>      }
> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> index 96be4fd..b27a630 100644
> --- a/hw/virtio/virtio-pci.c
> +++ b/hw/virtio/virtio-pci.c
> @@ -293,41 +293,48 @@ static int 
> virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
>              return r;
>          }
>          virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
> -        if (modern) {
> -            if (fast_mmio) {
> -                memory_region_add_eventfd(modern_mr, modern_addr, 0,
> -                                          false, n, notifier);
> -            } else {
> -                memory_region_add_eventfd(modern_mr, modern_addr, 2,
> -                                          false, n, notifier);
> +
> +        if (kvm_eventfds_enabled()) {
> +            if (modern) {
> +                if (fast_mmio) {
> +                    memory_region_add_eventfd(modern_mr, modern_addr, 0,
> +                                              false, n, notifier);
> +                } else {
> +                    memory_region_add_eventfd(modern_mr, modern_addr, 2,
> +                                              false, n, notifier);
> +                }
> +                if (modern_pio) {
> +                    memory_region_add_eventfd(modern_notify_mr, 0, 2,
> +                                                  true, n, notifier);
> +                }
>              }
> -            if (modern_pio) {
> -                memory_region_add_eventfd(modern_notify_mr, 0, 2,
> -                                              true, n, notifier);
> +            if (legacy) {
> +                memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
> +                                          true, n, notifier);
>              }
> -        }
> -        if (legacy) {
> -            memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
> -                                      true, n, notifier);
> +        } else if (!set_handler) {
> +            virtio_queue_set_host_notifier_forwarding(vq);
>          }
>      } else {
> -        if (modern) {
> -            if (fast_mmio) {
> -                memory_region_del_eventfd(modern_mr, modern_addr, 0,
> -                                          false, n, notifier);
> -            } else {
> -                memory_region_del_eventfd(modern_mr, modern_addr, 2,
> -                                          false, n, notifier);
> +        if (kvm_eventfds_enabled()) {
> +            if (modern) {
> +                if (fast_mmio) {
> +                    memory_region_del_eventfd(modern_mr, modern_addr, 0,
> +                                              false, n, notifier);
> +                } else {
> +                    memory_region_del_eventfd(modern_mr, modern_addr, 2,
> +                                              false, n, notifier);
> +                }
> +                if (modern_pio) {
> +                    memory_region_del_eventfd(modern_notify_mr, 0, 2,
> +                                              true, n, notifier);
> +                }
>              }
> -            if (modern_pio) {
> -                memory_region_del_eventfd(modern_notify_mr, 0, 2,
> +            if (legacy) {
> +                memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
>                                            true, n, notifier);
>              }
>          }
> -        if (legacy) {
> -            memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
> -                                      true, n, notifier);
> -        }
>          virtio_queue_set_host_notifier_fd_handler(vq, false, false);
>          event_notifier_cleanup(notifier);
>      }
> diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
> index 1edef59..6fe268f 100644
> --- a/hw/virtio/virtio.c
> +++ b/hw/virtio/virtio.c
> @@ -89,6 +89,7 @@ struct VirtQueue
>      VirtIODevice *vdev;
>      EventNotifier guest_notifier;
>      EventNotifier host_notifier;
> +    bool forward_host_notifier;
>      QLIST_ENTRY(VirtQueue) node;
>  };
> 
> @@ -969,7 +970,13 @@ void virtio_queue_notify_vq(VirtQueue *vq)
> 
>  void virtio_queue_notify(VirtIODevice *vdev, int n)
>  {
> -    virtio_queue_notify_vq(&vdev->vq[n]);
> +    VirtQueue *vq = &vdev->vq[n];
> +
> +    if (vq->forward_host_notifier) {
> +        event_notifier_set(&vq->host_notifier);
> +    } else {
> +        virtio_queue_notify_vq(&vdev->vq[n]);
> +    }
>  }
> 
>  uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
> @@ -1715,6 +1722,21 @@ void 
> virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
>          /* Test and clear notifier before after disabling event,
>           * in case poll callback didn't have time to run. */
>          virtio_queue_host_notifier_read(&vq->host_notifier);
> +
> +        vq->forward_host_notifier = false;
> +    }
> +}
> +
> +static bool forwarding_warning;
> +
> +void virtio_queue_set_host_notifier_forwarding(VirtQueue *vq)
> +{
> +    vq->forward_host_notifier = true;
> +
> +    if (!forwarding_warning) {
> +        forwarding_warning = true;
> +        error_report("KVM does not support eventfd binding, "
> +                     "using userspace event forwarding (slow)");
>      }
>  }
> 
> diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
> index 205fadf..f288ccb 100644
> --- a/include/hw/virtio/virtio.h
> +++ b/include/hw/virtio/virtio.h
> @@ -245,6 +245,7 @@ void virtio_queue_set_guest_notifier_fd_handler(VirtQueue 
> *vq, bool assign,
>  EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
>  void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
>                                                 bool set_handler);
> +void virtio_queue_set_host_notifier_forwarding(VirtQueue *vq);
>  void virtio_queue_notify_vq(VirtQueue *vq);
>  void virtio_irq(VirtQueue *vq);
>  VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector);
>