[PATCH V1 8/8] vfio-pci: recover from unmap-all-vaddr failure

[Steve Sistare]

If there are multiple containers and unmap-all fails for some container, we
need to remap vaddr for the other containers for which unmap-all succeeded.
Recover by walking all address ranges of all containers to restore the vaddr
for each.  Do so by invoking the vfio listener callback, and passing a new
"remap" flag that tells it to restore a mapping without re-allocating new
userland data structures.

Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
---
 hw/vfio/common.c                      | 45 ++++++++++++++++++++++++++++++++---
 hw/vfio/cpr-legacy.c                  | 44 ++++++++++++++++++++++++++++++++++
 include/hw/vfio/vfio-common.h         |  4 +++-
 include/hw/vfio/vfio-container-base.h |  1 +
 4 files changed, 90 insertions(+), 4 deletions(-)

diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 5c7baad..da2e0ec 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -586,11 +586,12 @@ static void vfio_listener_region_add(MemoryListener 
*listener,
 {
     VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
                                                  listener);
-    vfio_container_region_add(bcontainer, section);
+    vfio_container_region_add(bcontainer, section, false);
 }
 
 void vfio_container_region_add(VFIOContainerBase *bcontainer,
-                               MemoryRegionSection *section)
+                               MemoryRegionSection *section,
+                               bool remap)
 {
     hwaddr iova, end;
     Int128 llend, llsize;
@@ -626,6 +627,30 @@ void vfio_container_region_add(VFIOContainerBase 
*bcontainer,
         int iommu_idx;
 
         trace_vfio_listener_region_add_iommu(iova, end);
+
+        /*
+         * If remap, then VFIO_DMA_UNMAP_FLAG_VADDR has been called, and we
+         * want to remap the vaddr.  vfio_container_region_add was already
+         * called in the past, so the giommu already exists.  Find it and
+         * replay it, which calls vfio_dma_map further down the stack.
+         */
+
+        if (remap) {
+            hwaddr as_offset = section->offset_within_address_space;
+            hwaddr iommu_offset = as_offset - section->offset_within_region;
+
+            QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) {
+                if (giommu->iommu_mr == iommu_mr &&
+                    giommu->iommu_offset == iommu_offset) {
+                    memory_region_iommu_replay(giommu->iommu_mr, &giommu->n);
+                    return;
+                }
+            }
+            error_report("Container cannot find iommu region %s offset %lx",
+                memory_region_name(section->mr), iommu_offset);
+            goto fail;
+        }
+
         /*
          * FIXME: For VFIO iommu types which have KVM acceleration to
          * avoid bouncing all map/unmaps through qemu this way, this
@@ -676,7 +701,21 @@ void vfio_container_region_add(VFIOContainerBase 
*bcontainer,
      * about changes.
      */
     if (memory_region_has_ram_discard_manager(section->mr)) {
-        vfio_register_ram_discard_listener(bcontainer, section);
+        /*
+         * If remap, then VFIO_DMA_UNMAP_FLAG_VADDR has been called, and we
+         * want to remap the vaddr.  vfio_container_region_add was already
+         * called in the past, so the ram discard listener already exists.
+         * Call its populate function directly, which calls vfio_dma_map.
+         */
+        if (remap)  {
+            VFIORamDiscardListener *vrdl =
+                vfio_find_ram_discard_listener(bcontainer, section);
+            if (vrdl->listener.notify_populate(&vrdl->listener, section)) {
+                error_report("listener.notify_populate failed");
+            }
+        } else {
+            vfio_register_ram_discard_listener(bcontainer, section);
+        }
         return;
     }
 
diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c
index bc51ebe..c4b95a8 100644
--- a/hw/vfio/cpr-legacy.c
+++ b/hw/vfio/cpr-legacy.c
@@ -29,9 +29,18 @@ static bool vfio_dma_unmap_vaddr_all(VFIOContainer 
*container, Error **errp)
         error_setg_errno(errp, errno, "vfio_dma_unmap_vaddr_all");
         return false;
     }
+    container->vaddr_unmapped = true;
     return true;
 }
 
+static void vfio_region_remap(MemoryListener *listener,
+                              MemoryRegionSection *section)
+{
+    VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+                                                 remap_listener);
+    vfio_container_region_add(bcontainer, section, true);
+}
+
 static bool vfio_can_cpr_exec(VFIOContainer *container, Error **errp)
 {
     if (!ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UPDATE_VADDR)) {
@@ -95,6 +104,37 @@ static const VMStateDescription vfio_container_vmstate = {
     }
 };
 
+static int vfio_cpr_fail_notifier(NotifierWithReturn *notifier,
+                                  MigrationEvent *e, Error **errp)
+{
+    VFIOContainer *container =
+        container_of(notifier, VFIOContainer, cpr_exec_notifier);
+    VFIOContainerBase *bcontainer = &container->bcontainer;
+
+    if (e->type != MIG_EVENT_PRECOPY_FAILED) {
+        return 0;
+    }
+
+    if (container->vaddr_unmapped) {
+        /*
+         * Force a call to vfio_region_remap for each mapped section by
+         * temporarily registering a listener, which calls vfio_dma_map
+         * further down the stack. Set reused so vfio_dma_map restores vaddr.
+         */
+        bcontainer->reused = true;
+        bcontainer->remap_listener = (MemoryListener) {
+            .name = "vfio recover",
+            .region_add = vfio_region_remap
+        };
+        memory_listener_register(&bcontainer->remap_listener,
+                                 bcontainer->space->as);
+        memory_listener_unregister(&bcontainer->remap_listener);
+        bcontainer->reused = false;
+        container->vaddr_unmapped = false;
+    }
+    return 0;
+}
+
 bool vfio_legacy_cpr_register_container(VFIOContainerBase *bcontainer,
                                         Error **errp)
 {
@@ -107,6 +147,9 @@ bool vfio_legacy_cpr_register_container(VFIOContainerBase 
*bcontainer,
 
     vmstate_register(NULL, -1, &vfio_container_vmstate, container);
 
+    migration_add_notifier_mode(&container->cpr_exec_notifier,
+                                vfio_cpr_fail_notifier,
+                                MIG_MODE_CPR_EXEC);
     return true;
 }
 
@@ -115,4 +158,5 @@ void vfio_legacy_cpr_unregister_container(VFIOContainerBase 
*bcontainer)
     VFIOContainer *container = VFIO_CONTAINER(bcontainer);
 
     vmstate_unregister(NULL, &vfio_container_vmstate, container);
+    migration_remove_notifier(&container->cpr_exec_notifier);
 }
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 7c4283b..1902c8f 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -81,6 +81,8 @@ typedef struct VFIOContainer {
     VFIOContainerBase bcontainer;
     int fd; /* /dev/vfio/vfio, empowered by the attached groups */
     unsigned iommu_type;
+    NotifierWithReturn cpr_exec_notifier;
+    bool vaddr_unmapped;
     QLIST_HEAD(, VFIOGroup) group_list;
 } VFIOContainer;
 
@@ -292,7 +294,7 @@ int vfio_get_dirty_bitmap(const VFIOContainerBase 
*bcontainer, uint64_t iova,
                           uint64_t size, ram_addr_t ram_addr, Error **errp);
 
 void vfio_container_region_add(VFIOContainerBase *bcontainer,
-                               MemoryRegionSection *section);
+                               MemoryRegionSection *section, bool remap);
 void vfio_listener_register(VFIOContainerBase *bcontainer);
 
 /* Returns 0 on success, or a negative errno. */
diff --git a/include/hw/vfio/vfio-container-base.h 
b/include/hw/vfio/vfio-container-base.h
index 82ccf0c..3d30365 100644
--- a/include/hw/vfio/vfio-container-base.h
+++ b/include/hw/vfio/vfio-container-base.h
@@ -37,6 +37,7 @@ typedef struct VFIOContainerBase {
     Object parent;
     VFIOAddressSpace *space;
     MemoryListener listener;
+    MemoryListener remap_listener;
     Error *error;
     bool initialized;
     bool reused;
-- 
1.8.3.1