[Qemu-ppc] [PATCH RFC 3/4] sPAPR: Support multiple IOMMU groups in PHB for EEH operations

[Gavin Shan]

Currently, EEH works based on the assumption that every VFIO PHB
has only one attached IOMMU group, which won't be true any more.
It means every PHB would have multiple attached IOMMU groups. In
order to apply the request EEH operation to the specified IOMMU
group (PE), the changes to host kernel is required. Also, QEMU
need provide the IOMMU group ID to specify the precise target for
the EEH operations.

This reuses the PE address to carry IOMMU group (PE) ID, which is
passed to host kernel to specify the precise target for the EEH
operations.

Signed-off-by: Gavin Shan <address@hidden>
---
 hw/ppc/spapr_pci.c          | 30 ++++++++++++++-------
 hw/ppc/spapr_pci_vfio.c     | 64 ++++++++++++++++++++++++++++++++++++++++-----
 include/hw/pci-host/spapr.h |  6 ++---
 3 files changed, 82 insertions(+), 18 deletions(-)

diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
index a0cca22..93d55ab 100644
--- a/hw/ppc/spapr_pci.c
+++ b/hw/ppc/spapr_pci.c
@@ -30,6 +30,7 @@
 #include "hw/pci/pci_host.h"
 #include "hw/ppc/spapr.h"
 #include "hw/pci-host/spapr.h"
+#include "hw/vfio/vfio.h"
 #include "exec/address-spaces.h"
 #include <libfdt.h>
 #include "trace.h"
@@ -477,6 +478,7 @@ static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
     sPAPRPHBState *sphb;
     sPAPRPHBClass *spc;
     PCIDevice *pdev;
+    int groupid;
     uint32_t addr, option;
     uint64_t buid;
 
@@ -495,10 +497,6 @@ static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    /*
-     * We always have PE address of form "00BB0001". "BB"
-     * represents the bus number of PE's primary bus.
-     */
     option = rtas_ld(args, 3);
     switch (option) {
     case RTAS_GET_PE_ADDR:
@@ -508,7 +506,16 @@ static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
             goto param_error_exit;
         }
 
-        rtas_st(rets, 1, (pci_bus_num(pdev->bus) << 16) + 1);
+        /*
+         * We treat the (IOMMU group ID + 1) as the PE address
+         * because zero is invalid PE address.
+         */
+        groupid = vfio_get_group_id(pdev);
+        if (groupid < 0) {
+            goto param_error_exit;
+        }
+
+        rtas_st(rets, 1, groupid + 1);
         break;
     case RTAS_GET_PE_MODE:
         rtas_st(rets, 1, RTAS_PE_MODE_SHARED);
@@ -532,6 +539,7 @@ static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
 {
     sPAPRPHBState *sphb;
     sPAPRPHBClass *spc;
+    uint32_t addr;
     uint64_t buid;
     int state, ret;
 
@@ -539,6 +547,7 @@ static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
+    addr = rtas_ld(args, 0);
     buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
@@ -550,7 +559,7 @@ static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    ret = spc->eeh_get_state(sphb, &state);
+    ret = spc->eeh_get_state(sphb, addr, &state);
     rtas_st(rets, 0, ret);
     if (ret != RTAS_OUT_SUCCESS) {
         return;
@@ -576,7 +585,7 @@ static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
 {
     sPAPRPHBState *sphb;
     sPAPRPHBClass *spc;
-    uint32_t option;
+    uint32_t addr, option;
     uint64_t buid;
     int ret;
 
@@ -584,6 +593,7 @@ static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
+    addr = rtas_ld(args, 0);
     buid = rtas_ldq(args, 1);
     option = rtas_ld(args, 3);
     sphb = spapr_pci_find_phb(spapr, buid);
@@ -596,7 +606,7 @@ static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    ret = spc->eeh_reset(sphb, option);
+    ret = spc->eeh_reset(sphb, addr, option);
     rtas_st(rets, 0, ret);
     return;
 
@@ -612,6 +622,7 @@ static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
 {
     sPAPRPHBState *sphb;
     sPAPRPHBClass *spc;
+    uint32_t addr;
     uint64_t buid;
     int ret;
 
@@ -619,6 +630,7 @@ static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
+    addr = rtas_ld(args, 0);
     buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
@@ -630,7 +642,7 @@ static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    ret = spc->eeh_configure(sphb);
+    ret = spc->eeh_configure(sphb, addr);
     rtas_st(rets, 0, ret);
     return;
 
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index cca45ed..8579ace 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -72,10 +72,24 @@ static void spapr_phb_vfio_finish_realize(sPAPRPHBState 
*sphb, Error **errp)
                                 spapr_tce_get_iommu(tcet));
 }
 
+static int spapr_phb_vfio_eeh_enabled(sPAPRPHBState *sphb, int groupid)
+{
+    static int eeh_enabled_flag = -1;
+
+    if (eeh_enabled_flag >= 0)
+        return eeh_enabled_flag;
+
+    eeh_enabled_flag = vfio_container_ioctl(&sphb->iommu_as, groupid,
+                                            VFIO_CHECK_EXTENSION,
+                                           (void *)VFIO_EEH);
+    return eeh_enabled_flag;
+}
+
 static void spapr_phb_vfio_eeh_reenable(sPAPRPHBVFIOState *svphb)
 {
     struct vfio_eeh_pe_op op = {
         .argsz = sizeof(op),
+        .flags = 0,
         .op    = VFIO_EEH_PE_ENABLE
     };
 
@@ -98,6 +112,7 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
                                          unsigned int addr, int option)
 {
     sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
+    int flags, groupid = addr - 1;
     struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
     int ret;
 
@@ -121,6 +136,11 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState 
*sphb,
             return RTAS_OUT_PARAM_ERROR;
         }
 
+        groupid = vfio_get_group_id(pdev);
+        if (groupid < 0) {
+            return RTAS_OUT_PARAM_ERROR;
+        }
+
         op.op = VFIO_EEH_PE_ENABLE;
         break;
     }
@@ -134,6 +154,13 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState 
*sphb,
         return RTAS_OUT_PARAM_ERROR;
     }
 
+    flags = spapr_phb_vfio_eeh_enabled(sphb, groupid);
+    if (flags == VFIO_EEH_DISABLED) {
+        return RTAS_OUT_HW_ERROR;
+    }
+
+    op.flags |= flags;
+    op.groupid = groupid;
     ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
                                VFIO_EEH_PE_OP, &op);
     if (ret < 0) {
@@ -143,13 +170,22 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState 
*sphb,
     return RTAS_OUT_SUCCESS;
 }
 
-static int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state)
+static int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb,
+                                        unsigned int addr,
+                                        int *state)
 {
     sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
     struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
-    int ret;
+    int flags, ret;
+
+    flags = spapr_phb_vfio_eeh_enabled(sphb, addr - 1);
+    if (flags == VFIO_EEH_DISABLED) {
+        return RTAS_OUT_PARAM_ERROR;
+    }
 
     op.op = VFIO_EEH_PE_GET_STATE;
+    op.flags |= flags;
+    op.groupid = addr - 1;
     ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
                                VFIO_EEH_PE_OP, &op);
     if (ret < 0) {
@@ -203,11 +239,12 @@ static void spapr_phb_vfio_eeh_pre_reset(sPAPRPHBState 
*sphb)
        pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
 }
 
-static int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
+static int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb,
+                                    unsigned int addr, int option)
 {
     sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
     struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
-    int ret;
+    int flags, ret;
 
     switch (option) {
     case RTAS_SLOT_RESET_DEACTIVATE:
@@ -225,6 +262,13 @@ static int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, 
int option)
         return RTAS_OUT_PARAM_ERROR;
     }
 
+    flags = spapr_phb_vfio_eeh_enabled(sphb, addr - 1);
+    if (flags == VFIO_EEH_DISABLED) {
+        return RTAS_OUT_HW_ERROR;
+    }
+
+    op.flags |= flags;
+    op.groupid = addr - 1;
     ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
                                VFIO_EEH_PE_OP, &op);
     if (ret < 0) {
@@ -234,13 +278,21 @@ static int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, 
int option)
     return RTAS_OUT_SUCCESS;
 }
 
-static int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
+static int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb,
+                                        unsigned int addr)
 {
     sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
     struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
-    int ret;
+    int flags, ret;
+
+    flags = spapr_phb_vfio_eeh_enabled(sphb, addr - 1);
+    if (flags == VFIO_EEH_DISABLED) {
+        return RTAS_OUT_HW_ERROR;
+    }
 
     op.op = VFIO_EEH_PE_CONFIGURE;
+    op.flags |= flags;
+    op.groupid = addr - 1;
     ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
                                VFIO_EEH_PE_OP, &op);
     if (ret < 0) {
diff --git a/include/hw/pci-host/spapr.h b/include/hw/pci-host/spapr.h
index 7de5e02..e01bb74 100644
--- a/include/hw/pci-host/spapr.h
+++ b/include/hw/pci-host/spapr.h
@@ -50,9 +50,9 @@ struct sPAPRPHBClass {
 
     void (*finish_realize)(sPAPRPHBState *sphb, Error **errp);
     int (*eeh_set_option)(sPAPRPHBState *sphb, unsigned int addr, int option);
-    int (*eeh_get_state)(sPAPRPHBState *sphb, int *state);
-    int (*eeh_reset)(sPAPRPHBState *sphb, int option);
-    int (*eeh_configure)(sPAPRPHBState *sphb);
+    int (*eeh_get_state)(sPAPRPHBState *sphb, unsigned int addr, int *state);
+    int (*eeh_reset)(sPAPRPHBState *sphb, unsigned int addr, int option);
+    int (*eeh_configure)(sPAPRPHBState *sphb, unsigned int addr);
 };
 
 typedef struct spapr_pci_msi {
-- 
2.1.0