Re: [Qemu-devel] [PATCH v4 RESEND 3/3] IOMMU: enable intel_iommu map and unmap notifiers

[Aviv B.D.]

On Wed, Oct 19, 2016 at 12:33 PM, Peter Xu <address@hidden> wrote:

> On Mon, Oct 17, 2016 at 06:44:24PM +0300, Aviv B.D wrote:
> > From: "Aviv Ben-David" <address@hidden>
> >
> > Adds a list of registered vtd_as's to intel iommu state to save
> > iteration over each PCI device in a search of the corrosponding domain.
> >
> > Signed-off-by: Aviv Ben-David <address@hidden>
> > ---
> >  hw/i386/intel_iommu.c          | 102 ++++++++++++++++++++++++++++++
> ++++++++---
> >  hw/i386/intel_iommu_internal.h |   2 +
> >  include/hw/i386/intel_iommu.h  |   9 ++++
> >  3 files changed, 106 insertions(+), 7 deletions(-)
> >
> > diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> > index dcf45f0..34fc1e8 100644
> > --- a/hw/i386/intel_iommu.c
> > +++ b/hw/i386/intel_iommu.c
> > @@ -51,6 +51,9 @@ static int vtd_dbgflags = VTD_DBGBIT(GENERAL) |
> VTD_DBGBIT(CSR);
> >  #define VTD_DPRINTF(what, fmt, ...) do {} while (0)
> >  #endif
> >
> > +static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t
> bus_num,
> > +                                    uint8_t devfn, VTDContextEntry *ce);
> > +
> >  static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t
> val,
> >                              uint64_t wmask, uint64_t w1cmask)
> >  {
> > @@ -142,6 +145,23 @@ static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState
> *s, hwaddr addr,
> >      return new_val;
> >  }
> >
> > +static int vtd_get_did_dev(IntelIOMMUState *s, uint8_t bus_num,
> uint8_t devfn,
> > +                           uint16_t *domain_id)
> > +{
> > +    VTDContextEntry ce;
> > +    int ret_fr;
> > +
> > +    assert(domain_id);
> > +
> > +    ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
> > +    if (ret_fr) {
> > +        return -1;
> > +    }
> > +
> > +    *domain_id =  VTD_CONTEXT_ENTRY_DID(ce.hi);
>                     ^ one more space
>
> > +    return 0;
> > +}
> > +
> >  /* GHashTable functions */
> >  static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
> >  {
> > @@ -683,9 +703,6 @@ static int vtd_gpa_to_slpte(VTDContextEntry *ce,
> uint64_t gpa,
> >          *reads = (*reads) && (slpte & VTD_SL_R);
> >          *writes = (*writes) && (slpte & VTD_SL_W);
> >          if (!(slpte & access_right_check)) {
> > -            VTD_DPRINTF(GENERAL, "error: lack of %s permission for "
> > -                        "gpa 0x%"PRIx64 " slpte 0x%"PRIx64,
> > -                        (flags == IOMMU_WO ? "write" : "read"), gpa,
> slpte);
>
> Could I ask why we are removing these lines? It can be useful if we
> have permission issues.
>

I will return Those lines if flags & NO_FAIL == 0

>
> >              return (flags == IOMMU_RW || flags == IOMMU_WO) ?
> >                     -VTD_FR_WRITE : -VTD_FR_READ;
> >          }
> > @@ -734,9 +751,6 @@ static int vtd_dev_to_context_entry(IntelIOMMUState
> *s, uint8_t bus_num,
> >      }
> >
> >      if (!vtd_context_entry_present(ce)) {
> > -        VTD_DPRINTF(GENERAL,
> > -                    "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") "
> > -                    "is not present", devfn, bus_num);
>
> Here as well. Any reason to remove it?
>
>
Here as well...


> >          return -VTD_FR_CONTEXT_ENTRY_P;
> >      } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
> >                 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
> > @@ -1065,6 +1079,55 @@ static void 
> > vtd_iotlb_domain_invalidate(IntelIOMMUState
> *s, uint16_t domain_id)
> >                                  &domain_id);
> >  }
> >
> > +static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s,
> > +                                           uint16_t domain_id, hwaddr
> addr,
> > +                                           uint8_t am)
> > +{
>
> The logic of this function looks strange to me.
>
> > +    IntelIOMMUNotifierNode *node;
> > +
> > +    QLIST_FOREACH(node, &(s->notifiers_list), next) {
> > +        VTDAddressSpace *vtd_as = node->vtd_as;
> > +        uint16_t vfio_domain_id;
> > +        int ret = vtd_get_did_dev(s, pci_bus_num(vtd_as->bus),
> vtd_as->devfn,
> > +                                  &vfio_domain_id);
> > +        if (!ret && domain_id == vfio_domain_id) {
> > +            IOMMUTLBEntry entry;
> > +
> > +            /* notify unmap */
> > +            if (node->notifier_flag & IOMMU_NOTIFIER_UNMAP) {
>
> First of all, if we are talking about VFIO, notifier_flag should
> always be MAP|UNMAP. So in that case, for newly mapped entries, looks
> like we will first send an UNMAP, then a MAP?
>

You are correct, there is no valid reason to have notifier_flag other than
MAP|UNMAP
at least for VFIO.
I'm not sure if in the feature there won't be good reason to do otherwise,
so my
code support this scenario...


> > +                VTD_DPRINTF(GENERAL, "Remove addr 0x%"PRIx64 " mask %d",
> > +                            addr, am);
> > +                entry.target_as = &address_space_memory;
> > +                entry.iova = addr & VTD_PAGE_MASK_4K;
> > +                entry.translated_addr = 0;
> > +                entry.addr_mask = ~VTD_PAGE_MASK(VTD_PAGE_SHIFT_4K +
> am);
> > +                entry.perm = IOMMU_NONE;
> > +                memory_region_notify_iommu(&node->vtd_as->iommu,
> entry);
> > +            }
> > +
> > +            /* notify map */
> > +            if (node->notifier_flag & IOMMU_NOTIFIER_MAP) {
> > +                hwaddr original_addr = addr;
> > +                VTD_DPRINTF(GENERAL, "add addr 0x%"PRIx64 " mask %d",
> addr, am);
> > +                while (addr < original_addr + (1 << am) *
> VTD_PAGE_SIZE) {
> > +                    /* call to vtd_iommu_translate */
> > +                    IOMMUTLBEntry entry = s->iommu_ops.translate(
> > +
>  &node->vtd_as->iommu,
> > +                                                         addr,
> > +                                                         IOMMU_NO_FAIL);
> > +                    if (entry.perm != IOMMU_NONE) {
> > +                        addr += entry.addr_mask + 1;
> > +                        memory_region_notify_iommu(&node->vtd_as->iommu,
> entry);
> > +                    } else {
> > +                        addr += VTD_PAGE_SIZE;
>
> IIUC, here is the point that we found "the page is gone" (so this is
> an UNMAP invalidation), and we should do memory_region_iommu_notify()
> for the whole area with IOMMU_NONE. Then we just quit the loop since
> continuous translate()s should fail as well if the first page is
> missing.
>
> Please correct if I am wrong.
>

If I remember correctly I encounter a few cases where there was hole of
unmaped
memory in the middle of otherwise mapped pages. If I remember correctly it
was
with linux kernel 4.4, but I'm not sure.


> Thanks,
>
> -- peterx
>