Re: [Qemu-devel] [PATCHv2 03/11] libqos: Move BAR assignment to common code

[Laurent Vivier]

On 19/10/2016 14:25, David Gibson wrote:
> The PCI backends in libqos each supply an iomap() and iounmap() function
> which is used to set up a specified PCI BAR.  But PCI BAR allocation takes
> place entirely within PCI space, so doesn't really need per-backend
> versions.  For example, Linux includes generic BAR allocation code used on
> platforms where that isn't done by firmware.
> 
> This patch merges the BAR allocation from the two existing backends into a
> single simplified copy.  The back ends just need to set up some parameters
> describing the window of PCI IO and PCI memory addresses which are
> available for allocation.  Like both the existing versions the new one uses
> a simple bump allocator.
> 
> Note that (again like the existing versions) this doesn't really handle
> 64-bit memory BARs properly.  It is actually used for such a BAR by the
> ivshmem test, and apparently the 32-bit MMIO BAR logic is close enough to
> work, as long as the BAR isn't too big.  Fixing that to properly handle
> 64-bit BAR allocation is a problem for another time.
> 
> Signed-off-by: David Gibson <address@hidden>

Reviewed-by: Laurent Vivier <address@hidden>

> ---
>  tests/libqos/pci-pc.c    | 87 ++--------------------------------------------
>  tests/libqos/pci-spapr.c | 89 
> ++----------------------------------------------
>  tests/libqos/pci.c       | 56 ++++++++++++++++++++++++++++--
>  tests/libqos/pci.h       |  7 ++--
>  4 files changed, 63 insertions(+), 176 deletions(-)
> 
> diff --git a/tests/libqos/pci-pc.c b/tests/libqos/pci-pc.c
> index 51dff8a..d0eb84a 100644
> --- a/tests/libqos/pci-pc.c
> +++ b/tests/libqos/pci-pc.c
> @@ -17,7 +17,6 @@
>  #include "hw/pci/pci_regs.h"
>  
>  #include "qemu-common.h"
> -#include "qemu/host-utils.h"
>  
>  
>  #define ACPI_PCIHP_ADDR         0xae00
> @@ -26,14 +25,6 @@
>  typedef struct QPCIBusPC
>  {
>      QPCIBus bus;
> -
> -    uint32_t pci_hole_start;
> -    uint32_t pci_hole_size;
> -    uint32_t pci_hole_alloc;
> -
> -    uint16_t pci_iohole_start;
> -    uint16_t pci_iohole_size;
> -    uint16_t pci_iohole_alloc;
>  } QPCIBusPC;
>  
>  static uint8_t qpci_pc_pio_readb(QPCIBus *bus, uint32_t addr)
> @@ -132,71 +123,6 @@ static void qpci_pc_config_writel(QPCIBus *bus, int 
> devfn, uint8_t offset, uint3
>      outl(0xcfc, value);
>  }
>  
> -static void *qpci_pc_iomap(QPCIBus *bus, QPCIDevice *dev, int barno, 
> uint64_t *sizeptr)
> -{
> -    QPCIBusPC *s = container_of(bus, QPCIBusPC, bus);
> -    static const int bar_reg_map[] = {
> -        PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2,
> -        PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_5,
> -    };
> -    int bar_reg;
> -    uint32_t addr;
> -    uint64_t size;
> -    uint32_t io_type;
> -
> -    g_assert(barno >= 0 && barno <= 5);
> -    bar_reg = bar_reg_map[barno];
> -
> -    qpci_config_writel(dev, bar_reg, 0xFFFFFFFF);
> -    addr = qpci_config_readl(dev, bar_reg);
> -
> -    io_type = addr & PCI_BASE_ADDRESS_SPACE;
> -    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> -        addr &= PCI_BASE_ADDRESS_IO_MASK;
> -    } else {
> -        addr &= PCI_BASE_ADDRESS_MEM_MASK;
> -    }
> -
> -    size = (1ULL << ctzl(addr));
> -    if (size == 0) {
> -        return NULL;
> -    }
> -    if (sizeptr) {
> -        *sizeptr = size;
> -    }
> -
> -    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> -        uint16_t loc;
> -
> -        g_assert(QEMU_ALIGN_UP(s->pci_iohole_alloc, size) + size
> -                 <= s->pci_iohole_size);
> -        s->pci_iohole_alloc = QEMU_ALIGN_UP(s->pci_iohole_alloc, size);
> -        loc = s->pci_iohole_start + s->pci_iohole_alloc;
> -        s->pci_iohole_alloc += size;
> -
> -        qpci_config_writel(dev, bar_reg, loc | PCI_BASE_ADDRESS_SPACE_IO);
> -
> -        return (void *)(intptr_t)loc;
> -    } else {
> -        uint64_t loc;
> -
> -        g_assert(QEMU_ALIGN_UP(s->pci_hole_alloc, size) + size
> -                 <= s->pci_hole_size);
> -        s->pci_hole_alloc = QEMU_ALIGN_UP(s->pci_hole_alloc, size);
> -        loc = s->pci_hole_start + s->pci_hole_alloc;
> -        s->pci_hole_alloc += size;
> -
> -        qpci_config_writel(dev, bar_reg, loc);
> -
> -        return (void *)(intptr_t)loc;
> -    }
> -}
> -
> -static void qpci_pc_iounmap(QPCIBus *bus, void *data)
> -{
> -    /* FIXME */
> -}
> -
>  QPCIBus *qpci_init_pc(QGuestAllocator *alloc)
>  {
>      QPCIBusPC *ret;
> @@ -227,16 +153,9 @@ QPCIBus *qpci_init_pc(QGuestAllocator *alloc)
>      ret->bus.config_writew = qpci_pc_config_writew;
>      ret->bus.config_writel = qpci_pc_config_writel;
>  
> -    ret->bus.iomap = qpci_pc_iomap;
> -    ret->bus.iounmap = qpci_pc_iounmap;
> -
> -    ret->pci_hole_start = 0xE0000000;
> -    ret->pci_hole_size = 0x20000000;
> -    ret->pci_hole_alloc = 0;
> -
> -    ret->pci_iohole_start = 0xc000;
> -    ret->pci_iohole_size = 0x4000;
> -    ret->pci_iohole_alloc = 0;
> +    ret->bus.pio_alloc_ptr = 0xc000;
> +    ret->bus.mmio_alloc_ptr = 0xE0000000;
> +    ret->bus.mmio_limit = 0x100000000ULL;
>  
>      return &ret->bus;
>  }
> diff --git a/tests/libqos/pci-spapr.c b/tests/libqos/pci-spapr.c
> index 2d26a94..70a24b5 100644
> --- a/tests/libqos/pci-spapr.c
> +++ b/tests/libqos/pci-spapr.c
> @@ -34,14 +34,6 @@ typedef struct QPCIBusSPAPR {
>  
>      uint64_t mmio32_cpu_base;
>      QPCIWindow mmio32;
> -
> -    uint64_t pci_hole_start;
> -    uint64_t pci_hole_size;
> -    uint64_t pci_hole_alloc;
> -
> -    uint32_t pci_iohole_start;
> -    uint32_t pci_iohole_size;
> -    uint32_t pci_iohole_alloc;
>  } QPCIBusSPAPR;
>  
>  /*
> @@ -167,72 +159,6 @@ static void qpci_spapr_config_writel(QPCIBus *bus, int 
> devfn, uint8_t offset,
>      qrtas_ibm_write_pci_config(s->alloc, s->buid, config_addr, 4, value);
>  }
>  
> -static void *qpci_spapr_iomap(QPCIBus *bus, QPCIDevice *dev, int barno,
> -                              uint64_t *sizeptr)
> -{
> -    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> -    static const int bar_reg_map[] = {
> -        PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2,
> -        PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_5,
> -    };
> -    int bar_reg;
> -    uint32_t addr;
> -    uint64_t size;
> -    uint32_t io_type;
> -
> -    g_assert(barno >= 0 && barno <= 5);
> -    bar_reg = bar_reg_map[barno];
> -
> -    qpci_config_writel(dev, bar_reg, 0xFFFFFFFF);
> -    addr = qpci_config_readl(dev, bar_reg);
> -
> -    io_type = addr & PCI_BASE_ADDRESS_SPACE;
> -    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> -        addr &= PCI_BASE_ADDRESS_IO_MASK;
> -    } else {
> -        addr &= PCI_BASE_ADDRESS_MEM_MASK;
> -    }
> -
> -    size = (1ULL << ctzl(addr));
> -    if (size == 0) {
> -        return NULL;
> -    }
> -    if (sizeptr) {
> -        *sizeptr = size;
> -    }
> -
> -    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> -        uint16_t loc;
> -
> -        g_assert(QEMU_ALIGN_UP(s->pci_iohole_alloc, size) + size
> -                 <= s->pci_iohole_size);
> -        s->pci_iohole_alloc = QEMU_ALIGN_UP(s->pci_iohole_alloc, size);
> -        loc = s->pci_iohole_start + s->pci_iohole_alloc;
> -        s->pci_iohole_alloc += size;
> -
> -        qpci_config_writel(dev, bar_reg, loc | PCI_BASE_ADDRESS_SPACE_IO);
> -
> -        return (void *)(unsigned long)loc;
> -    } else {
> -        uint64_t loc;
> -
> -        g_assert(QEMU_ALIGN_UP(s->pci_hole_alloc, size) + size
> -                 <= s->pci_hole_size);
> -        s->pci_hole_alloc = QEMU_ALIGN_UP(s->pci_hole_alloc, size);
> -        loc = s->pci_hole_start + s->pci_hole_alloc;
> -        s->pci_hole_alloc += size;
> -
> -        qpci_config_writel(dev, bar_reg, loc);
> -
> -        return (void *)(unsigned long)loc;
> -    }
> -}
> -
> -static void qpci_spapr_iounmap(QPCIBus *bus, void *data)
> -{
> -    /* FIXME */
> -}
> -
>  #define SPAPR_PCI_BASE               (1ULL << 45)
>  
>  #define SPAPR_PCI_MMIO32_WIN_SIZE    0x80000000 /* 2 GiB */
> @@ -270,9 +196,6 @@ QPCIBus *qpci_init_spapr(QGuestAllocator *alloc)
>      ret->bus.config_writew = qpci_spapr_config_writew;
>      ret->bus.config_writel = qpci_spapr_config_writel;
>  
> -    ret->bus.iomap = qpci_spapr_iomap;
> -    ret->bus.iounmap = qpci_spapr_iounmap;
> -
>      /* FIXME: We assume the default location of the PHB for now.
>       * Ideally we'd parse the device tree deposited in the guest to
>       * get the window locations */
> @@ -287,15 +210,9 @@ QPCIBus *qpci_init_spapr(QGuestAllocator *alloc)
>      ret->mmio32.pci_base = 0x80000000; /* 2 GiB */
>      ret->mmio32.size = SPAPR_PCI_MMIO32_WIN_SIZE;
>  
> -    ret->pci_hole_start = 0xC0000000;
> -    ret->pci_hole_size =
> -        ret->mmio32.pci_base + ret->mmio32.size - ret->pci_hole_start;
> -    ret->pci_hole_alloc = 0;
> -
> -    ret->pci_iohole_start = 0xc000;
> -    ret->pci_iohole_size =
> -        ret->pio.pci_base + ret->pio.size - ret->pci_iohole_start;
> -    ret->pci_iohole_alloc = 0;
> +    ret->bus.pio_alloc_ptr = 0xc000;
> +    ret->bus.mmio_alloc_ptr = ret->mmio32.pci_base;
> +    ret->bus.mmio_limit = ret->mmio32.pci_base + ret->mmio32.size;
>  
>      return &ret->bus;
>  }
> diff --git a/tests/libqos/pci.c b/tests/libqos/pci.c
> index 55b01df..bf1c532 100644
> --- a/tests/libqos/pci.c
> +++ b/tests/libqos/pci.c
> @@ -14,6 +14,7 @@
>  #include "libqos/pci.h"
>  
>  #include "hw/pci/pci_regs.h"
> +#include "qemu/host-utils.h"
>  
>  void qpci_device_foreach(QPCIBus *bus, int vendor_id, int device_id,
>                           void (*func)(QPCIDevice *dev, int devfn, void 
> *data),
> @@ -290,12 +291,63 @@ void qpci_io_writel(QPCIDevice *dev, void *data, 
> uint32_t value)
>  
>  void *qpci_iomap(QPCIDevice *dev, int barno, uint64_t *sizeptr)
>  {
> -    return dev->bus->iomap(dev->bus, dev, barno, sizeptr);
> +    QPCIBus *bus = dev->bus;
> +    static const int bar_reg_map[] = {
> +        PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2,
> +        PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_5,
> +    };
> +    int bar_reg;
> +    uint32_t addr, size;
> +    uint32_t io_type;
> +    uint64_t loc;
> +
> +    g_assert(barno >= 0 && barno <= 5);
> +    bar_reg = bar_reg_map[barno];
> +
> +    qpci_config_writel(dev, bar_reg, 0xFFFFFFFF);
> +    addr = qpci_config_readl(dev, bar_reg);
> +
> +    io_type = addr & PCI_BASE_ADDRESS_SPACE;
> +    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> +        addr &= PCI_BASE_ADDRESS_IO_MASK;
> +    } else {
> +        addr &= PCI_BASE_ADDRESS_MEM_MASK;
> +    }
> +
> +    g_assert(addr); /* Must have *some* size bits */
> +
> +    size = 1U << ctz32(addr);
> +    if (sizeptr) {
> +        *sizeptr = size;
> +    }
> +
> +    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> +        loc = QEMU_ALIGN_UP(bus->pio_alloc_ptr, size);
> +
> +        g_assert(loc >= bus->pio_alloc_ptr);
> +        g_assert(loc + size <= QPCI_PIO_LIMIT); /* Keep PIO below 64kiB */
> +
> +        bus->pio_alloc_ptr = loc + size;
> +
> +        qpci_config_writel(dev, bar_reg, loc | PCI_BASE_ADDRESS_SPACE_IO);
> +    } else {
> +        loc = QEMU_ALIGN_UP(bus->mmio_alloc_ptr, size);
> +
> +        /* Check for space */
> +        g_assert(loc >= bus->mmio_alloc_ptr);
> +        g_assert(loc + size <= bus->mmio_limit);
> +
> +        bus->mmio_alloc_ptr = loc + size;
> +
> +        qpci_config_writel(dev, bar_reg, loc);
> +    }
> +
> +    return (void *)(uintptr_t)loc;
>  }
>  
>  void qpci_iounmap(QPCIDevice *dev, void *data)
>  {
> -    dev->bus->iounmap(dev->bus, data);
> +    /* FIXME */
>  }
>  
>  void qpci_plug_device_test(const char *driver, const char *id,
> diff --git a/tests/libqos/pci.h b/tests/libqos/pci.h
> index 72a2245..f6f916d 100644
> --- a/tests/libqos/pci.h
> +++ b/tests/libqos/pci.h
> @@ -22,8 +22,7 @@
>  typedef struct QPCIDevice QPCIDevice;
>  typedef struct QPCIBus QPCIBus;
>  
> -struct QPCIBus
> -{
> +struct QPCIBus {
>      uint8_t (*pio_readb)(QPCIBus *bus, uint32_t addr);
>      uint16_t (*pio_readw)(QPCIBus *bus, uint32_t addr);
>      uint32_t (*pio_readl)(QPCIBus *bus, uint32_t addr);
> @@ -51,8 +50,8 @@ struct QPCIBus
>      void (*config_writel)(QPCIBus *bus, int devfn,
>                            uint8_t offset, uint32_t value);
>  
> -    void *(*iomap)(QPCIBus *bus, QPCIDevice *dev, int barno, uint64_t 
> *sizeptr);
> -    void (*iounmap)(QPCIBus *bus, void *data);
> +    uint16_t pio_alloc_ptr;
> +    uint64_t mmio_alloc_ptr, mmio_limit;
>  };
>  
>  struct QPCIDevice
>