Re: [RFC 1/2] arm: acpi: pci-expender-bus: Make arm to support PXB-PCIE

[Michael S. Tsirkin]

On Thu, Feb 13, 2020 at 03:49:51PM +0800, Yubo Miao wrote:
> From: miaoyubo <address@hidden>
> 
> Currently virt machine is not supported by pxb-pcie,
> and only one main host bridge described in ACPI tables.
> Under this circumstance, different io numas for differnt devices
> is not possible, in order to present io numas to the guest,
> especially for host pssthrough devices. PXB-PCIE is supproted
> by arm and certain resource is allocated for each pxb-pcie
> in acpi table.
> 
> Signed-off-by: miaoyubo <address@hidden>
> ---
>  hw/arm/virt-acpi-build.c | 234 +++++++++++++++++++++++++++++++++++++--
>  hw/pci-host/gpex.c       |   4 +
>  include/hw/arm/virt.h    |   1 +
>  3 files changed, 228 insertions(+), 11 deletions(-)
> 
> diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
> index bd5f771e9b..2e449d0098 100644
> --- a/hw/arm/virt-acpi-build.c
> +++ b/hw/arm/virt-acpi-build.c
> @@ -49,6 +49,8 @@
>  #include "kvm_arm.h"
>  #include "migration/vmstate.h"
>  
> +#include "hw/arm/virt.h"
> +#include "hw/pci/pci_bus.h"
>  #define ARM_SPI_BASE 32
>  
>  static void acpi_dsdt_add_cpus(Aml *scope, int smp_cpus)
> @@ -152,20 +154,227 @@ static void acpi_dsdt_add_virtio(Aml *scope,
>  }
>  
>  static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap,
> -                              uint32_t irq, bool use_highmem, bool 
> highmem_ecam)
> +                              uint32_t irq, bool use_highmem, bool 
> highmem_ecam,
> +                              VirtMachineState *vms)
>  {
>      int ecam_id = VIRT_ECAM_ID(highmem_ecam);
> -    Aml *method, *crs, *ifctx, *UUID, *ifctx1, *elsectx, *buf;
> +    Aml *method, *crs, *ifctx, *UUID, *ifctx1, *elsectx, *buf, *dev;
>      int i, bus_no;
> +    int count = 0;
>      hwaddr base_mmio = memmap[VIRT_PCIE_MMIO].base;
>      hwaddr size_mmio = memmap[VIRT_PCIE_MMIO].size;
>      hwaddr base_pio = memmap[VIRT_PCIE_PIO].base;
>      hwaddr size_pio = memmap[VIRT_PCIE_PIO].size;
>      hwaddr base_ecam = memmap[ecam_id].base;
>      hwaddr size_ecam = memmap[ecam_id].size;
> +    /*
> +     * 0x600000 would be enough for pxb device
> +     * if it is too small, there is no enough space
> +     * for a pcie device plugged in a pcie-root port
> +     */
> +    hwaddr size_addr = 0x600000;
> +    hwaddr size_io = 0x4000;
>      int nr_pcie_buses = size_ecam / PCIE_MMCFG_SIZE_MIN;
> +    int root_bus_limit = 0xFF;
> +    PCIBus *bus = NULL;
> +    bus = VIRT_MACHINE(vms)->bus;
> +
> +    if (bus) {
> +        QLIST_FOREACH(bus, &bus->child, sibling) {
> +            uint8_t bus_num = pci_bus_num(bus);
> +            uint8_t numa_node = pci_bus_numa_node(bus);
> +
> +            if (!pci_bus_is_root(bus)) {
> +                continue;
> +            }
> +            if (bus_num < root_bus_limit) {
> +                root_bus_limit = bus_num - 1;
> +            }
> +            count++;
> +            dev = aml_device("PC%.02X", bus_num);
> +            aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A08")));
> +            aml_append(dev, aml_name_decl("_CID", aml_string("PNP0A03")));
> +            aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
> +            aml_append(dev, aml_name_decl("_CCA", aml_int(1)));
> +            aml_append(dev, aml_name_decl("_SEG", aml_int(0)));
> +            aml_append(dev, aml_name_decl("_BBN", aml_int(bus_num)));
> +            aml_append(dev, aml_name_decl("_UID", aml_int(bus_num)));
> +            aml_append(dev, aml_name_decl("_STR", aml_unicode("pxb 
> Device")));
> +            if (numa_node != NUMA_NODE_UNASSIGNED) {
> +                method = aml_method("_PXM", 0, AML_NOTSERIALIZED);
> +                aml_append(method, aml_return(aml_int(numa_node)));
> +                aml_append(dev, method);
> +            }
> +            /* Declare the PCI Routing Table. */
> +            Aml *rt_pkg = aml_varpackage(nr_pcie_buses * PCI_NUM_PINS);
> +            for (bus_no = 0; bus_no < nr_pcie_buses; bus_no++) {
> +                for (i = 0; i < PCI_NUM_PINS; i++) {
> +                    int gsi = (i + bus_no) % (PCI_NUM_PINS);
> +                    Aml *pkg = aml_package(4);
> +                    aml_append(pkg, aml_int((bus_no << 16) | 0xFFFF));
> +                    aml_append(pkg, aml_int(i));
> +                    aml_append(pkg, aml_name("GSI%d", gsi));
> +                    aml_append(pkg, aml_int(0));
> +                    aml_append(rt_pkg, pkg);
> +                }
> +            }
> +            aml_append(dev, aml_name_decl("_PRT", rt_pkg));
> +
> +            for (i = 0; i < PCI_NUM_PINS; i++) {
> +                uint32_t irqs =  irq + i;
> +                Aml *dev_gsi = aml_device("GSI%d", i);
> +                aml_append(dev_gsi, aml_name_decl("_HID",
> +                           aml_string("PNP0C0F")));
> +                aml_append(dev_gsi, aml_name_decl("_UID", aml_int(0)));
> +                crs = aml_resource_template();
> +                aml_append(crs,
> +                           aml_interrupt(AML_CONSUMER, AML_LEVEL,
> +                                         AML_ACTIVE_HIGH,
> +                                         AML_EXCLUSIVE, &irqs, 1));
> +                aml_append(dev_gsi, aml_name_decl("_PRS", crs));
> +                crs = aml_resource_template();
> +                aml_append(crs,
> +                           aml_interrupt(AML_CONSUMER, AML_LEVEL,
> +                                         AML_ACTIVE_HIGH,
> +                                         AML_EXCLUSIVE, &irqs, 1));
> +                aml_append(dev_gsi, aml_name_decl("_CRS", crs));
> +                method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
> +                aml_append(dev_gsi, method);
> +                aml_append(dev, dev_gsi);
> +            }
> +
> +            method = aml_method("_CBA", 0, AML_NOTSERIALIZED);
> +            aml_append(method, aml_return(aml_int(base_ecam)));
> +            aml_append(dev, method);
> +
> +            method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
> +            Aml *rbuf = aml_resource_template();
> +            aml_append(rbuf,
> +                       aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED,
> +                                           AML_POS_DECODE, 0x0000,
> +                                           bus_num, bus_num + 1, 0x0000,
> +                                           2));
> +            aml_append(rbuf,
> +                       aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
> +                                        AML_MAX_FIXED, AML_NON_CACHEABLE,
> +                                        AML_READ_WRITE, 0x0000,
> +                                        base_mmio + size_mmio -
> +                                        size_addr * count,
> +                                        base_mmio + size_mmio - 1 -
> +                                        size_addr * (count - 1),
> +                                        0x0000, size_addr));
> +            aml_append(rbuf,
> +                       aml_dword_io(AML_MIN_FIXED, AML_MAX_FIXED,
> +                       AML_POS_DECODE, AML_ENTIRE_RANGE,
> +                       0x0000, (size_pio) / 2 - size_io * count,
> +                       (size_pio / 2) - 1 - size_io * (count - 1),
> +                       base_pio, size_io));
> +
> +            if (use_highmem) {
> +                hwaddr base_mmio_high = memmap[VIRT_HIGH_PCIE_MMIO].base;
> +                hwaddr size_mmio_high = memmap[VIRT_HIGH_PCIE_MMIO].size;
> +
> +                aml_append(rbuf,
> +                       aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED,
> +                                        AML_MAX_FIXED, AML_NON_CACHEABLE,
> +                                        AML_READ_WRITE, 0x0000,
> +                                        base_mmio_high + size_mmio_high -
> +                                        size_addr * count,
> +                                        base_mmio_high + size_mmio_high -
> +                                        1 - size_addr * (count - 1),
> +                                        0x0000, size_addr));
> +            }
> +
> +            aml_append(method, aml_name_decl("RBUF", rbuf));
> +            aml_append(method, aml_return(rbuf));
> +            aml_append(dev, method);
> +            /* Declare an _OSC (OS Control Handoff) method */
> +            aml_append(dev, aml_name_decl("SUPP", aml_int(0)));
> +            aml_append(dev, aml_name_decl("CTRL", aml_int(0)));
> +            method = aml_method("_OSC", 4, AML_NOTSERIALIZED);
> +            aml_append(method,
> +            aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1"));
> +
> +            /*
> +             * PCI Firmware Specification 3.0
> +             * 4.5.1. _OSC Interface for PCI Host Bridge Devices
> +             * The _OSC interface for a PCI/PCI-X/PCI Express hierarchy is
> +             * identified by the Universal Unique IDentifier (UUID)
> +             * 33DB4D5B-1FF7-401C-9657-7441C03DD766
> +             */
> +            UUID = aml_touuid("33DB4D5B-1FF7-401C-9657-7441C03DD766");
> +            ifctx = aml_if(aml_equal(aml_arg(0), UUID));
> +            aml_append(ifctx,
> +                aml_create_dword_field(aml_arg(3), aml_int(4), "CDW2"));
> +            aml_append(ifctx,
> +                aml_create_dword_field(aml_arg(3), aml_int(8), "CDW3"));
> +            aml_append(ifctx, aml_store(aml_name("CDW2"), aml_name("SUPP")));
> +            aml_append(ifctx, aml_store(aml_name("CDW3"), aml_name("CTRL")));
> +            /*
> +             * Allow OS control for all 5 features:
> +             * PCIeHotplug SHPCHotplug PME AER PCIeCapability.
> +             */
> +            aml_append(ifctx,
> +                       aml_store(aml_and(aml_name("CTRL"), aml_int(0x1F), 
> NULL),
> +                       aml_name("CTRL")));
> +
> +            ifctx1 = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(0x1))));
> +            aml_append(ifctx1,
> +                       aml_store(aml_or(aml_name("CDW1"), aml_int(0x08), 
> NULL),
> +                       aml_name("CDW1")));
> +            aml_append(ifctx, ifctx1);
> +
> +            ifctx1 = aml_if(aml_lnot(aml_equal(aml_name("CDW3"),
> +                            aml_name("CTRL"))));
> +            aml_append(ifctx1,
> +                       aml_store(aml_or(aml_name("CDW1"), aml_int(0x10), 
> NULL),
> +                                 aml_name("CDW1")));
> +            aml_append(ifctx, ifctx1);
> +
> +            aml_append(ifctx, aml_store(aml_name("CTRL"), aml_name("CDW3")));
> +            aml_append(ifctx, aml_return(aml_arg(3)));
> +            aml_append(method, ifctx);
> +
> +            elsectx = aml_else();
> +            aml_append(elsectx,
> +                       aml_store(aml_or(aml_name("CDW1"), aml_int(4), NULL),
> +                       aml_name("CDW1")));
> +            aml_append(elsectx, aml_return(aml_arg(3)));
> +            aml_append(method, elsectx);
> +            aml_append(dev, method);
> +
> +            method = aml_method("_DSM", 4, AML_NOTSERIALIZED);
> +
> +            /*
> +             * PCI Firmware Specification 3.0
> +             * 4.6.1. _DSM for PCI Express Slot Information
> +             * The UUID in _DSM in this context is
> +             * {E5C937D0-3553-4D7A-9117-EA4D19C3434D}
> +             */
> +            UUID = aml_touuid("E5C937D0-3553-4D7A-9117-EA4D19C3434D");
> +            ifctx = aml_if(aml_equal(aml_arg(0), UUID));
> +            ifctx1 = aml_if(aml_equal(aml_arg(2), aml_int(0)));
> +            uint8_t byte_list[1] = {1};
> +            buf = aml_buffer(1, byte_list);
> +            aml_append(ifctx1, aml_return(buf));
> +            aml_append(ifctx, ifctx1);
> +            aml_append(method, ifctx);
> +
> +            byte_list[0] = 0;
> +            buf = aml_buffer(1, byte_list);
> +            aml_append(method, aml_return(buf));
> +            aml_append(dev, method);
> +
> +            Aml *dev_rp0 = aml_device("%s", "RP0");
> +            aml_append(dev_rp0, aml_name_decl("_ADR", aml_int(0)));
> +            aml_append(dev, dev_rp0);
> +
> +            aml_append(scope, dev);
> +
> +        }
> +    }


There's a bunch of code duplicated here. Please refactor creating
APIs for reused code.

>  
> -    Aml *dev = aml_device("%s", "PCI0");
> +    dev = aml_device("%s", "PCI0");
>      aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A08")));
>      aml_append(dev, aml_name_decl("_CID", aml_string("PNP0A03")));
>      aml_append(dev, aml_name_decl("_SEG", aml_int(0)));
> @@ -219,16 +428,18 @@ static void acpi_dsdt_add_pci(Aml *scope, const 
> MemMapEntry *memmap,
>      Aml *rbuf = aml_resource_template();
>      aml_append(rbuf,
>          aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
> -                            0x0000, 0x0000, nr_pcie_buses - 1, 0x0000,
> -                            nr_pcie_buses));
> +                            0x0000, 0x0000, root_bus_limit, 0x0000,
> +                            root_bus_limit + 1));
>      aml_append(rbuf,
>          aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
>                           AML_NON_CACHEABLE, AML_READ_WRITE, 0x0000, 
> base_mmio,
> -                         base_mmio + size_mmio - 1, 0x0000, size_mmio));
> +                         base_mmio + size_mmio - 1 - size_addr * count,
> +                         0x0000, size_mmio - size_addr * count));
>      aml_append(rbuf,
>          aml_dword_io(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
> -                     AML_ENTIRE_RANGE, 0x0000, 0x0000, size_pio - 1, 
> base_pio,
> -                     size_pio));
> +                     AML_ENTIRE_RANGE, 0x0000, 0x0000,
> +                     size_pio / 2 - 1 - size_io * count, base_pio,
> +                     size_pio / 2 - size_io * count));
>  
>      if (use_highmem) {
>          hwaddr base_mmio_high = memmap[VIRT_HIGH_PCIE_MMIO].base;
> @@ -238,8 +449,9 @@ static void acpi_dsdt_add_pci(Aml *scope, const 
> MemMapEntry *memmap,
>              aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
>                               AML_NON_CACHEABLE, AML_READ_WRITE, 0x0000,
>                               base_mmio_high,
> -                             base_mmio_high + size_mmio_high - 1, 0x0000,
> -                             size_mmio_high));
> +                             base_mmio_high + size_mmio_high - 1 -
> +                             size_addr * count,
> +                             0x0000, size_mmio_high - size_addr * count));
>      }
>  
>      aml_append(method, aml_name_decl("RBUF", rbuf));
> @@ -744,7 +956,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, 
> VirtMachineState *vms)
>      acpi_dsdt_add_virtio(scope, &memmap[VIRT_MMIO],
>                      (irqmap[VIRT_MMIO] + ARM_SPI_BASE), 
> NUM_VIRTIO_TRANSPORTS);
>      acpi_dsdt_add_pci(scope, memmap, (irqmap[VIRT_PCIE] + ARM_SPI_BASE),
> -                      vms->highmem, vms->highmem_ecam);
> +                      vms->highmem, vms->highmem_ecam, vms);
>      if (vms->acpi_dev) {
>          build_ged_aml(scope, "\\_SB."GED_DEVICE,
>                        HOTPLUG_HANDLER(vms->acpi_dev),
> diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c
> index 0ca604dc62..2c18cdfec4 100644
> --- a/hw/pci-host/gpex.c
> +++ b/hw/pci-host/gpex.c
> @@ -36,6 +36,7 @@
>  #include "hw/qdev-properties.h"
>  #include "migration/vmstate.h"
>  #include "qemu/module.h"
> +#include "hw/arm/virt.h"
>  
>  /****************************************************************************
>   * GPEX host
> @@ -98,6 +99,9 @@ static void gpex_host_realize(DeviceState *dev, Error 
> **errp)
>                                       pci_swizzle_map_irq_fn, s, &s->io_mmio,
>                                       &s->io_ioport, 0, 4, TYPE_PCIE_BUS);
>  
> +#ifdef __aarch64__
> +    VIRT_MACHINE(qdev_get_machine())->bus = pci->bus;
> +#endif
>      qdev_set_parent_bus(DEVICE(&s->gpex_root), BUS(pci->bus));
>      pci_bus_set_route_irq_fn(pci->bus, gpex_route_intx_pin_to_irq);
>      qdev_init_nofail(DEVICE(&s->gpex_root));
> diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
> index 71508bf40c..cfc65dd19b 100644
> --- a/include/hw/arm/virt.h
> +++ b/include/hw/arm/virt.h
> @@ -140,6 +140,7 @@ typedef struct {
>      DeviceState *gic;
>      DeviceState *acpi_dev;
>      Notifier powerdown_notifier;
> +    PCIBus *bus;
>  } VirtMachineState;


Given you only support one bus, why not just look the device
up based on type?

>  
>  #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)
> -- 
> 2.19.1
>