[Qemu-devel] [RFC] Implement qword CPU access to device MMIO regions

[Max Filippov]

From: Max Filippov <address@hidden>

For newly developed device verification we need to be able to distinguish qword
and dword access to its MMIO regions from CPU.

This goal is achieved with the following steps:
- io_mem_read[3] and io_mem_write[3] are used for the ldq/stq callbacks;
- softmmu io_read and io_write templates invoke these callbacks for 64 bit wide
  access if they are defined;
- new registration method (cpu_register_io_memory2) is defined for the devices
  that provide 64 bit wide MMIO access.

This implementation maintains backwards compatibility for the devices
that provide only 8-16-32 bit io memory access callbacks, allowing wider access
callbacks to be registered via cpu_register_io_memory2.

Legacy behavior of emulating 64 bit access via two adjacent 32 bit accesses
holds both for devices registered via cpu_register_io_memory and for those
registered via cpu_register_io_memory2 that do not provide 64 bit callbacks.

This implementation is developed for pre-memory API QEMU version.
If this approach proves to be useful I will port it to the new memory API. 

Signed-off-by: Max Filippov <address@hidden>
CC: Neil Turton <address@hidden>
CC: Konstantin Ushakov <address@hidden>
---
 cpu-common.h       |   23 +++++++++-
 exec.c             |  132 ++++++++++++++++++++++++++++++++++++++++------------
 softmmu_template.h |   10 ++++
 3 files changed, 134 insertions(+), 31 deletions(-)

diff --git a/cpu-common.h b/cpu-common.h
index c9878ba..173d3c2 100644
--- a/cpu-common.h
+++ b/cpu-common.h
@@ -35,8 +35,26 @@ typedef unsigned long ram_addr_t;
 
 /* memory API */
 
-typedef void CPUWriteMemoryFunc(void *opaque, target_phys_addr_t addr, 
uint32_t value);
+typedef void CPUWriteMemoryFunc(void *opaque, target_phys_addr_t addr,
+        uint32_t value);
+typedef void CPUWriteMemoryFunc64(void *opaque, target_phys_addr_t addr,
+        uint64_t value);
 typedef uint32_t CPUReadMemoryFunc(void *opaque, target_phys_addr_t addr);
+typedef uint64_t CPUReadMemoryFunc64(void *opaque, target_phys_addr_t addr);
+
+enum {
+    IO_MEM_FUNC32_NUM = 3,
+    IO_MEM_FUNC_NUM = 4,
+};
+typedef struct CPUWriteMemoryFuncs {
+    CPUWriteMemoryFunc * func32[IO_MEM_FUNC32_NUM];
+    CPUWriteMemoryFunc64 *func64;
+} CPUWriteMemoryFuncs;
+
+typedef struct CPUReadMemoryFuncs {
+    CPUReadMemoryFunc * func32[IO_MEM_FUNC32_NUM];
+    CPUReadMemoryFunc64 *func64;
+} CPUReadMemoryFuncs;
 
 void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
                                       ram_addr_t size,
@@ -81,6 +99,9 @@ ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr);
 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
                            CPUWriteMemoryFunc * const *mem_write,
                            void *opaque, enum device_endian endian);
+int cpu_register_io_memory2(const CPUReadMemoryFuncs *mem_read,
+                           const CPUWriteMemoryFuncs *mem_write,
+                           void *opaque, enum device_endian endian);
 void cpu_unregister_io_memory(int table_address);
 
 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
diff --git a/exec.c b/exec.c
index d0cbf15..0133c9f 100644
--- a/exec.c
+++ b/exec.c
@@ -205,8 +205,8 @@ static void io_mem_init(void);
 static void memory_map_init(void);
 
 /* io memory support */
-CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
-CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
+CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][IO_MEM_FUNC_NUM];
+CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][IO_MEM_FUNC_NUM];
 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
 static char io_mem_used[IO_MEM_NB_ENTRIES];
 static int io_mem_watch;
@@ -3310,13 +3310,13 @@ static void unassigned_mem_writel(void *opaque, 
target_phys_addr_t addr, uint32_
 #endif
 }
 
-static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
+static CPUReadMemoryFunc * const unassigned_mem_read[IO_MEM_FUNC_NUM] = {
     unassigned_mem_readb,
     unassigned_mem_readw,
     unassigned_mem_readl,
 };
 
-static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
+static CPUWriteMemoryFunc * const unassigned_mem_write[IO_MEM_FUNC_NUM] = {
     unassigned_mem_writeb,
     unassigned_mem_writew,
     unassigned_mem_writel,
@@ -3640,8 +3640,8 @@ static int get_free_io_mem_idx(void)
  */
 
 typedef struct SwapEndianContainer {
-    CPUReadMemoryFunc *read[3];
-    CPUWriteMemoryFunc *write[3];
+    CPUReadMemoryFunc *read[IO_MEM_FUNC_NUM];
+    CPUWriteMemoryFunc *write[IO_MEM_FUNC_NUM];
     void *opaque;
 } SwapEndianContainer;
 
@@ -3669,10 +3669,20 @@ static uint32_t swapendian_mem_readl(void *opaque, 
target_phys_addr_t addr)
     return val;
 }
 
-static CPUReadMemoryFunc * const swapendian_readfn[3]={
+static CPUReadMemoryFunc64 swapendian_mem_readq;
+static uint64_t swapendian_mem_readq(void *opaque, target_phys_addr_t addr)
+{
+    uint64_t val;
+    SwapEndianContainer *c = opaque;
+    val = bswap64(((CPUReadMemoryFunc64 *)c->read[3])(c->opaque, addr));
+    return val;
+}
+
+static CPUReadMemoryFunc * const swapendian_readfn[IO_MEM_FUNC_NUM] = {
     swapendian_mem_readb,
     swapendian_mem_readw,
-    swapendian_mem_readl
+    swapendian_mem_readl,
+    (void *)swapendian_mem_readq,
 };
 
 static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
@@ -3696,10 +3706,19 @@ static void swapendian_mem_writel(void *opaque, 
target_phys_addr_t addr,
     c->write[2](c->opaque, addr, bswap32(val));
 }
 
-static CPUWriteMemoryFunc * const swapendian_writefn[3]={
+static CPUWriteMemoryFunc64 swapendian_mem_writeq;
+static void swapendian_mem_writeq(void *opaque, target_phys_addr_t addr,
+                                  uint64_t val)
+{
+    SwapEndianContainer *c = opaque;
+    ((CPUWriteMemoryFunc64 *)c->write[3])(c->opaque, addr, bswap64(val));
+}
+
+static CPUWriteMemoryFunc * const swapendian_writefn[IO_MEM_FUNC_NUM] = {
     swapendian_mem_writeb,
     swapendian_mem_writew,
-    swapendian_mem_writel
+    swapendian_mem_writel,
+    (void *)swapendian_mem_writeq,
 };
 
 static void swapendian_init(int io_index)
@@ -3709,12 +3728,16 @@ static void swapendian_init(int io_index)
 
     /* Swap mmio for big endian targets */
     c->opaque = io_mem_opaque[io_index];
-    for (i = 0; i < 3; i++) {
+    for (i = 0; i < IO_MEM_FUNC_NUM; i++) {
         c->read[i] = io_mem_read[io_index][i];
         c->write[i] = io_mem_write[io_index][i];
 
-        io_mem_read[io_index][i] = swapendian_readfn[i];
-        io_mem_write[io_index][i] = swapendian_writefn[i];
+        if (io_mem_read[io_index][i]) {
+            io_mem_read[io_index][i] = swapendian_readfn[i];
+        }
+        if (io_mem_write[io_index][i]) {
+            io_mem_write[io_index][i] = swapendian_writefn[i];
+        }
     }
     io_mem_opaque[io_index] = c;
 }
@@ -3736,6 +3759,7 @@ static void swapendian_del(int io_index)
 static int cpu_register_io_memory_fixed(int io_index,
                                         CPUReadMemoryFunc * const *mem_read,
                                         CPUWriteMemoryFunc * const *mem_write,
+                                        int func_num,
                                         void *opaque, enum device_endian 
endian)
 {
     int i;
@@ -3750,13 +3774,18 @@ static int cpu_register_io_memory_fixed(int io_index,
             return -1;
     }
 
-    for (i = 0; i < 3; ++i) {
-        io_mem_read[io_index][i]
-            = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
-    }
-    for (i = 0; i < 3; ++i) {
-        io_mem_write[io_index][i]
-            = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
+    for (i = 0; i < IO_MEM_FUNC_NUM; ++i) {
+        if (i < IO_MEM_FUNC32_NUM) {
+            io_mem_read[io_index][i]
+                = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
+            io_mem_write[io_index][i]
+                = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
+        } else {
+            io_mem_read[io_index][i]
+                = (i < func_num ? mem_read[i] : NULL);
+            io_mem_write[io_index][i]
+                = (i < func_num ? mem_write[i] : NULL);
+        }
     }
     io_mem_opaque[io_index] = opaque;
 
@@ -3783,7 +3812,27 @@ int cpu_register_io_memory(CPUReadMemoryFunc * const 
*mem_read,
                            CPUWriteMemoryFunc * const *mem_write,
                            void *opaque, enum device_endian endian)
 {
-    return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, 
endian);
+    return cpu_register_io_memory_fixed(
+            0, mem_read, mem_write, IO_MEM_FUNC32_NUM,
+            opaque, endian);
+}
+
+int cpu_register_io_memory2(const CPUReadMemoryFuncs *mem_read,
+                           const CPUWriteMemoryFuncs *mem_write,
+                           void *opaque, enum device_endian endian)
+{
+    CPUReadMemoryFunc *mem_read_fns[IO_MEM_FUNC_NUM];
+    CPUWriteMemoryFunc *mem_write_fns[IO_MEM_FUNC_NUM];
+
+    memcpy(mem_read_fns, mem_read->func32, sizeof(mem_read->func32));
+    mem_read_fns[3] = (void *)mem_read->func64;
+
+    memcpy(mem_write_fns, mem_write->func32, sizeof(mem_write->func32));
+    mem_write_fns[3] = (void *)mem_write->func64;
+
+    return cpu_register_io_memory_fixed(
+            0, mem_read_fns, mem_write_fns, IO_MEM_FUNC_NUM,
+            opaque, endian);
 }
 
 void cpu_unregister_io_memory(int io_table_address)
@@ -3793,7 +3842,7 @@ void cpu_unregister_io_memory(int io_table_address)
 
     swapendian_del(io_index);
 
-    for (i=0;i < 3; i++) {
+    for (i = 0; i < IO_MEM_FUNC_NUM; i++) {
         io_mem_read[io_index][i] = unassigned_mem_read[i];
         io_mem_write[io_index][i] = unassigned_mem_write[i];
     }
@@ -3806,14 +3855,14 @@ static void io_mem_init(void)
     int i;
 
     cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
-                                 unassigned_mem_write, NULL,
-                                 DEVICE_NATIVE_ENDIAN);
+                                 unassigned_mem_write, IO_MEM_FUNC32_NUM,
+                                 NULL, DEVICE_NATIVE_ENDIAN);
     cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
-                                 unassigned_mem_write, NULL,
-                                 DEVICE_NATIVE_ENDIAN);
+                                 unassigned_mem_write, IO_MEM_FUNC32_NUM,
+                                 NULL, DEVICE_NATIVE_ENDIAN);
     cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
-                                 notdirty_mem_write, NULL,
-                                 DEVICE_NATIVE_ENDIAN);
+                                 notdirty_mem_write, IO_MEM_FUNC32_NUM,
+                                 NULL, DEVICE_NATIVE_ENDIAN);
     for (i=0; i<5; i++)
         io_mem_used[i] = 1;
 
@@ -3917,7 +3966,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, 
uint8_t *buf,
                     addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                 /* XXX: could force cpu_single_env to NULL to avoid
                    potential bugs */
-                if (l >= 4 && ((addr1 & 3) == 0)) {
+                if (l >= 8 && ((addr1 & 7) == 0) && io_mem_write[io_index][3]) 
{
+                    /* 32 bit write access */
+                    uint64_t val = ldq_p(buf);
+                    ((CPUWriteMemoryFunc64 *)io_mem_write[io_index][3])(
+                           io_mem_opaque[io_index], addr1, val);
+                    l = 8;
+                } else if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit write access */
                     val = ldl_p(buf);
                     io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, 
val);
@@ -3956,7 +4011,14 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, 
uint8_t *buf,
                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
                 if (p)
                     addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
-                if (l >= 4 && ((addr1 & 3) == 0)) {
+                if (l >= 8 && ((addr1 & 7) == 0) && io_mem_read[io_index][3]) {
+                    /* 64 bit read access */
+                    uint64_t val = ((CPUReadMemoryFunc64 *)
+                            io_mem_read[io_index][3])(
+                                io_mem_opaque[io_index], addr1);
+                    stq_p(buf, val);
+                    l = 8;
+                } else if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit read access */
                     val = io_mem_read[io_index][2](io_mem_opaque[io_index], 
addr1);
                     stl_p(buf, val);
@@ -4262,6 +4324,10 @@ static inline uint64_t 
ldq_phys_internal(target_phys_addr_t addr,
         if (p)
             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
 
+        if (io_mem_read[io_index][3]) {
+            val = ((CPUReadMemoryFunc64 *)io_mem_read[io_index][3])(
+                    io_mem_opaque[io_index], addr);
+        } else {
         /* XXX This is broken when device endian != cpu endian.
                Fix and add "endian" variable check */
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -4271,6 +4337,7 @@ static inline uint64_t 
ldq_phys_internal(target_phys_addr_t addr,
         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
         val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], 
addr + 4) << 32;
 #endif
+        }
     } else {
         /* RAM case */
         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
@@ -4437,6 +4504,10 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t 
val)
         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
         if (p)
             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
+        if (io_mem_write[io_index][3]) {
+            ((CPUWriteMemoryFunc64 *)io_mem_write[io_index][3])(
+                    io_mem_opaque[io_index], addr, val);
+        } else {
 #ifdef TARGET_WORDS_BIGENDIAN
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
@@ -4444,6 +4515,7 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t 
val)
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 
32);
 #endif
+        }
     } else {
         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
             (addr & ~TARGET_PAGE_MASK);
diff --git a/softmmu_template.h b/softmmu_template.h
index 36eb2e8..c767441 100644
--- a/softmmu_template.h
+++ b/softmmu_template.h
@@ -74,6 +74,10 @@ static inline DATA_TYPE glue(io_read, 
SUFFIX)(target_phys_addr_t physaddr,
 #if SHIFT <= 2
     res = io_mem_read[index][SHIFT](io_mem_opaque[index], physaddr);
 #else
+    if (io_mem_read[index][3]) {
+        res = ((CPUReadMemoryFunc64 *)
+                io_mem_read[index][3])(io_mem_opaque[index], physaddr);
+    } else {
 #ifdef TARGET_WORDS_BIGENDIAN
     res = (uint64_t)io_mem_read[index][2](io_mem_opaque[index], physaddr) << 
32;
     res |= io_mem_read[index][2](io_mem_opaque[index], physaddr + 4);
@@ -81,6 +85,7 @@ static inline DATA_TYPE glue(io_read, 
SUFFIX)(target_phys_addr_t physaddr,
     res = io_mem_read[index][2](io_mem_opaque[index], physaddr);
     res |= (uint64_t)io_mem_read[index][2](io_mem_opaque[index], physaddr + 4) 
<< 32;
 #endif
+    }
 #endif /* SHIFT > 2 */
     return res;
 }
@@ -217,6 +222,10 @@ static inline void glue(io_write, 
SUFFIX)(target_phys_addr_t physaddr,
 #if SHIFT <= 2
     io_mem_write[index][SHIFT](io_mem_opaque[index], physaddr, val);
 #else
+    if (io_mem_write[index][3]) {
+        ((CPUWriteMemoryFunc64 *)
+         io_mem_write[index][3])(io_mem_opaque[index], physaddr, val);
+    } else {
 #ifdef TARGET_WORDS_BIGENDIAN
     io_mem_write[index][2](io_mem_opaque[index], physaddr, val >> 32);
     io_mem_write[index][2](io_mem_opaque[index], physaddr + 4, val);
@@ -224,6 +233,7 @@ static inline void glue(io_write, 
SUFFIX)(target_phys_addr_t physaddr,
     io_mem_write[index][2](io_mem_opaque[index], physaddr, val);
     io_mem_write[index][2](io_mem_opaque[index], physaddr + 4, val >> 32);
 #endif
+    }
 #endif /* SHIFT > 2 */
 }
 
-- 
1.7.2.5