Re: [Qemu-devel] [PATCH] MTRR support on x86, part 1

[Carl-Daniel Hailfinger]

On 11.12.2008 21:59, Anthony Liguori wrote:
> Carl-Daniel Hailfinger wrote:
>> The current codebase ignores MTRR (Memory Type Range Register)
>> configuration writes and reads because Qemu does not implement caching.
>> All BIOS/firmware in know of for x86 do implement a mode called
>> Cache-as-RAM (CAR) which locks down the CPU cache lines and uses the CPU
>> cache like RAM before RAM is enabled. Qemu assumes RAM is accessible
>> from the start, but it would be nice to be able to run real
>> BIOS/firmware in Qemu. For that, we need CAR support and for CAR support
>> we have to support MTRRs.
>>
>> This patch is a first step in that direction. MTRRs are MSRs supported
>> by all recent x86 CPUs, even old i586. Besides influencing cache, the
>> MTRRs can be written and read back, so discarding MTRR writes violates
>> the expectations of existing code out there.
>> Handle common x86 MTRR reads and writes, but don't act on them.
>>
>> Signed-off-by: Carl-Daniel Hailfinger
>> <address@hidden>
>>
>> Index: target-i386/cpu.h
>> ===================================================================
>> --- target-i386/cpu.h    (revision 5879)
>> +++ target-i386/cpu.h    (working copy)
>> @@ -261,8 +261,25 @@
>>  
>>  #define MSR_IA32_PERF_STATUS            0x198
>>  
>> +#define MSR_MTRRphysBase(reg)        (0x200 + 2 * (reg))
>> +#define MSR_MTRRphysMask(reg)        (0x200 + 2 * (reg) + 1)
>> +
>> +#define MSR_MTRRfix64K_00000        0x250
>> +#define MSR_MTRRfix16K_80000        0x258
>> +#define MSR_MTRRfix16K_A0000        0x259
>> +#define MSR_MTRRfix4K_C0000        0x268
>> +#define MSR_MTRRfix4K_C8000        0x269
>> +#define MSR_MTRRfix4K_D0000        0x26a
>> +#define MSR_MTRRfix4K_D8000        0x26b
>> +#define MSR_MTRRfix4K_E0000        0x26c
>> +#define MSR_MTRRfix4K_E8000        0x26d
>> +#define MSR_MTRRfix4K_F0000        0x26e
>> +#define MSR_MTRRfix4K_F8000        0x26f
>>   
>
> I'm not a huge fan of the naming convention here.

The offer to use another naming scheme still stands. If you have any
suggestion, I'll act upon it.


>>  #define MSR_PAT                         0x277
>>  
>> +#define MSR_MTRRdefType            0x2ff
>> +
>>  #define MSR_EFER                        0xc0000080
>>  
>>  #define MSR_EFER_SCE   (1 << 0)
>> @@ -629,6 +646,14 @@
>>      uint32_t cpuid_ext3_features;
>>      uint32_t cpuid_apic_id;
>>  
>> +    /* MTRRs */
>> +    uint64_t mtrr_fixed[11];
>> +    uint64_t mtrr_deftype;
>> +    struct {
>> +        uint64_t base;
>> +        uint64_t mask;
>> +    } mtrr_var[8];
>>   
>
> These have to be saved/restored or else you'll potentially break live
> migration/savevm/loadvm.

I added save/restore support.
Thanks a lot for giving me the hints I needed to implement this.

Regards,
Carl-Daniel

The current codebase ignores MTRR (Memory Type Range Register)
configuration writes and reads because Qemu does not implement caching.
All BIOS/firmware in know of for x86 do implement a mode called
Cache-as-RAM (CAR) which locks down the CPU cache lines and uses the CPU
cache like RAM before RAM is enabled. Qemu assumes RAM is accessible
from the start, but it would be nice to be able to run real
BIOS/firmware in Qemu. For that, we need CAR support and for CAR support
we have to support MTRRs.

This patch is a first step in that direction. MTRRs are MSRs supported
by all recent x86 CPUs, even old i586. Besides influencing cache, the
MTRRs can be written and read back, so discarding MTRR writes violates
the expectations of existing code out there.
Handle common x86 MTRR reads and writes, but don't act on them.

Signed-off-by: Carl-Daniel Hailfinger <address@hidden>

Index: target-i386/machine.c
===================================================================
--- target-i386/machine.c       (revision 5987)
+++ target-i386/machine.c       (working copy)
@@ -134,6 +134,15 @@
     qemu_put_be16s(f, &env->intercept_dr_write);
     qemu_put_be32s(f, &env->intercept_exceptions);
     qemu_put_8s(f, &env->v_tpr);
+
+    /* MTRRs */
+    for(i = 0; i < 11; i++)
+        qemu_put_be64s(f, &env->mtrr_fixed[i]);
+    qemu_put_be64s(f, &env->mtrr_deftype);
+    for(i = 0; i < 8; i++) {
+        qemu_put_be64s(f, &env->mtrr_var[i].base);
+        qemu_put_be64s(f, &env->mtrr_var[i].mask);
+    }
 }
 
 #ifdef USE_X86LDOUBLE
@@ -169,7 +178,7 @@
     int32_t a20_mask;
 
     if (version_id != 3 && version_id != 4 && version_id != 5
-        && version_id != 6 && version_id != 7)
+        && version_id != 6 && version_id != 7 && version_id != 8)
         return -EINVAL;
     for(i = 0; i < CPU_NB_REGS; i++)
         qemu_get_betls(f, &env->regs[i]);
@@ -302,6 +311,18 @@
         qemu_get_be32s(f, &env->intercept_exceptions);
         qemu_get_8s(f, &env->v_tpr);
     }
+
+    if (version_id >= 8) {
+        /* MTRRs */
+        for(i = 0; i < 11; i++)
+            qemu_put_be64s(f, &env->mtrr_fixed[i]);
+        qemu_put_be64s(f, &env->mtrr_deftype);
+        for(i = 0; i < 8; i++) {
+            qemu_put_be64s(f, &env->mtrr_var[i].base);
+            qemu_put_be64s(f, &env->mtrr_var[i].mask);
+        }
+    }
+
     /* XXX: ensure compatiblity for halted bit ? */
     /* XXX: compute redundant hflags bits */
     env->hflags = hflags;
Index: target-i386/cpu.h
===================================================================
--- target-i386/cpu.h   (revision 5987)
+++ target-i386/cpu.h   (working copy)
@@ -261,8 +261,25 @@
 
 #define MSR_IA32_PERF_STATUS            0x198
 
+#define MSR_MTRRphysBase(reg)          (0x200 + 2 * (reg))
+#define MSR_MTRRphysMask(reg)          (0x200 + 2 * (reg) + 1)
+
+#define MSR_MTRRfix64K_00000           0x250
+#define MSR_MTRRfix16K_80000           0x258
+#define MSR_MTRRfix16K_A0000           0x259
+#define MSR_MTRRfix4K_C0000            0x268
+#define MSR_MTRRfix4K_C8000            0x269
+#define MSR_MTRRfix4K_D0000            0x26a
+#define MSR_MTRRfix4K_D8000            0x26b
+#define MSR_MTRRfix4K_E0000            0x26c
+#define MSR_MTRRfix4K_E8000            0x26d
+#define MSR_MTRRfix4K_F0000            0x26e
+#define MSR_MTRRfix4K_F8000            0x26f
+
 #define MSR_PAT                         0x277
 
+#define MSR_MTRRdefType                        0x2ff
+
 #define MSR_EFER                        0xc0000080
 
 #define MSR_EFER_SCE   (1 << 0)
@@ -629,6 +646,14 @@
     uint32_t cpuid_ext3_features;
     uint32_t cpuid_apic_id;
 
+    /* MTRRs */
+    uint64_t mtrr_fixed[11];
+    uint64_t mtrr_deftype;
+    struct {
+        uint64_t base;
+        uint64_t mask;
+    } mtrr_var[8];
+
 #ifdef USE_KQEMU
     int kqemu_enabled;
     int last_io_time;
@@ -776,7 +801,7 @@
 #define cpu_signal_handler cpu_x86_signal_handler
 #define cpu_list x86_cpu_list
 
-#define CPU_SAVE_VERSION 7
+#define CPU_SAVE_VERSION 8
 
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
Index: target-i386/op_helper.c
===================================================================
--- target-i386/op_helper.c     (revision 5987)
+++ target-i386/op_helper.c     (working copy)
@@ -3069,6 +3069,46 @@
         env->kernelgsbase = val;
         break;
 #endif
+    case MSR_MTRRphysBase(0):
+    case MSR_MTRRphysBase(1):
+    case MSR_MTRRphysBase(2):
+    case MSR_MTRRphysBase(3):
+    case MSR_MTRRphysBase(4):
+    case MSR_MTRRphysBase(5):
+    case MSR_MTRRphysBase(6):
+    case MSR_MTRRphysBase(7):
+        env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysBase(0)) / 2].base = val;
+        break;
+    case MSR_MTRRphysMask(0):
+    case MSR_MTRRphysMask(1):
+    case MSR_MTRRphysMask(2):
+    case MSR_MTRRphysMask(3):
+    case MSR_MTRRphysMask(4):
+    case MSR_MTRRphysMask(5):
+    case MSR_MTRRphysMask(6):
+    case MSR_MTRRphysMask(7):
+        env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysMask(0)) / 2].mask = val;
+        break;
+    case MSR_MTRRfix64K_00000:
+        env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix64K_00000] = val;
+        break;
+    case MSR_MTRRfix16K_80000:
+    case MSR_MTRRfix16K_A0000:
+        env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix16K_80000 + 1] = val;
+        break;
+    case MSR_MTRRfix4K_C0000:
+    case MSR_MTRRfix4K_C8000:
+    case MSR_MTRRfix4K_D0000:
+    case MSR_MTRRfix4K_D8000:
+    case MSR_MTRRfix4K_E0000:
+    case MSR_MTRRfix4K_E8000:
+    case MSR_MTRRfix4K_F0000:
+    case MSR_MTRRfix4K_F8000:
+        env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix4K_C0000 + 3] = val;
+        break;
+    case MSR_MTRRdefType:
+        env->mtrr_deftype = val;
+        break;
     default:
         /* XXX: exception ? */
         break;
@@ -3141,6 +3181,46 @@
         }
         break;
 #endif
+    case MSR_MTRRphysBase(0):
+    case MSR_MTRRphysBase(1):
+    case MSR_MTRRphysBase(2):
+    case MSR_MTRRphysBase(3):
+    case MSR_MTRRphysBase(4):
+    case MSR_MTRRphysBase(5):
+    case MSR_MTRRphysBase(6):
+    case MSR_MTRRphysBase(7):
+        val = env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysBase(0)) / 2].base;
+        break;
+    case MSR_MTRRphysMask(0):
+    case MSR_MTRRphysMask(1):
+    case MSR_MTRRphysMask(2):
+    case MSR_MTRRphysMask(3):
+    case MSR_MTRRphysMask(4):
+    case MSR_MTRRphysMask(5):
+    case MSR_MTRRphysMask(6):
+    case MSR_MTRRphysMask(7):
+        val = env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysMask(0)) / 2].mask;
+        break;
+    case MSR_MTRRfix64K_00000:
+        val = env->mtrr_fixed[0];
+        break;
+    case MSR_MTRRfix16K_80000:
+    case MSR_MTRRfix16K_A0000:
+        val = env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix16K_80000 + 1];
+        break;
+    case MSR_MTRRfix4K_C0000:
+    case MSR_MTRRfix4K_C8000:
+    case MSR_MTRRfix4K_D0000:
+    case MSR_MTRRfix4K_D8000:
+    case MSR_MTRRfix4K_E0000:
+    case MSR_MTRRfix4K_E8000:
+    case MSR_MTRRfix4K_F0000:
+    case MSR_MTRRfix4K_F8000:
+        val = env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix4K_C0000 + 3];
+        break;
+    case MSR_MTRRdefType:
+        val = env->mtrr_deftype;
+        break;
     default:
         /* XXX: exception ? */
         val = 0;