Re: [Qemu-devel] [RFC] memory consumption of Qemu is twice as much as the previous version in KVM

[Gonglei (Arei)]

Attaching the backtrace [call kvm_set_user_memory_region() twice]:

In the KVM side:

[3287581.559570] gonglei: vzalloc, size=10485760
[3287581.564108] CPU: 19 PID: 3240 Comm: qemu-system-x86 Tainted: G           
OE  ---- -------   3.10.0-327.44.58.25_6.x86_64 #1
[3287581.575761] Hardware name: To be filled by O.E.M. RH2288H V3/BC11HGSA0, 
BIOS 1.26 12/22/2014
[3287581.776627] Call Trace:
[3287581.779427]  [<ffffffff81643dbc>] dump_stack+0x19/0x1b
[3287581.784922]  [<ffffffffa02e70e9>] kvm_kvzalloc.part.55+0x23/0x2e [kvm]
[3287581.791803]  [<ffffffffa02a52c3>] kvm_kvzalloc+0x23/0x30 [kvm]
[3287581.797989]  [<ffffffffa02bcd02>] kvm_arch_create_memslot+0x52/0x1a0 [kvm]
[3287581.805213]  [<ffffffffa02a5a60>] __kvm_set_memory_region+0x790/0xa90 [kvm]
[3287581.812520]  [<ffffffffa02a5d8b>] kvm_set_memory_region+0x2b/0x40 [kvm]
[3287581.819492]  [<ffffffffa02a61f4>] kvm_vm_ioctl+0x454/0x700 [kvm]
[3287581.825850]  [<ffffffff811fd6c5>] do_vfs_ioctl+0x2e5/0x4c0
[3287581.831681]  [<ffffffff8164f633>] ? __do_page_fault+0x183/0x470
[3287581.837953]  [<ffffffff811fd941>] SyS_ioctl+0xa1/0xc0
[3287581.843329]  [<ffffffff81654149>] system_call_fastpath+0x16/0x1b

[3287588.130837] gonglei: vzalloc, size=10485760
[3287588.135367] CPU: 19 PID: 3240 Comm: qemu-system-x86 Tainted: G           
OE  ---- -------   3.10.0-327.44.58.25_6.x86_64 #1
[3287588.347737] Call Trace:
[3287588.350543]  [<ffffffff81643dbc>] dump_stack+0x19/0x1b
[3287588.359764]  [<ffffffffa02e70e9>] kvm_kvzalloc.part.55+0x23/0x2e [kvm]
[3287588.366633]  [<ffffffffa02a52c3>] kvm_kvzalloc+0x23/0x30 [kvm]
[3287588.372791]  [<ffffffffa02bcd02>] kvm_arch_create_memslot+0x52/0x1a0 [kvm]
[3287588.380014]  [<ffffffffa02a5a60>] __kvm_set_memory_region+0x790/0xa90 [kvm]
[3287588.387298]  [<ffffffffa02a5d8b>] kvm_set_memory_region+0x2b/0x40 [kvm]
[3287588.394261]  [<ffffffffa02a61f4>] kvm_vm_ioctl+0x454/0x700 [kvm]
[3287588.400585]  [<ffffffff811fd6c5>] do_vfs_ioctl+0x2e5/0x4c0
[3287588.406420]  [<ffffffff8164f633>] ? __do_page_fault+0x183/0x470
[3287588.412687]  [<ffffffff811fd941>] SyS_ioctl+0xa1/0xc0
[3287588.418084]  [<ffffffff81654149>] system_call_fastpath+0x16/0x1b


In the Qemu side:

gdb) bt
#0  0x00007fb4449e39ad in kvm_set_user_memory_region (kml=0x7fb4463da640, 
slot=0x7fb4463da710)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:260
#1  0x00007fb4449e4a71 in kvm_set_phys_mem (kml=0x7fb4463da640, 
section=0x7ffeec275c30, add=true)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:824
#2  0x00007fb4449e4aff in kvm_region_add (listener=0x7fb4463da640, 
section=0x7ffeec275c30)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:838
#3  0x00007fb4449ebc8b in address_space_update_topology_pass (as=0x7fb4453caa60 
<address_space_memory>, old_view=0x7fb44645aec0, 
    new_view=0x7fb44645ee20, adding=true) at /mnt/sdb/gonglei/qemu/memory.c:870
#4  0x00007fb4449ebd52 in address_space_update_topology (as=0x7fb4453caa60 
<address_space_memory>)
    at /mnt/sdb/gonglei/qemu/memory.c:885
#5  0x00007fb4449ebe7e in memory_region_transaction_commit () at 
/mnt/sdb/gonglei/qemu/memory.c:925
#6  0x00007fb4449eeb9a in memory_region_update_container_subregions 
(subregion=0x7fb44645e770)
    at /mnt/sdb/gonglei/qemu/memory.c:1902
#7  0x00007fb4449eec09 in memory_region_add_subregion_common 
(mr=0x7fb4463c7640, offset=4294967296, subregion=0x7fb44645e770)
    at /mnt/sdb/gonglei/qemu/memory.c:1912
#8  0x00007fb4449eec4f in memory_region_add_subregion (mr=0x7fb4463c7640, 
offset=4294967296, subregion=0x7fb44645e770)
    at /mnt/sdb/gonglei/qemu/memory.c:1921
#9  0x00007fb444a5c91a in pc_memory_init (pcms=0x7fb4463c3fc0, 
system_memory=0x7fb4463c7640, rom_memory=0x7fb446468d40, 
    ram_memory=0x7ffeec275ee8) at /mnt/sdb/gonglei/qemu/hw/i386/pc.c:1338
#10 0x00007fb444a5ee16 in pc_init1 (machine=0x7fb4463c3fc0, 
host_type=0x7fb444e4740c "i440FX-pcihost", 
    pci_type=0x7fb444e47405 "i440FX") at 
/mnt/sdb/gonglei/qemu/hw/i386/pc_piix.c:158
#11 0x00007fb444a5f8f5 in pc_init_v2_6 (machine=0x7fb4463c3fc0)
    at /mnt/sdb/gonglei/qemu/hw/i386/pc_piix.c:432
#12 0x00007fb444b0970b in main (argc=81, argv=0x7ffeec2763c8, 
envp=0x7fb445a313e0) at vl.c:4839
(gdb) p *slot
$1 = {start_addr = 4294967296, memory_size = 5368709120, ram = 0x7fb2f4a00000, 
slot = 1, flags = 0}
 (gdb) c
Continuing.

Program received signal SIGINT, Interrupt.
0x00007fb4449e39ad in kvm_set_user_memory_region (kml=0x7fb445429d00 
<smram_listener>, slot=0x7fb44787d2b0)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:260
260         while(debug_halt);
Missing separate debuginfos, use: debuginfo-install 
cyrus-sasl-md5-2.1.26-17.x86_64 libdb-5.3.21-17.1.x86_64
(gdb) bt
#0  0x00007fb4449e39ad in kvm_set_user_memory_region (kml=0x7fb445429d00 
<smram_listener>, slot=0x7fb44787d2b0)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:260
#1  0x00007fb4449e4a71 in kvm_set_phys_mem (kml=0x7fb445429d00 
<smram_listener>, section=0x7ffeec275ed0, add=true)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:824
#2  0x00007fb4449e4aff in kvm_region_add (listener=0x7fb445429d00 
<smram_listener>, section=0x7ffeec275ed0)
    at /mnt/sdb/gonglei/qemu/kvm-all.c:838
#3  0x00007fb4449ef9d1 in listener_add_address_space (listener=0x7fb445429d00 
<smram_listener>, 
    as=0x7fb445429ba0 <smram_address_space>) at 
/mnt/sdb/gonglei/qemu/memory.c:2179
#4  0x00007fb4449efb65 in memory_listener_register (listener=0x7fb445429d00 
<smram_listener>, 
    filter=0x7fb445429ba0 <smram_address_space>) at 
/mnt/sdb/gonglei/qemu/memory.c:2208
#5  0x00007fb4449e4ef2 in kvm_memory_listener_register (s=0x7fb4463d95a0, 
kml=0x7fb445429d00 <smram_listener>, 
    as=0x7fb445429ba0 <smram_address_space>, as_id=1) at 
/mnt/sdb/gonglei/qemu/kvm-all.c:950
#6  0x00007fb444add51b in register_smram_listener (n=0x7fb445429cd0 
<smram_machine_done>, unused=0x0)
    at /mnt/sdb/gonglei/qemu/target-i386/kvm.c:1094
#7  0x00007fb444e195f8 in notifier_list_notify (list=0x7fb44542bc60 
<machine_init_done_notifiers>, data=0x0) at util/notify.c:40
#8  0x00007fb444b04f48 in qemu_run_machine_init_done_notifiers () at vl.c:2890
#9  0x00007fb444b09958 in main (argc=81, argv=0x7ffeec2763c8, 
envp=0x7fb445a313e0) at vl.c:4940
(gdb) p *slot
$2 = {start_addr = 4294967296, memory_size = 5368709120, ram = 0x7fb2f4a00000, 
slot = 5, flags = 0}

Regards,
-Gonglei


> -----Original Message-----
> From: Gonglei (Arei)
> Sent: Monday, May 22, 2017 3:05 PM
> To: 'Paolo Bonzini'
> Cc: address@hidden; address@hidden
> Subject: [RFC] memory consumption of Qemu is twice as much as the previous
> version in KVM
> 
> Hi Paolo,
> 
> I found that the latest Qemu eat 2 time memory in KVM since Qemu-2.3.0.
> 
> Replication Steps:
> 
> 1. I created a CentOS 7 with 4U8G using Qemu-2.3.0,
> 
> # grep kvm_kvzalloc /proc/vmallocinfo | awk '{total+=$2}; END {print total}'
> 16932864
> # grep kvm_kvzalloc /proc/vmallocinfo
> 0xffffc900205c7000-0xffffc90020fc8000 10489856 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=2560 vmalloc vpages N1=2560
> 0xffffc90020fc8000-0xffffc90020fce000   24576 kvm_kvzalloc+0x3c/0x40 [kvm]
> pages=5 vmalloc N1=5
> 0xffffc90020fce000-0xffffc90020fd4000   24576 kvm_kvzalloc+0x3c/0x40 [kvm]
> pages=5 vmalloc N1=5
> 0xffffc90020fd4000-0xffffc90020fd8000   16384 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=3 vmalloc N1=3
> 0xffffc9002438b000-0xffffc9002498c000 6295552 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=1536 vmalloc vpages N1=1536
> 0xffffc9002498c000-0xffffc90024990000   16384 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=3 vmalloc N1=3
> 0xffffc90024990000-0xffffc90024994000   16384 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=3 vmalloc N1=3
> 0xffffc90024994000-0xffffc90024997000   12288 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=2 vmalloc N1=2
> 0xffffc90024a75000-0xffffc90024a7e000   36864 kvm_kvzalloc+0x3c/0x40
> [kvm] pages=8 vmalloc N1=8
> 
> PS: There is only this VM in my host.
> 
> 2. Do the same test using the latest Qemu:
> 
> # grep kvm_kvzalloc /proc/vmallocinfo | awk '{total+=$2}; END {print total}'
> 33865728
> linux-PsHdkO:~ # grep kvm_kvzalloc /proc/vmallocinfo
> 0xffffc9001f181000-0xffffc9001fb82000 10489856 kvm_kvzalloc+0x25/0x30
> [kvm] pages=2560 vmalloc vpages N1=2560
> 0xffffc9001fb82000-0xffffc9001fb88000   24576 kvm_kvzalloc+0x25/0x30
> [kvm] pages=5 vmalloc N1=5
> 0xffffc9001fb88000-0xffffc9001fb8e000   24576 kvm_kvzalloc+0x25/0x30
> [kvm] pages=5 vmalloc N1=5
> 0xffffc9001fb8e000-0xffffc9001fb92000   16384 kvm_kvzalloc+0x25/0x30
> [kvm] pages=3 vmalloc N1=3
> 0xffffc90020854000-0xffffc90021255000 10489856 kvm_kvzalloc+0x25/0x30
> [kvm] pages=2560 vmalloc vpages N1=2560
> 0xffffc90021255000-0xffffc9002125b000   24576 kvm_kvzalloc+0x25/0x30
> [kvm] pages=5 vmalloc N1=5
> 0xffffc9002125b000-0xffffc90021261000   24576 kvm_kvzalloc+0x25/0x30
> [kvm] pages=5 vmalloc N1=5
> 0xffffc90021261000-0xffffc90021265000   16384 kvm_kvzalloc+0x25/0x30
> [kvm] pages=3 vmalloc N1=3
> 0xffffc9002616e000-0xffffc90026172000   16384 kvm_kvzalloc+0x25/0x30
> [kvm] pages=3 vmalloc N1=3
> 0xffffc90026172000-0xffffc90026176000   16384 kvm_kvzalloc+0x25/0x30
> [kvm] pages=3 vmalloc N1=3
> 0xffffc90026176000-0xffffc90026179000   12288 kvm_kvzalloc+0x25/0x30
> [kvm] pages=2 vmalloc N1=2
> 0xffffc900261a9000-0xffffc900261ad000   16384 kvm_kvzalloc+0x25/0x30
> [kvm] pages=3 vmalloc N1=3
> 0xffffc900261ad000-0xffffc900261b1000   16384 kvm_kvzalloc+0x25/0x30
> [kvm] pages=3 vmalloc N1=3
> 0xffffc900261b1000-0xffffc900261b4000   12288 kvm_kvzalloc+0x25/0x30
> [kvm] pages=2 vmalloc N1=2
> 0xffffc900280fe000-0xffffc900286ff000 6295552 kvm_kvzalloc+0x25/0x30 [kvm]
> pages=1536 vmalloc vpages N1=1536
> 0xffffc900286ff000-0xffffc90028d00000 6295552 kvm_kvzalloc+0x25/0x30 [kvm]
> pages=1536 vmalloc vpages N1=1536
> 0xffffc90028d87000-0xffffc90028d90000   36864 kvm_kvzalloc+0x25/0x30
> [kvm] pages=8 vmalloc N1=8
> 0xffffc90028d9c000-0xffffc90028da5000   36864 kvm_kvzalloc+0x25/0x30
> [kvm] pages=8 vmalloc N1=8
> 
> 
> 3. I found the first bad commit by 'git biscet'
> 
> linux-arei:/mnt/sdb/gonglei/opensource/qemu # git bisect bad
> 6410848bec38089424d54a6a8f10d4cf77182b5d is the first bad commit
> commit 6410848bec38089424d54a6a8f10d4cf77182b5d
> Author: Paolo Bonzini <address@hidden>
> Date:   Thu Jun 18 18:30:16 2015 +0200
> 
>     target-i386: register a separate KVM address space including SMRAM
> regions
> 
>     Signed-off-by: Paolo Bonzini <address@hidden>
> 
> :040000 040000 b2435d7cd0829e6416b316f1ae2856e6f7b0023d
> 1acb81aecaf50f2d313b33f2b61a24f7f0bd6f07 M      target-i386
> linux-PsHdkO:/mnt/sdb/gonglei/opensource/qemu #
> 
> 
> Any ideas about this change? Do we really need to trigger two times memory
> region allocation?
> 
> 
> Thanks,
> -Gonglei
>