Re: [Qemu-devel] segmentation fault in qemu-kvm-0.14.0

[Peter Lieven]

Am 09.03.2011 um 08:26 schrieb Stefan Weil:

> Am 08.03.2011 23:53, schrieb Peter Lieven:
>> Hi,
>> 
>> during testing of qemu-kvm-0.14.0 i can reproduce the following segfault. i 
>> have seen similar crash already in 0.13.0, but had no time to debug.
>> my guess is that this segfault is related to the threaded vnc server which 
>> was introduced in qemu 0.13.0. the bug is only triggerable if a vnc
>> client is attached. it might also be connected to a resolution change in the 
>> guest. i have a backtrace attached. the debugger is still running if someone
>> needs more output
>> 
>> Reading symbols from /usr/local/bin/qemu-system-x86_64...done.
>> (gdb) r -net tap,vlan=141,script=no,downscript=no,ifname=tap0 -net 
>> nic,vlan=141,model=rtl8139,macaddr=52:54:00:ff:00:93 -drive 
>> format=host_device,file=/dev/mapper/iqn.2001-05.co
>> m.equallogic:0-8a0906-e6b70e107-e87000e7acf4d4e5-lieven-winxp-r17453,if=ide,boot=on,cache=none,aio=native
>>  -m 1024 -monitor tcp:0:4001,server,nowait -vnc :1 -name 'lieven-winxp-te
>> st' -boot order=c,menu=on -k de -pidfile /var/run/qemu/vm-265.pid -mem-path 
>> /hugepages -mem-prealloc -cpu qemu64,model_id='Intel(R) Xeon(R) CPU E5640 @ 
>> 2.67GHz',-n
>> x -rtc base=localtime,clock=vm -vga cirrus -usb -usbdevice tablet
>> Starting program: /usr/local/bin/qemu-system-x86_64 -net 
>> tap,vlan=141,script=no,downscript=no,ifname=tap0 -net 
>> nic,vlan=141,model=rtl8139,macaddr=52:54:00:ff:00:93 -drive format
>> =host_device,file=/dev/mapper/iqn.2001-05.com.equallogic:0-8a0906-e6b70e107-e87000e7acf4d4e5-lieven-winxp-r17453,if=ide,boot=on,cache=none,aio=native
>>  -m 1024 -monitor tcp:0:4001,
>> server,nowait -vnc :1 -name 'lieven-winxp-test' -boot order=c,menu=on -k de 
>> -pidfile /var/run/qemu/vm-265.pid -mem-path /hugepages -mem-prealloc -cpu 
>> qemu64,model_id='Intel(R
>> ) Xeon(R) CPU E5640 @ 2.67GHz',-nx -rtc base=localtime,clock=vm -vga cirrus 
>> -usb -usbdevice tablet
>> [Thread debugging using libthread_db enabled]
>> 
>> [New Thread 0x7ffff694e700 (LWP 29042)]
>> [New Thread 0x7ffff6020700 (LWP 29043)]
>> [New Thread 0x7ffff581f700 (LWP 29074)]
>> [Thread 0x7ffff581f700 (LWP 29074) exited]
>> [New Thread 0x7ffff581f700 (LWP 29124)]
>> [Thread 0x7ffff581f700 (LWP 29124) exited]
>> [New Thread 0x7ffff581f700 (LWP 29170)]
>> [Thread 0x7ffff581f700 (LWP 29170) exited]
>> [New Thread 0x7ffff581f700 (LWP 29246)]
>> [Thread 0x7ffff581f700 (LWP 29246) exited]
>> [New Thread 0x7ffff581f700 (LWP 29303)]
>> [Thread 0x7ffff581f700 (LWP 29303) exited]
>> [New Thread 0x7ffff581f700 (LWP 29349)]
>> [Thread 0x7ffff581f700 (LWP 29349) exited]
>> [New Thread 0x7ffff581f700 (LWP 29399)]
>> [Thread 0x7ffff581f700 (LWP 29399) exited]
>> [New Thread 0x7ffff581f700 (LWP 29471)]
>> [Thread 0x7ffff581f700 (LWP 29471) exited]
>> [New Thread 0x7ffff581f700 (LWP 29521)]
>> [Thread 0x7ffff581f700 (LWP 29521) exited]
>> [New Thread 0x7ffff581f700 (LWP 29593)]
>> [Thread 0x7ffff581f700 (LWP 29593) exited]
>> [New Thread 0x7ffff581f700 (LWP 29703)]
>> [Thread 0x7ffff581f700 (LWP 29703) exited]
>> 
>> Program received signal SIGSEGV, Segmentation fault.
>> 0x0000000000000000 in ?? ()
>> (gdb)
>> 
>> (gdb) thread apply all bt full
>> 
>> Thread 3 (Thread 0x7ffff6020700 (LWP 29043)):
>> #0 0x00007ffff79c385c in pthread_cond_wait@@GLIBC_2.3.2 ()
>> from /lib/libpthread.so.0
>> No symbol table info available.
>> #1 0x00000000004d3ae1 in qemu_cond_wait (cond=0x1612d50, mutex=0x1612d80)
>> at qemu-thread.c:133
>> err = 0
>> __func__ = "qemu_cond_wait"
>> #2 0x00000000004d2b39 in vnc_worker_thread_loop (queue=0x1612d50)
>> at ui/vnc-jobs-async.c:198
>> job = 0x7ffff058cd20
>> entry = 0x0
>> tmp = 0x0
>> vs = {csock = -1, ds = 0x15cb380, dirty = {{0, 0, 0, 0,
>> 0} <repeats 2048 times>}, vd = 0x1607ff0, need_update = 0,
>> force_update = 0, features = 243, absolute = 0, last_x = 0,
>> last_y = 0, client_width = 0, client_height = 0, vnc_encoding = 7,
>> major = 0, minor = 0, challenge = '\000' <repeats 15 times>,
>> info = 0x0, output = {capacity = 3194, offset = 2723,
>> buffer = 0x1fbbfd0 ""}, input = {capacity = 0, offset = 0,
>> buffer = 0x0}, write_pixels = 0x4c4bc9 <vnc_write_pixels_generic>,
>> clientds = {flags = 0 '\000', width = 720, height = 400,
>> linesize = 2880,
>> data = 0x7ffff6021010 <Address 0x7ffff6021010 out of bounds>,
>> pf = {bits_per_pixel = 32 ' ', bytes_per_pixel = 4 '\004',
>> depth = 24 '\030', rmask = 0, gmask = 0, bmask = 0, amask = 0,
>> rshift = 16 '\020', gshift = 8 '\b', bshift = 0 '\000',
>> ashift = 24 '\030', rmax = 255 '\377', gmax = 255 '\377',
>> bmax = 255 '\377', amax = 255 '\377', rbits = 8 '\b',
>> gbits = 8 '\b', bbits = 8 '\b', abits = 8 '\b'}},
>> audio_cap = 0x0, as = {freq = 0, nchannels = 0, fmt = AUD_FMT_U8,
>> endianness = 0}, read_handler = 0, read_handler_expect = 0,
>> modifiers_state = '\000' <repeats 255 times>, led = 0x0,
>> abort = false, output_mutex = {lock = {__data = {__lock = 0,
>> __count = 0, __owner = 0, __nusers = 0, __kind = 0,
>> __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
>> __size = '\000' <repeats 39 times>, __align = 0}}, tight = {
>> type = 7, quality = 255 '\377', compression = 9 '\t',
>> pixel24 = 1 '\001', tight = {capacity = 3146708, offset = 1376,
>> buffer = 0x7ffff4684010 ""}, tmp = {capacity = 3194,
>> offset = 2672, buffer = 0x1fbbfd0 ""}, zlib = {
>> capacity = 141451, offset = 0,
>> buffer = 0x1805da0 
>> "B--G\317\253\031=\257f\364\274\232\321\363jF\317\253\241\326y5"}, gradient 
>> = {capacity = 0, offset = 0, buffer = 0x0},
>> levels = {9, 9, 9, 0}, stream = {{
>> next_in = 0x7ffff4684460 
>> "\002\003\002\003\002\002\002\002\002\002\002\002\002", avail_in = 0, 
>> total_in = 9048240,
>> next_out = 0x1805df6 
>> "d\276\345c\363\216\237\212\377\314\003\236\246\\$\361\025~\032\311\232\067q&_$\231\251y\262*!\231\067\236\067\363\214\347\315\240\361\221,\274T\257\221\314\333\341\251\362R\373\232_\311a\272\002\061sgt\317\030\332\262~\300\063\267]\307\267\343\033",
>>  avail_out = 141365,
>> total_out = 1034664, msg = 0x0, state = 0x16ea550,
>> zalloc = 0x4ca2d8 <vnc_zlib_zalloc>,
>> zfree = 0x4ca315 <vnc_zlib_zfree>, opaque = 0x7ffff6015920,
>> data_type = 0, adler = 1285581469, reserved = 0}, {
>> next_in = 0x7ffff4684022 "\002\002", avail_in = 0,
>> total_in = 46491,
>> next_out = 0x1805da8 "\257f\364\274\232\321\363jF\317\253\241\326y5", 
>> avail_out = 141443, total_out = 9905, msg = 0x0, state = 0x17a3f00,
>> zalloc = 0x4ca2d8 <vnc_zlib_zalloc>,
>> zfree = 0x4ca315 <vnc_zlib_zfree>, opaque = 0x7ffff6015920,
>> data_type = 0, adler = 2415253306, reserved = 0}, {
>> next_in = 0x7ffff4684570 "\017(`", avail_in = 0,
>> total_in = 2188024,
>> next_out = 0x1805dbd 
>> "z\222\024#\233O\214g\341\352\211/U\310s\017\361\245\020\262\343\211TO\222\371\304\207\fryK|\251E\222\343\311+\237\351`>\355\312sR\265\320\272~\001",
>>  avail_out = 141422, total_out = 100512, msg = 0x0,
>> state = 0x1682950, zalloc = 0x4ca2d8 <vnc_zlib_zalloc>,
>> zfree = 0x4ca315 <vnc_zlib_zfree>, opaque = 0x7ffff6015920,
>> data_type = 0, adler = 3999694267, reserved = 0}, {
>> next_in = 0x0, avail_in = 0, total_in = 0, next_out = 0x0,
>> avail_out = 0, total_out = 0, msg = 0x0, state = 0x0,
>> zalloc = 0, zfree = 0, opaque = 0x0, data_type = 0, adler = 0,
>> reserved = 0}}}, zlib = {zlib = {capacity = 0, offset = 0,
>> buffer = 0x0}, tmp = {capacity = 0, offset = 0, buffer = 0x0},
>> stream = {next_in = 0x0, avail_in = 0, total_in = 0,
>> next_out = 0x0, avail_out = 0, total_out = 0, msg = 0x0,
>> state = 0x0, zalloc = 0, zfree = 0, opaque = 0x0, data_type = 0,
>> adler = 0, reserved = 0}, level = 0}, hextile = {
>> send_tile = 0x4cd21d <send_hextile_tile_generic_32>},
>> mouse_mode_notifier = {notify = 0x80, node = {tqe_next = 0x1612da8,
>> tqe_prev = 0x7ffff6020700}}, next = {tqe_next = 0x7ffff6020700,
>> tqe_prev = 0x7ffff7ffe128}}
>> n_rectangles = 40
>> saved_offset = 2
>> flush = true
>> #3 0x00000000004d302c in vnc_worker_thread (arg=0x1612d50)
>> at ui/vnc-jobs-async.c:302
>> queue = 0x1612d50
>> #4 0x00007ffff79be9ca in start_thread () from /lib/libpthread.so.0
>> No symbol table info available.
>> #5 0x00007ffff6c3970d in clone () from /lib/libc.so.6
>> No symbol table info available.
>> #6 0x0000000000000000 in ?? ()
>> No symbol table info available.
>> 
>> Thread 2 (Thread 0x7ffff694e700 (LWP 29042)):
>> #0 0x00007ffff6c31197 in ioctl () from /lib/libc.so.6
>> No symbol table info available.
>> #1 0x0000000000434c12 in kvm_run (env=0x118c2e0)
>> at /usr/src/qemu-kvm-0.14.0/qemu-kvm.c:582
>> r = 0
>> kvm = 0x1172538
>> run = 0x7ffff7fc7000
>> fd = 15
>> #2 0x0000000000435ed4 in kvm_cpu_exec (env=0x118c2e0)
>> at /usr/src/qemu-kvm-0.14.0/qemu-kvm.c:1233
>> r = 0
>> #3 0x00000000004364f7 in kvm_main_loop_cpu (env=0x118c2e0)
>> at /usr/src/qemu-kvm-0.14.0/qemu-kvm.c:1419
>> run_cpu = 1
>> #4 0x0000000000436640 in ap_main_loop (_env=0x118c2e0)
>> at /usr/src/qemu-kvm-0.14.0/qemu-kvm.c:1466
>> env = 0x118c2e0
>> signals = {__val = {18446744067267100671,
>> 18446744073709551615 <repeats 15 times>}}
>> data = 0x0
>> #5 0x00007ffff79be9ca in start_thread () from /lib/libpthread.so.0
>> No symbol table info available.
>> #6 0x00007ffff6c3970d in clone () from /lib/libc.so.6
>> No symbol table info available.
>> #7 0x0000000000000000 in ?? ()
>> No symbol table info available.
>> 
>> Thread 1 (Thread 0x7ffff7ff0700 (LWP 29038)):
>> #0 0x0000000000000000 in ?? ()
>> No symbol table info available.
>> #1 0x000000000041d669 in main_loop_wait (nonblocking=0)
>> at /usr/src/qemu-kvm-0.14.0/vl.c:1388
>> pioh = 0x1652870
>> ioh = 0x1613330
>> rfds = {fds_bits = {0 <repeats 16 times>}}
>> wfds = {fds_bits = {1048576, 0 <repeats 15 times>}}
>> xfds = {fds_bits = {0 <repeats 16 times>}}
>> ret = 1
>> nfds = 21
>> tv = {tv_sec = 0, tv_usec = 999998}
>> timeout = 1000
>> #2 0x0000000000436a4a in kvm_main_loop ()
>> at /usr/src/qemu-kvm-0.14.0/qemu-kvm.c:1589
>> mask = {__val = {268443712, 0 <repeats 15 times>}}
>> sigfd = 19
>> #3 0x000000000041d785 in main_loop () at /usr/src/qemu-kvm-0.14.0/vl.c:1429
>> r = 0
>> #4 0x000000000042166a in main (argc=33, argv=0x7fffffffe658,
>> envp=0x7fffffffe768) at /usr/src/qemu-kvm-0.14.0/vl.c:3201
>> gdbstub_dev = 0x0
>> i = 64
>> snapshot = 0
>> linux_boot = 0
>> icount_option = 0x0
>> initrd_filename = 0x0
>> kernel_filename = 0x0
>> kernel_cmdline = 0x5e57a3 ""
>> boot_devices = "c\000d", '\000' <repeats 29 times>
>> ds = 0x15cb380
>> dcl = 0x0
>> cyls = 0
>> heads = 0
>> secs = 0
>> translation = 0
>> hda_opts = 0x0
>> opts = 0x1171a80
>> olist = 0x7fffffffe3f0
>> optind = 33
>> optarg = 0x7fffffffebce "tablet"
>> loadvm = 0x0
>> machine = 0x962cc0
>> cpu_model = 0x7fffffffeb51 "qemu64,model_id=Intel(R) Xeon(R) CPU", ' ' 
>> <repeats 11 times>, "E5640 @ 2.67GHz,-nx"
>> tb_size = 0
>> pid_file = 0x7fffffffeb10 "/var/run/qemu/vm-265.pid"
>> incoming = 0x0
>> show_vnc_port = 0
>> defconfig = 1
>> (gdb) info threads
>> 3 Thread 0x7ffff6020700 (LWP 29043) 0x00007ffff79c385c in 
>> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
>> 2 Thread 0x7ffff694e700 (LWP 29042) 0x00007ffff6c31197 in ioctl ()
>> from /lib/libc.so.6
>> * 1 Thread 0x7ffff7ff0700 (LWP 29038) 0x0000000000000000 in ?? ()
>> (gdb)
>> 
>> 
>> Help appreciated!
>> 
>> Thanks,
>> Peter
> 

hi stefan,

> Hi Peter,
> 
> did you apply this patch which fixes one of the known vnc problems
> (but is still missing in qemu git master):
> 
> http://lists.nongnu.org/archive/html/qemu-devel/2011-03/msg00256.html

it seems that this patch is not compatible with 0.14.0. 0.14.0 uses
vnc_set_bits and vnc_and_bits and no bitmap functions.
does the problem you observed affect these functions as well?
then we might need to backport here. my intention is to get 0.14.0 stable
for daily operation.

> 
> Then you can read this thread:
> 
> http://lists.nongnu.org/archive/html/qemu-devel/2011-03/msg00313.html

you mention a patch that was on qemu-devel recently. which patch do you refer
to?

> 
> And finally the following modifications of ui/vnc.c might help to see
> whether you experience the same kind of crash as I get here in
> my environment. They add assertions for bad memory access
> which occurs sometimes when a vnc client-server connection exists and
> the screen is refreshed after a resolution change.
> The code line with the //~ comment also includes a fix which
> works for me.

i added these checks and also turned the //~ comment into an assert. I will
now run some tests again.

Thanks,
Peter

> 
> Regards,
> Stefan W.
> 
> @@ -2382,6 +2384,10 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
>     int y;
>     uint8_t *guest_row;
>     uint8_t *server_row;
> +
> +    size_t guest_size = vd->guest.ds->linesize * vd->guest.ds->height;
> +    size_t server_size = vd->server->linesize * vd->server->height;
> +
>     int cmp_bytes;
>     VncState *vs;
>     int has_dirty = 0;
> @@ -2399,11 +2405,15 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
>      * Update server dirty map.
>      */
>     cmp_bytes = 16 * ds_get_bytes_per_pixel(vd->ds);
> +    if (cmp_bytes > vd->ds->surface->linesize) {
> +        //~ fix crash: cmp_bytes = vd->ds->surface->linesize;
> +    }
>     guest_row  = vd->guest.ds->data;
>     server_row = vd->server->data;
>     for (y = 0; y < vd->guest.ds->height; y++) {
>         if (!bitmap_empty(vd->guest.dirty[y], VNC_DIRTY_BITS)) {
>             int x;
> +            size_t size_offset = 0;
>             uint8_t *guest_ptr;
>             uint8_t *server_ptr;
> 
> @@ -2412,6 +2422,9 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
> 
>             for (x = 0; x < vd->guest.ds->width;
>                     x += 16, guest_ptr += cmp_bytes, server_ptr += cmp_bytes) 
> {
> +                assert(size_offset + cmp_bytes <= guest_size);
> +                assert(size_offset + cmp_bytes <= server_size);
> +                size_offset += cmp_bytes;
>                 if (!test_and_clear_bit((x / 16), vd->guest.dirty[y]))
>                     continue;
>                 if (memcmp(server_ptr, guest_ptr, cmp_bytes) == 0)
> @@ -2427,6 +2440,10 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
>         }
>         guest_row  += ds_get_linesize(vd->ds);
>         server_row += ds_get_linesize(vd->ds);
> +        assert(guest_size >= ds_get_linesize(vd->ds));
> +        guest_size -= ds_get_linesize(vd->ds);
> +        assert(server_size >= ds_get_linesize(vd->ds));
> +        server_size -= ds_get_linesize(vd->ds);
>     }
>     return has_dirty;
> }
>