[Qemu-devel] [Bug 1570134] Re: While committing snapshot qemu crashes with SIGABRT

[Matthew Schumacher]

It still fails with ext4:

#0  0x00007fbaa12b33f8 in raise () at /lib64/libc.so.6
#1  0x00007fbaa12b4ffa in abort () at /lib64/libc.so.6
#2  0x00007fbaa12abc17 in __assert_fail_base () at /lib64/libc.so.6
#3  0x00007fbaa12abcc2 in  () at /lib64/libc.so.6
#4  0x00005646b990f926 in mirror_run (s=0x5646bc50f480) at block/mirror.c:335
        next_sector = 36659200
        next_chunk = 286400
        nb_chunks = 80
        end = 209715200
        sectors_per_chunk = 128
        source = 0x5646bcb70000
        sector_num = 36648960
        delay_ns = 0
        delay_ns = 0
        cnt = 15360
        should_complete = <optimized out>
        s = 0x5646bc50f480
        data = <optimized out>
        bs = 0x5646bcb70000
        sector_num = <optimized out>
        end = <optimized out>
        length = <optimized out>
        last_pause_ns = <optimized out>
        bdi = {cluster_size = 65536, vm_state_offset = 107374182400, is_dirty = 
false, unallocated_blocks_are_zero = true, can_write_zeroes_with_unmap = true, 
needs_compressed_writes = false}
        backing_filename = "\000"
        ret = <optimized out>
        n = 1048576
        target_cluster_size = <optimized out>
        __PRETTY_FUNCTION__ = "mirror_run"
#5  0x00005646b990f926 in mirror_run (opaque=0x5646bc50f480) at 
block/mirror.c:613
        delay_ns = 0
        cnt = 15360
        should_complete = <optimized out>
        s = 0x5646bc50f480
        data = <optimized out>
        bs = 0x5646bcb70000
        sector_num = <optimized out>
        end = <optimized out>
        length = <optimized out>
        last_pause_ns = <optimized out>
        bdi = {cluster_size = 65536, vm_state_offset = 107374182400, is_dirty = 
false, unallocated_blocks_are_zero = true, can_write_zeroes_with_unmap = true, 
needs_compressed_writes = false}
        backing_filename = "\000"
        ret = <optimized out>
        n = 1048576
        target_cluster_size = <optimized out>
        __PRETTY_FUNCTION__ = "mirror_run"
#6  0x00005646b997568a in coroutine_trampoline (i0=<optimized out>, 
i1=<optimized out>) at util/coroutine-ucontext.c:78
        self = 0x5646bc5115b0
        co = 0x5646bc5115b0
#7  0x00007fbaa12c8560 in __start_context () at /lib64/libc.so.6
#8  0x00005646bd2b98b0 in  ()
#9  0x0000000000000000 in  ()

qemu-system-x86_64: block/mirror.c:335: mirror_iteration: Assertion
`hbitmap_next == next_sector' failed.


I can't seem to get stable snapshotting and blockpull with a loaded VM.

Interestingly enough, the last command libvirt passes to qemu is:

2016-04-14 20:47:58.196+0000: 18932: debug : qemuMonitorJSONCommandWithFd:294 : 
Send command '{"execute":"query-block-jobs","id":"libvirt-69"}' for write with 
FD -1
2016-04-14 20:47:58.196+0000: 18932: info : qemuMonitorSend:1005 : 
QEMU_MONITOR_SEND_MSG: mon=0x7f1874001a30 
msg={"execute":"query-block-jobs","id":"libvirt-69"}
2016-04-14 20:47:58.197+0000: 18929: info : qemuMonitorIOWrite:529 : 
QEMU_MONITOR_IO_WRITE: mon=0x7f1874001a30 
buf={"execute":"query-block-jobs","id":"libvirt-69"}


Odd that it would SIGABRT on a smile query-block-jobs.

Even more interesting is that it crashes on the first or second or third
snapshot/block-commit cycle when using EXT4, but would sometimes go for
30-40 cycles on ZFS.

Any ideas?  I'm certainly willing to test and help in any way I can.

Thanks!

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1570134

Title:
  While committing snapshot qemu crashes with SIGABRT

Status in QEMU:
  New

Bug description:
  Information:

  OS: Slackware64-Current
  Compiled with: gcc version 5.3.0 (GCC)  / glibc 2.23
  Compiled using: 

  CFLAGS="-O2 -fPIC" \
  CXXFLAGS="-O2 -fPIC" \
  LDFLAGS="-L/usr/lib64" \
  ./configure \
    --prefix=/usr \
    --sysconfdir=/etc \
    --localstatedir=/var \
    --libdir=/usr/lib64 \
    --enable-spice \
    --enable-kvm \
    --enable-glusterfs \
    --enable-libiscsi \
    --enable-libusb \
    --target-list=x86_64-softmmu,i386-softmmu \
    --enable-debug

  Source: qemu-2.5.1.tar.bz2

  Running as:

  /usr/bin/qemu-system-x86_64 -name test1,debug-threads=on -S -machine
  pc-1.1,accel=kvm,usb=off -m 4096 -realtime mlock=off -smp
  2,sockets=2,cores=1,threads=1 -uuid
  4b30ec13-6609-4a56-8731-d400c38189ef -no-user-config -nodefaults
  -chardev
  
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-4-test1/monitor.sock,server,nowait
  -mon chardev=charmonitor,id=monitor,mode=control -rtc
  base=localtime,clock=vm,driftfix=slew -global kvm-
  pit.lost_tick_policy=discard -no-shutdown -boot strict=on -device
  piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive
  file=/datastore/vm/test1/test1.img,format=qcow2,if=none,id=drive-
  virtio-disk0 -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive
  =drive-virtio-disk0,id=virtio-disk0,bootindex=2 -drive if=none,id
  =drive-ide0-1-0,readonly=on -device ide-cd,bus=ide.1,unit=0,drive
  =drive-ide0-1-0,id=ide0-1-0,bootindex=1 -netdev
  tap,fd=23,id=hostnet0,vhost=on,vhostfd=25 -device virtio-net
  pci,netdev=hostnet0,id=net0,mac=52:54:00:66:2e:0f,bus=pci.0,addr=0x3
  -vnc 0.0.0.0:0 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device
  virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on

  File system:  zfs v0.6.5.6

  While running: 
  virsh blockcommit test1 vda --active --pivot --verbose

  VM running very heavy IO load

  GDB reporting:

  #0  0x00007fd80132c3f8 in raise () at /lib64/libc.so.6
  #1  0x00007fd80132dffa in abort () at /lib64/libc.so.6
  #2  0x00007fd801324c17 in __assert_fail_base () at /lib64/libc.so.6
  #3  0x00007fd801324cc2 in  () at /lib64/libc.so.6
  #4  0x000055d9918d7572 in bdrv_replace_in_backing_chain (old=0x55d993ed9c10, 
new=0x55d9931ccc10) at block.c:2096
          __PRETTY_FUNCTION__ = "bdrv_replace_in_backing_chain"
  #5  0x000055d991911869 in mirror_exit (job=0x55d993fef830, 
opaque=0x55d999bbefe0) at block/mirror.c:376
          to_replace = 0x55d993ed9c10
          s = 0x55d993fef830
          data = 0x55d999bbefe0
          replace_aio_context = <optimized out>
          src = 0x55d993ed9c10
  #6  0x000055d9918da1dc in block_job_defer_to_main_loop_bh 
(opaque=0x55d9940ce850) at blockjob.c:481
          data = 0x55d9940ce850
          aio_context = 0x55d9931a2610
  #7  0x000055d9918d014b in aio_bh_poll (address@hidden) at async.c:92
          bh = <optimized out>
          bhp = <optimized out>
          next = 0x55d99440f910
          ret = 1
  #8  0x000055d9918dc8c0 in aio_dispatch (ctx=0x55d9931a2610) at aio-posix.c:305
          node = <optimized out>
          progress = false
  #9  0x000055d9918d000e in aio_ctx_dispatch (source=<optimized out>, 
callback=<optimized out>, user_data=<optimized out>) at async.c:231
          ctx = <optimized out>
  #10 0x00007fd8037cf787 in g_main_context_dispatch () at 
/usr/lib64/libglib-2.0.so.0
  #11 0x000055d9918db03b in main_loop_wait () at main-loop.c:211
          context = 0x55d9931a3200
          pfds = <optimized out>
          ret = 0
          spin_counter = 1
          ret = 0
          timeout = 4294967295
          timeout_ns = <optimized out>
  #12 0x000055d9918db03b in main_loop_wait (timeout=<optimized out>) at 
main-loop.c:256
          ret = 0
          spin_counter = 1
          ret = 0
          timeout = 4294967295
          timeout_ns = <optimized out>
  #13 0x000055d9918db03b in main_loop_wait (nonblocking=<optimized out>) at 
main-loop.c:504
          ret = 0
          timeout = 4294967295
          timeout_ns = <optimized out>
  #14 0x000055d991679cc4 in main () at vl.c:1923
          nonblocking = <optimized out>
          last_io = 2
          i = <optimized out>
          snapshot = <optimized out>
          linux_boot = <optimized out>
          initrd_filename = <optimized out>
          kernel_filename = <optimized out>
          kernel_cmdline = <optimized out>
          boot_order = <optimized out>
          boot_once = <optimized out>
          ds = <optimized out>
          cyls = <optimized out>
          heads = <optimized out>
          secs = <optimized out>
          translation = <optimized out>
          hda_opts = <optimized out>
          opts = <optimized out>
          machine_opts = <optimized out>
          icount_opts = <optimized out>
          olist = <optimized out>
          optind = 49
          optarg = 0x7fffc6d27f43 "timestamp=on"
          loadvm = <optimized out>
          machine_class = 0x55d993194d10
          cpu_model = <optimized out>
          vga_model = 0x0
          qtest_chrdev = <optimized out>
          qtest_log = <optimized out>
          pid_file = <optimized out>
          incoming = <optimized out>
          defconfig = <optimized out>
          userconfig = false
          log_mask = <optimized out>
          log_file = <optimized out>
          trace_events = <optimized out>
          trace_file = <optimized out>
          maxram_size = <optimized out>
          ram_slots = <optimized out>
          vmstate_dump_file = <optimized out>
          main_loop_err = 0x0
          err = 0x0
          __func__ = "main"
  #15 0x000055d991679cc4 in main (argc=<optimized out>, argv=<optimized out>, 
envp=<optimized out>) at vl.c:4699
          i = <optimized out>
          snapshot = <optimized out>
          linux_boot = <optimized out>
          initrd_filename = <optimized out>
          kernel_filename = <optimized out>
          kernel_cmdline = <optimized out>
          boot_order = <optimized out>
          boot_once = <optimized out>
          ds = <optimized out>
          cyls = <optimized out>
          heads = <optimized out>
          secs = <optimized out>
          translation = <optimized out>
          hda_opts = <optimized out>
          opts = <optimized out>
          machine_opts = <optimized out>
          icount_opts = <optimized out>
          olist = <optimized out>
          optind = 49
          optarg = 0x7fffc6d27f43 "timestamp=on"
          loadvm = <optimized out>
          machine_class = 0x55d993194d10
          cpu_model = <optimized out>
          vga_model = 0x0
          qtest_chrdev = <optimized out>
          qtest_log = <optimized out>
          pid_file = <optimized out>
          incoming = <optimized out>
          defconfig = <optimized out>
          userconfig = false
          log_mask = <optimized out>
          log_file = <optimized out>
          trace_events = <optimized out>
          trace_file = <optimized out>
          maxram_size = <optimized out>
          ram_slots = <optimized out>
          vmstate_dump_file = <optimized out>
          main_loop_err = 0x0
          err = 0x0
          __func__ = "main"


  I can reproduce this at will, and can provide more information per a
  dev's request.

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1570134/+subscriptions