Re: [Qemu-devel] [PATCH COLO-Frame v12 18/38] COLO: Flush PVM's cached RAM into SVM's memory

[Hailiang Zhang]

On 2015/12/15 19:07, Changlong Xie wrote:
> On 12/15/2015 04:22 PM, zhanghailiang wrote:
> > During the time of VM's running, PVM may dirty some pages, we will transfer
> > PVM's dirty pages to SVM and store them into SVM's RAM cache at next checkpoint
> > time. So, the content of SVM's RAM cache will always be some with PVM's memory
> "some" => "same"

Fixed, thanks.

> Thanks
>      -Xie
> > after checkpoint.
> >
> > Instead of flushing all content of PVM's RAM cache into SVM's MEMORY,
> > we do this in a more efficient way:
> > Only flush any page that dirtied by PVM since last checkpoint.
> > In this way, we can ensure SVM's memory same with PVM's.
> >
> > Besides, we must ensure flush RAM cache before load device state.
> >
> > Signed-off-by: zhanghailiang <address@hidden>
> > Signed-off-by: Li Zhijian <address@hidden>
> > Signed-off-by: Gonglei <address@hidden>
> > Reviewed-by: Dr. David Alan Gilbert <address@hidden>
> > ---
> > v12:
> > - Add a trace point in the end of colo_flush_ram_cache() (Dave's suggestion)
> > - Add Reviewed-by tag
> > v11:
> > - Move the place of 'need_flush' (Dave's suggestion)
> > - Remove unused 'DPRINTF("Flush ram_cache\n")'
> > v10:
> > - trace the number of dirty pages that be received.
> >
> > Signed-off-by: zhanghailiang <address@hidden>
> > ---
> >   include/migration/migration.h |  1 +
> >   migration/colo.c              |  2 --
> >   migration/ram.c               | 38 ++++++++++++++++++++++++++++++++++++++
> >   trace-events                  |  2 ++
> >   4 files changed, 41 insertions(+), 2 deletions(-)
> >
> > diff --git a/include/migration/migration.h b/include/migration/migration.h
> > index e41372d..221176b 100644
> > --- a/include/migration/migration.h
> > +++ b/include/migration/migration.h
> > @@ -336,4 +336,5 @@ PostcopyState postcopy_state_set(PostcopyState new_state);
> >   /* ram cache */
> >   int colo_init_ram_cache(void);
> >   void colo_release_ram_cache(void);
> > +void colo_flush_ram_cache(void);
> >   #endif
> > diff --git a/migration/colo.c b/migration/colo.c
> > index a4d49ff..e40cdb9 100644
> > --- a/migration/colo.c
> > +++ b/migration/colo.c
> > @@ -401,8 +401,6 @@ void *colo_process_incoming_thread(void *opaque)
> >           }
> >           qemu_mutex_unlock_iothread();
> >
> > -        /* TODO: flush vm state */
> > -
> >           ret = colo_put_cmd(mis->to_src_file, COLO_COMMAND_VMSTATE_LOADED);
> >           if (ret < 0) {
> >               goto out;
> > diff --git a/migration/ram.c b/migration/ram.c
> > index 3d5947b..8ff7f7c 100644
> > --- a/migration/ram.c
> > +++ b/migration/ram.c
> > @@ -2458,6 +2458,7 @@ static int ram_load(QEMUFile *f, void *opaque, int 
> > version_id)
> >        * be atomic
> >        */
> >       bool postcopy_running = postcopy_state_get() >= 
> > POSTCOPY_INCOMING_LISTENING;
> > +    bool need_flush = false;
> >
> >       seq_iter++;
> >
> > @@ -2492,6 +2493,7 @@ static int ram_load(QEMUFile *f, void *opaque, int 
> > version_id)
> >               /* After going into COLO, we should load the Page into colo_cache 
> > */
> >               if (ram_cache_enable) {
> >                   host = colo_cache_from_block_offset(block, addr);
> > +                need_flush = true;
> >               } else {
> >                   host = host_from_ram_block_offset(block, addr);
> >               }
> > @@ -2585,6 +2587,10 @@ static int ram_load(QEMUFile *f, void *opaque, int 
> > version_id)
> >       }
> >
> >       rcu_read_unlock();
> > +
> > +    if (!ret  && ram_cache_enable && need_flush) {
> > +        colo_flush_ram_cache();
> > +    }
> >       DPRINTF("Completed load of VM with exit code %d seq iteration "
> >               "%" PRIu64 "\n", ret, seq_iter);
> >       return ret;
> > @@ -2657,6 +2663,38 @@ void colo_release_ram_cache(void)
> >       rcu_read_unlock();
> >   }
> >
> > +/*
> > + * Flush content of RAM cache into SVM's memory.
> > + * Only flush the pages that be dirtied by PVM or SVM or both.
> > + */
> > +void colo_flush_ram_cache(void)
> > +{
> > +    RAMBlock *block = NULL;
> > +    void *dst_host;
> > +    void *src_host;
> > +    ram_addr_t offset = 0;
> > +
> > +    trace_colo_flush_ram_cache_begin(migration_dirty_pages);
> > +    rcu_read_lock();
> > +    block = QLIST_FIRST_RCU(&ram_list.blocks);
> > +    while (block) {
> > +        ram_addr_t ram_addr_abs;
> > +        offset = migration_bitmap_find_dirty(block, offset, &ram_addr_abs);
> > +        migration_bitmap_clear_dirty(ram_addr_abs);
> > +        if (offset >= block->used_length) {
> > +            offset = 0;
> > +            block = QLIST_NEXT_RCU(block, next);
> > +        } else {
> > +            dst_host = block->host + offset;
> > +            src_host = block->colo_cache + offset;
> > +            memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
> > +        }
> > +    }
> > +    rcu_read_unlock();
> > +    trace_colo_flush_ram_cache_end();
> > +    assert(migration_dirty_pages == 0);
> > +}
> > +
> >   static SaveVMHandlers savevm_ram_handlers = {
> >       .save_live_setup = ram_save_setup,
> >       .save_live_iterate = ram_save_iterate,
> > diff --git a/trace-events b/trace-events
> > index 39fdd8d..7f76029 100644
> > --- a/trace-events
> > +++ b/trace-events
> > @@ -1264,6 +1264,8 @@ migration_throttle(void) ""
> >   ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x"
> >   ram_postcopy_send_discard_bitmap(void) ""
> >   ram_save_queue_pages(const char *rbname, size_t start, size_t len) "%s: start: %zx 
> > len: %zx"
> > +colo_flush_ram_cache_begin(uint64_t dirty_pages) "dirty_pages %" PRIu64
> > +colo_flush_ram_cache_end(void) ""
> >
> >   # hw/display/qxl.c
> >   disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
>
>
>
> .