Re: [Qemu-devel] [PATCH] Add warmup phase for live migration of large memory apps

[Shribman, Aidan]

> On Wed, May 11, 2011 at 8:58 AM, Shribman, Aidan 
> <address@hidden> wrote:
> > From: Aidan Shribman <address@hidden>
> >
> > [PATCH] Add warmup phase for live migration of large memory apps
> >
> > By invoking "migrate -w <url>" we initiate a background 
> live-migration
> > transferring of dirty pages continuously until invocation 
> of "migrate_end"
> > which attempts to complete the live migration operation.
> 
> What is the purpose of this patch?  How and when do I use it?
> 

The warmup patch adds none-converging background update of guest memory during 
live-migration such that on request of live-migration completion (via 
"migrate_end" command) we get much faster response. This is especially needed 
when running a payload of large enterprise applications which have high memory 
demands.

> Some nitpicks:
> 
> > @@ -81,6 +83,11 @@ int do_migrate(Monitor *mon, const QDict 
> *qdict, QObject **ret_data)
> >     int blk = qdict_get_try_bool(qdict, "blk", 0);
> >     int inc = qdict_get_try_bool(qdict, "inc", 0);
> >     const char *uri = qdict_get_str(qdict, "uri");
> > +    is_migrate_warmup = qdict_get_try_bool(qdict, "warmup", 0);
> > +
> > +    if (is_migrate_warmup) {
> > +        detach = 1; /* as we need migrate_end to complte */
> 
> s/complte/complete/
> 
> > +       }
> 
> Please follow the coding style and put the closing curly brace on the
> same column as the 'if' statement.
> 
> > +int qemu_savevm_state_warmup(Monitor *mon, QEMUFile *f) {
> > +    int ret = 1;
> 
> 1 is overwritten immediately.

A new patch follows with corrections:

---
From: Aidan Shribman <address@hidden>
Subject: Add warmup phase for live migration of large memory apps
 By invoking "migrate -w <url>" we initiate a background live-migration
 transferring of dirty pages continuously until invocation of "migrate_end"
 which attempts to complete the live migration operation.
 Qemu host: Ubuntu 10.10
 Testing: live migration (with/without warmup phase) tested successfully.

Signed-off-by: Benoit Hudzia <address@hidden>
Signed-off-by: Petter Svard <address@hidden>
Signed-off-by: Aidan Shribman <address@hidden>
---

 hmp-commands.hx |   33 +++++++++++++++++++++++++--------
 migration.c     |   23 ++++++++++++++++++++++-
 migration.h     |    2 ++
 qmp-commands.hx |   41 ++++++++++++++++++++++++++++++++++-------
 savevm.c        |   11 +++++++++++
 sysemu.h        |    1 +
 6 files changed, 95 insertions(+), 16 deletions(-)

diff --git a/hmp-commands.hx b/hmp-commands.hx
index e5585ba..215ea41 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -717,24 +717,28 @@ ETEXI
 
     {
         .name       = "migrate",
-        .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
-        .params     = "[-d] [-b] [-i] uri",
-        .help       = "migrate to URI (using -d to not wait for completion)"
-                     "\n\t\t\t -b for migration without shared storage with"
-                     " full copy of disk\n\t\t\t -i for migration without "
-                     "shared storage with incremental copy of disk "
-                     "(base image shared between src and destination)",
+        .args_type  = "detach:-d,blk:-b,inc:-i,warmup:-w,uri:s",
+        .params     = "[-d] [-b] [-i] [-w] uri",
+        .help       = "migrate to URI"
+                      "\n\t -d to not wait for completion"
+                      "\n\t -b for migration without shared storage with"
+                      " full copy of disk"
+                      "\n\t -i for migration without"
+                      " shared storage with incremental copy of disk"
+                      " (base image shared between source and destination)"
+                      "\n\t -w to enter warmup phase",
         .user_print = monitor_user_noop,       
        .mhandler.cmd_new = do_migrate,
     },
 
 
 STEXI
address@hidden migrate [-d] [-b] [-i] @var{uri}
address@hidden migrate [-d] [-b] [-i] [-w] @var{uri}
 @findex migrate
 Migrate to @var{uri} (using -d to not wait for completion).
        -b for migration with full copy of disk
        -i for migration with incremental copy of disk (base image is shared)
+    -w to enter warmup phase
 ETEXI
 
     {
@@ -753,6 +757,19 @@ Cancel the current VM migration.
 ETEXI
 
     {
+        .name       = "migrate_end",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Complete warmup and move to full live migration",
+        .mhandler.cmd = do_migrate_end,
+    },
+
+STEXI
address@hidden migrate_end
+Complete warmup and move to full live migration.
+ETEXI
+
+    {
         .name       = "migrate_set_speed",
         .args_type  = "value:o",
         .params     = "value",
diff --git a/migration.c b/migration.c
index 9ee8b17..c178a77 100644
--- a/migration.c
+++ b/migration.c
@@ -31,6 +31,8 @@
     do { } while (0)
 #endif
 
+static int is_migrate_warmup = 0;
+
 /* Migration speed throttling */
 static uint32_t max_throttle = (32 << 20);
 
@@ -81,6 +83,11 @@ int do_migrate(Monitor *mon, const QDict *qdict, QObject 
**ret_data)
     int blk = qdict_get_try_bool(qdict, "blk", 0);
     int inc = qdict_get_try_bool(qdict, "inc", 0);
     const char *uri = qdict_get_str(qdict, "uri");
+    is_migrate_warmup = qdict_get_try_bool(qdict, "warmup", 0);
+
+    if (is_migrate_warmup) {
+        detach = 1; /* as we need migrate_end to complete */
+    }
 
     if (current_migration &&
         current_migration->get_status(current_migration) == MIG_STATE_ACTIVE) {
@@ -361,7 +368,9 @@ void migrate_fd_put_ready(void *opaque)
     }
 
     DPRINTF("iterate\n");
-    if (qemu_savevm_state_iterate(s->mon, s->file) == 1) {
+    if (is_migrate_warmup) {
+            qemu_savevm_state_warmup(s->mon, s->file);
+    } else if (qemu_savevm_state_iterate(s->mon, s->file) == 1) {
         int state;
         int old_vm_running = vm_running;
 
@@ -448,3 +457,15 @@ int migrate_fd_close(void *opaque)
     qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
     return s->close(s);
 }
+
+void do_migrate_end(Monitor *mon, const QDict *qdict)
+{
+    if (!vm_running) {
+        return;
+    }
+    if (!is_migrate_warmup) {
+        return;
+    }
+    is_migrate_warmup = 0;
+}
+
diff --git a/migration.h b/migration.h
index d13ed4f..6a96b29 100644
--- a/migration.h
+++ b/migration.h
@@ -134,4 +134,6 @@ static inline FdMigrationState 
*migrate_to_fms(MigrationState *mig_state)
     return container_of(mig_state, FdMigrationState, mig_state);
 }
 
+void do_migrate_end(Monitor *mon, const QDict *qdict);
+
 #endif
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 793cf1c..1f85c09 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -431,13 +431,16 @@ EQMP
 
     {
         .name       = "migrate",
-        .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
-        .params     = "[-d] [-b] [-i] uri",
-        .help       = "migrate to URI (using -d to not wait for completion)"
-                     "\n\t\t\t -b for migration without shared storage with"
-                     " full copy of disk\n\t\t\t -i for migration without "
-                     "shared storage with incremental copy of disk "
-                     "(base image shared between src and destination)",
+        .args_type  = "detach:-d,blk:-b,inc:-i,xbrle:-x,warmup:-w,uri:s",
+        .params     = "[-d] [-b] [-i] [-w] uri",
+        .help       = "migrate to URI"
+                      "\n\t -d to not wait for completion"
+                      "\n\t -b for migration without shared storage with"
+                      " full copy of disk"
+                      "\n\t -i for migration without"
+                      " shared storage with incremental copy of disk"
+                      " (base image shared between source and destination)"
+                      "\n\t -w to enter warmup phase",
         .user_print = monitor_user_noop,       
        .mhandler.cmd_new = do_migrate,
     },
@@ -453,6 +456,7 @@ Arguments:
 - "blk": block migration, full disk copy (json-bool, optional)
 - "inc": incremental disk copy (json-bool, optional)
 - "uri": Destination URI (json-string)
+- "warmup": to enter warmup phase
 
 Example:
 
@@ -494,6 +498,29 @@ Example:
 EQMP
 
     {
+        .name       = "migrate_end",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Complete warmup and move to full live migration",
+        .mhandler.cmd = do_migrate_end,
+    },
+
+SQMP
+migrate_end
+-----------
+
+End the current migration warmup.
+
+Arguments: None.
+
+Example:
+
+-> { "execute": "migrate_end" }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "migrate_set_speed",
         .args_type  = "value:f",
         .params     = "value",
diff --git a/savevm.c b/savevm.c
index 4e49765..425c7b7 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1471,6 +1471,17 @@ int qemu_savevm_state_begin(Monitor *mon, QEMUFile *f, 
int blk_enable,
     return 0;
 }
 
+int qemu_savevm_state_warmup(Monitor *mon, QEMUFile *f)
+{
+    int ret;
+
+    if ((ret = qemu_savevm_state_iterate(mon, f)) < 0) {
+        return ret;
+    }
+
+    return 0;
+}
+
 int qemu_savevm_state_iterate(Monitor *mon, QEMUFile *f)
 {
     SaveStateEntry *se;
diff --git a/sysemu.h b/sysemu.h
index b81a70e..74e8a48 100644
--- a/sysemu.h
+++ b/sysemu.h
@@ -76,6 +76,7 @@ void main_loop_wait(int nonblocking);
 int qemu_savevm_state_begin(Monitor *mon, QEMUFile *f, int blk_enable,
                             int shared);
 int qemu_savevm_state_iterate(Monitor *mon, QEMUFile *f);
+int qemu_savevm_state_warmup(Monitor *mon, QEMUFile *f);
 int qemu_savevm_state_complete(Monitor *mon, QEMUFile *f);
 void qemu_savevm_state_cancel(Monitor *mon, QEMUFile *f);
 int qemu_loadvm_state(QEMUFile *f);
---