[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Qemu-devel] [PATCH COLO-Frame v9 12/32] COLO: Save PVM state to seconda
From: |
zhanghailiang |
Subject: |
[Qemu-devel] [PATCH COLO-Frame v9 12/32] COLO: Save PVM state to secondary side when do checkpoint |
Date: |
Wed, 2 Sep 2015 16:22:59 +0800 |
The main process of checkpoint is to synchronize SVM with PVM.
VM's state includes ram and device state. So we will migrate PVM's
state to SVM when do checkpoint, just like migration does.
We will cache PVM's state in slave, we use QEMUSizedBuffer
to store the data, we need to know the size of VM state, so in master,
we use qsb to store VM state temporarily, get the data size by call
qsb_get_length()
and then migrate the data to the qsb in the secondary side.
Signed-off-by: zhanghailiang <address@hidden>
Signed-off-by: Yang Hongyang <address@hidden>
Signed-off-by: Gonglei <address@hidden>
Signed-off-by: Li Zhijian <address@hidden>
---
migration/colo.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++-----
migration/ram.c | 46 +++++++++++++++++++++++++++--------
migration/savevm.c | 2 +-
3 files changed, 101 insertions(+), 17 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index 526ee85..f107032 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -18,6 +18,8 @@
#include "qemu/sockets.h"
static QEMUBH *colo_bh;
+/* colo buffer */
+#define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
bool colo_supported(void)
{
@@ -96,9 +98,12 @@ static int colo_ctl_get(QEMUFile *f, uint32_t require)
return value;
}
-static int colo_do_checkpoint_transaction(MigrationState *s)
+static int colo_do_checkpoint_transaction(MigrationState *s,
+ QEMUSizedBuffer *buffer)
{
int ret;
+ size_t size;
+ QEMUFile *trans = NULL;
ret = colo_ctl_put(s->to_dst_file, COLO_CMD_CHECKPOINT_REQUEST, 0);
if (ret < 0) {
@@ -109,15 +114,48 @@ static int colo_do_checkpoint_transaction(MigrationState
*s)
if (ret < 0) {
goto out;
}
+ /* Reset colo buffer and open it for write */
+ qsb_set_length(buffer, 0);
+ trans = qemu_bufopen("w", buffer);
+ if (!trans) {
+ error_report("Open colo buffer for write failed");
+ goto out;
+ }
+
+ /* suspend and save vm state to colo buffer */
+ qemu_mutex_lock_iothread();
+ vm_stop_force_state(RUN_STATE_COLO);
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("run", "stop");
+
+ /* Disable block migration */
+ s->params.blk = 0;
+ s->params.shared = 0;
+ qemu_savevm_state_header(trans);
+ qemu_savevm_state_begin(trans, &s->params);
+ qemu_mutex_lock_iothread();
+ qemu_savevm_state_complete(trans);
+ qemu_mutex_unlock_iothread();
- /* TODO: suspend and save vm state to colo buffer */
+ qemu_fflush(trans);
ret = colo_ctl_put(s->to_dst_file, COLO_CMD_VMSTATE_SEND, 0);
if (ret < 0) {
goto out;
}
+ /* we send the total size of the vmstate first */
+ size = qsb_get_length(buffer);
+ ret = colo_ctl_put(s->to_dst_file, COLO_CMD_VMSTATE_SIZE, size);
+ if (ret < 0) {
+ goto out;
+ }
- /* TODO: send vmstate to Secondary */
+ qsb_put_buffer(s->to_dst_file, buffer, size);
+ qemu_fflush(s->to_dst_file);
+ ret = qemu_file_get_error(s->to_dst_file);
+ if (ret < 0) {
+ goto out;
+ }
ret = colo_ctl_get(s->from_dst_file, COLO_CMD_VMSTATE_RECEIVED);
if (ret < 0) {
@@ -129,15 +167,25 @@ static int colo_do_checkpoint_transaction(MigrationState
*s)
goto out;
}
- /* TODO: resume Primary */
+ ret = 0;
+ /* resume master */
+ qemu_mutex_lock_iothread();
+ vm_start();
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("stop", "run");
out:
+ if (trans) {
+ qemu_fclose(trans);
+ }
+
return ret;
}
static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
+ QEMUSizedBuffer *buffer = NULL;
int fd, ret = 0;
/* Dup the fd of to_dst_file */
@@ -162,6 +210,13 @@ static void *colo_thread(void *opaque)
goto out;
}
+ buffer = qsb_create(NULL, COLO_BUFFER_BASE_SIZE);
+ if (buffer == NULL) {
+ ret = -ENOMEM;
+ error_report("Failed to allocate buffer!");
+ goto out;
+ }
+
qemu_mutex_lock_iothread();
vm_start();
qemu_mutex_unlock_iothread();
@@ -169,7 +224,7 @@ static void *colo_thread(void *opaque)
while (s->state == MIGRATION_STATUS_COLO) {
/* start a colo checkpoint */
- ret = colo_do_checkpoint_transaction(s);
+ ret = colo_do_checkpoint_transaction(s, buffer);
if (ret < 0) {
goto out;
}
@@ -182,7 +237,10 @@ out:
migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
MIGRATION_STATUS_COMPLETED);
- if (s->from_dst_file) {
+ qsb_free(buffer);
+ buffer = NULL;
+
+ if (s->from_dst_file) {
qemu_fclose(s->from_dst_file);
}
diff --git a/migration/ram.c b/migration/ram.c
index 7f007e6..68980be 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -38,6 +38,7 @@
#include "trace.h"
#include "exec/ram_addr.h"
#include "qemu/rcu_queue.h"
+#include "migration/colo.h"
#ifdef DEBUG_MIGRATION_RAM
#define DPRINTF(fmt, ...) \
@@ -1090,15 +1091,8 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t
new)
}
}
-/* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
- * long-running RCU critical section. When rcu-reclaims in the code
- * start to become numerous it will be necessary to reduce the
- * granularity of these critical sections.
- */
-
-static int ram_save_setup(QEMUFile *f, void *opaque)
+static int ram_save_init_globals(void)
{
- RAMBlock *block;
int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
mig_throttle_on = false;
@@ -1158,6 +1152,31 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
migration_bitmap_sync();
qemu_mutex_unlock_ramlist();
qemu_mutex_unlock_iothread();
+ rcu_read_unlock();
+
+ return 0;
+}
+
+/* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
+ * long-running RCU critical section. When rcu-reclaims in the code
+ * start to become numerous it will be necessary to reduce the
+ * granularity of these critical sections.
+ */
+
+static int ram_save_setup(QEMUFile *f, void *opaque)
+{
+ RAMBlock *block;
+
+ /*
+ * migration has already setup the bitmap, reuse it.
+ */
+ if (!migration_in_colo_state()) {
+ if (ram_save_init_globals() < 0) {
+ return -1;
+ }
+ }
+
+ rcu_read_lock();
qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
@@ -1257,7 +1276,8 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
while (true) {
int pages;
- pages = ram_find_and_save_block(f, true, &bytes_transferred);
+ pages = ram_find_and_save_block(f, !migration_in_colo_state(),
+ &bytes_transferred);
/* no more blocks to sent */
if (pages == 0) {
break;
@@ -1268,8 +1288,14 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
ram_control_after_iterate(f, RAM_CONTROL_FINISH);
rcu_read_unlock();
+ /*
+ * Since we need to reuse dirty bitmap in colo,
+ * don't cleanup the bitmap.
+ */
+ if (!migrate_enable_colo() || migration_has_failed(migrate_get_current()))
{
+ migration_end();
+ }
- migration_end();
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
return 0;
diff --git a/migration/savevm.c b/migration/savevm.c
index 6071215..c1b40ce 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -48,7 +48,7 @@
#include "qemu/iov.h"
#include "block/snapshot.h"
#include "block/qapi.h"
-
+#include "migration/colo.h"
#ifndef ETH_P_RARP
#define ETH_P_RARP 0x8035
--
1.8.3.1
- [Qemu-devel] [PATCH COLO-Frame v9 00/32] COarse-grain LOck-stepping(COLO) Virtual Machines for Non-stop Service (FT), zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 03/32] COLO: migrate colo related info to slave, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 01/32] configure: Add parameter for configure to enable/disable COLO support, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 05/32] migration: Integrate COLO checkpoint process into migration, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 04/32] migration: Add state records for migration incoming, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 09/32] COLO: Implement colo checkpoint protocol, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 11/32] QEMUSizedBuffer: Introduce two help functions for qsb, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 12/32] COLO: Save PVM state to secondary side when do checkpoint,
zhanghailiang <=
- [Qemu-devel] [PATCH COLO-Frame v9 08/32] COLO/migration: establish a new communication path from destination to source, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 02/32] migration: Introduce capability 'colo' to migration, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 06/32] migration: Integrate COLO checkpoint process into loadvm, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 16/32] COLO: synchronize PVM's state to SVM periodically, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 14/32] COLO: Load VMState into qsb before restore it, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 15/32] COLO: Flush PVM's cached RAM into SVM's memory, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 07/32] migration: Rename the'file' member of MigrationState and MigrationIncomingState, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 13/32] COLO: Load PVM's dirty pages into SVM's RAM cache temporarily, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 19/32] COLO: Implement failover work for Primary VM, zhanghailiang, 2015/09/02
- [Qemu-devel] [PATCH COLO-Frame v9 20/32] COLO: Implement failover work for Secondary VM, zhanghailiang, 2015/09/02