[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO p
From: |
zhanghailiang |
Subject: |
[Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO primary/secondary vm failover work |
Date: |
Thu, 9 Jul 2015 11:16:25 +0800 |
If there are some errors happen, we will give users(administrators) time to
get involved in failover verdict, which they can decide
which side should take over the work by using 'colo_lost_heartbeat' command.
Note: The default verdict is primary VM takes over work while secondary VM exit.
So if users choose secondary VM to take over work, please make sure that
Primary VM is dead, or there will be 'split-brain' problem.
Signed-off-by: zhanghailiang <address@hidden>
Signed-off-by: Li Zhijian <address@hidden>
Signed-off-by: Lai Jiangshan <address@hidden>
---
include/migration/colo.h | 4 ++
include/migration/failover.h | 1 +
migration/colo-failover.c | 15 ++++-
migration/colo.c | 149 +++++++++++++++++++++++++++++++++++++++++--
trace-events | 1 +
5 files changed, 165 insertions(+), 5 deletions(-)
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 815db2e..0364bab 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -43,4 +43,8 @@ int get_colo_mode(void);
int create_and_init_ram_cache(void);
void colo_flush_ram_cache(void);
void release_ram_cache(void);
+
+/* failover */
+void colo_do_failover(MigrationState *s);
+
#endif
diff --git a/include/migration/failover.h b/include/migration/failover.h
index 56d57d7..f85f97d 100644
--- a/include/migration/failover.h
+++ b/include/migration/failover.h
@@ -25,5 +25,6 @@ typedef enum COLOFailoverStatus {
int failover_set_state(int old_state, int new_state);
int failover_get_state(void);
void failover_request_active(Error **errp);
+bool failover_request_is_active(void);
#endif
diff --git a/migration/colo-failover.c b/migration/colo-failover.c
index 1bf22f5..7b0fa03 100644
--- a/migration/colo-failover.c
+++ b/migration/colo-failover.c
@@ -22,9 +22,17 @@ static COLOFailoverStatus failover_state;
static void colo_failover_bh(void *opaque)
{
+ int old_state;
+
qemu_bh_delete(failover_bh);
failover_bh = NULL;
- /*TODO: Do failover work */
+ old_state = failover_set_state(FAILOVER_STATUS_REQUEST,
+ FAILOVER_STATUS_HANDLING);
+ if (old_state != FAILOVER_STATUS_REQUEST) {
+ error_report(" Unkown error for failover, old_state=%d", old_state);
+ return;
+ }
+ colo_do_failover(NULL);
}
void failover_request_active(Error **errp)
@@ -54,6 +62,11 @@ int failover_get_state(void)
return atomic_read(&failover_state);
}
+bool failover_request_is_active(void)
+{
+ return ((failover_get_state() != FAILOVER_STATUS_NONE));
+}
+
void qmp_colo_lost_heartbeat(Error **errp)
{
if (get_colo_mode() == COLO_MODE_UNKNOWN) {
diff --git a/migration/colo.c b/migration/colo.c
index 6990c0c..243aa32 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -73,6 +73,83 @@ bool loadvm_in_colo_state(void)
return colo != NULL;
}
+static bool colo_runstate_is_stopped(void)
+{
+ return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
+}
+
+/*
+ * there are two way to entry this function
+ * 1. From colo checkpoint incoming thread, in this case
+ * we should protect it by iothread lock
+ * 2. From user command, because hmp/qmp command
+ * was happened in main loop, iothread lock will cause a
+ * dead lock.
+ */
+static void secondary_vm_do_failover(void)
+{
+ int old_state;
+
+ /* It means that VM exit from COLO state */
+ colo = NULL;
+
+ if (!autostart) {
+ error_report("\"-S\" qemu option will be ignored in secondary side");
+ /* recover runstate to normal migration finish state */
+ autostart = true;
+ }
+
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_COMPLETED);
+ if (old_state != FAILOVER_STATUS_HANDLING) {
+ error_report("Serious error while do failover for secondary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+ /* For Secondary VM, jump to incoming co */
+ if (migration_incoming_co) {
+ qemu_coroutine_enter(migration_incoming_co, NULL);
+ }
+}
+
+static void primary_vm_do_failover(void)
+{
+ MigrationState *s = migrate_get_current();
+ int old_state;
+
+ if (!colo_runstate_is_stopped()) {
+ vm_stop_force_state(RUN_STATE_COLO);
+ }
+
+ if (s->state != MIGRATION_STATUS_FAILED) {
+ migrate_set_state(s, MIGRATION_STATUS_COLO,
MIGRATION_STATUS_COMPLETED);
+ }
+
+ vm_start();
+
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_COMPLETED);
+ if (old_state != FAILOVER_STATUS_COMPLETED) {
+ error_report("Serious error while do failover for Primary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+}
+
+void colo_do_failover(MigrationState *s)
+{
+ /* Make sure vm stopped while failover */
+ if (!colo_runstate_is_stopped()) {
+ vm_stop_force_state(RUN_STATE_COLO);
+ }
+
+ if (get_colo_mode() == COLO_MODE_SECONDARY) {
+ secondary_vm_do_failover();
+ } else {
+ primary_vm_do_failover();
+ }
+}
+
/* colo checkpoint control helper */
static int colo_ctl_put(QEMUFile *f, uint64_t request)
{
@@ -144,11 +221,23 @@ static int colo_do_checkpoint_transaction(MigrationState
*s, QEMUFile *control)
goto out;
}
+ if (failover_request_is_active()) {
+ ret = -1;
+ goto out;
+ }
/* suspend and save vm state to colo buffer */
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("run", "stop");
+ /*
+ * failover request bh could be called after
+ * vm_stop_force_state so we check failover_request_is_active() again.
+ */
+ if (failover_request_is_active()) {
+ ret = -1;
+ goto out;
+ }
/* Disable block migration */
s->params.blk = 0;
@@ -209,7 +298,7 @@ static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
QEMUFile *colo_control = NULL;
- int ret;
+ int i, ret;
colo_control = qemu_fopen_socket(qemu_get_fd(s->file), "rb");
if (!colo_control) {
@@ -239,6 +328,11 @@ static void *colo_thread(void *opaque)
trace_colo_vm_state_change("stop", "run");
while (s->state == MIGRATION_STATUS_COLO) {
+ if (failover_request_is_active()) {
+ error_report("failover request");
+ goto out;
+ }
+
/* start a colo checkpoint */
if (colo_do_checkpoint_transaction(s, colo_control)) {
goto out;
@@ -246,7 +340,26 @@ static void *colo_thread(void *opaque)
}
out:
- migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ error_report("colo: some error happens in colo_thread");
+ /* Give users time (2s) to get involved in this verdict */
+ for (i = 0; i < 10; i++) {
+ if (failover_request_is_active()) {
+ error_report("Primary VM will take over work");
+ break;
+ }
+ usleep(200 * 1000);
+ }
+ qemu_mutex_lock_iothread();
+ if (!failover_request_is_active()) {
+ error_report("Primary VM will take over work in default");
+ failover_request_active(NULL);
+ }
+ qemu_mutex_unlock_iothread();
+
+ while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+ ;
+ }
+ failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
qsb_free(colo_buffer);
colo_buffer = NULL;
@@ -317,7 +430,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
QEMUFile *f = colo_in->file;
int fd = qemu_get_fd(f);
QEMUFile *ctl = NULL, *fb = NULL;
- int ret;
+ int i, ret;
uint64_t total_size;
colo = qemu_coroutine_self();
@@ -363,6 +476,11 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
}
+ if (failover_request_is_active()) {
+ error_report("failover request");
+ goto out;
+ }
+
/* suspend guest */
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
@@ -431,7 +549,30 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
out:
- colo = NULL;
+ error_report("Detect some error or get a failover request");
+ /* Give users time (2s) to get involved in this verdict */
+ for (i = 0; i < 10; i++) {
+ if (failover_request_is_active()) {
+ error_report("Secondary VM will take over work");
+ break;
+ }
+ usleep(200*1000);
+ }
+ /* check flag again*/
+ if (!failover_request_is_active()) {
+ /*
+ * We assume that Primary VM is still alive according to heartbeat,
+ * just kill Secondary VM
+ */
+ error_report("SVM is going to exit in default!");
+ exit(1);
+ } else {
+ /* if we went here, means Primary VM may dead, we are doing failover */
+ while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+ ;
+ }
+ failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
+ }
if (fb) {
qemu_fclose(fb);
diff --git a/trace-events b/trace-events
index 20551c8..66bcd08 100644
--- a/trace-events
+++ b/trace-events
@@ -1476,6 +1476,7 @@
rdma_start_outgoing_migration_after_rdma_source_init(void) ""
colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
colo_receive_message(const char *msg) "Receive '%s'"
failover_set_state(int new_state) "new state %d"
+colo_rcv_pkt(int result) "Result of net packets comparing is different: %d"
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
--
1.7.12.4
- [Qemu-devel] [PATCH COLO-Frame v7 31/34] COLO NIC: Implement NIC checkpoint and failover, (continued)
- [Qemu-devel] [PATCH COLO-Frame v7 31/34] COLO NIC: Implement NIC checkpoint and failover, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 33/34] COLO: Implement shutdown checkpoint, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 13/34] arch_init: Start to trace dirty pages of SVM, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 01/34] configure: Add parameter for configure to enable/disable COLO support, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 04/34] colo-comm/migration: skip colo info section for special cases, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 03/34] COLO: migrate colo related info to slave, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 19/34] COLO failover: Don't do failover during loading VM's state, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 23/34] COLO NIC: Implement colo nic device interface configure(), zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 18/34] qmp event: Add event notification for COLO error, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 20/34] COLO: Add new command parameter 'forward_nic' 'colo_script' for net, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 17/34] COLO failover: Implement COLO primary/secondary vm failover work,
zhanghailiang <=
- [Qemu-devel] [PATCH COLO-Frame v7 21/34] COLO NIC: Init/remove colo nic devices when add/cleanup tap devices, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 28/34] COLO: Do checkpoint according to the result of packets comparation, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 27/34] COLO: Handle nfnetlink message from proxy module, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 16/34] COLO failover: Introduce state to record failover process, zhanghailiang, 2015/07/08
- [Qemu-devel] [PATCH COLO-Frame v7 26/34] COLO NIC: Some init work related with proxy module, zhanghailiang, 2015/07/08