[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [Qemu-devel] [RFC/COLO: 1/3] COLO: Hybrid mode
|
From: |
zhanghailiang |
|
Subject: |
Re: [Qemu-devel] [RFC/COLO: 1/3] COLO: Hybrid mode |
|
Date: |
Wed, 5 Aug 2015 11:23:32 +0800 |
|
User-agent: |
Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Thunderbird/38.1.0 |
Seems pretty good overall~
For the part of migration parameters command, we have discussed before and
Markus promised to reconstruct this part in qemu 2.5 cycle. But for now,
it is OK.
Cc: Markus Armbruster <address@hidden>
On 2015/8/5 3:26, Dr. David Alan Gilbert (git) wrote:
From: "Dr. David Alan Gilbert" <address@hidden>
Automatically switch into a passive checkpoint mode when checkpoints are
repeatedly short. This saves CPU time on the SVM (since it's not running)
and the network traffic and PVM CPU time for the comparison processing.
Signed-off-by: Dr. David Alan Gilbert <address@hidden>
---
hmp.c | 26 ++++++++++
migration/colo.c | 136 +++++++++++++++++++++++++++++++++++++++++++-------
migration/migration.c | 65 +++++++++++++++++++++++-
qapi-schema.json | 22 ++++++--
qmp-commands.hx | 9 ++++
trace-events | 8 +++
6 files changed, 244 insertions(+), 22 deletions(-)
diff --git a/hmp.c b/hmp.c
index f34e2c2..8828756 100644
--- a/hmp.c
+++ b/hmp.c
@@ -289,6 +289,16 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict
*qdict)
monitor_printf(mon, " %s: %" PRId64,
MigrationParameter_lookup[MIGRATION_PARAMETER_DECOMPRESS_THREADS],
params->decompress_threads);
+ monitor_printf(mon, " %s: %" PRId64,
+ MigrationParameter_lookup[MIGRATION_PARAMETER_COLO_PASSIVE_COUNT],
+ params->colo_passive_count);
+ monitor_printf(mon, " %s: %" PRId64,
+ MigrationParameter_lookup[MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT],
+ params->colo_passive_limit);
+ monitor_printf(mon, " %s: %" PRId64,
+ MigrationParameter_lookup[MIGRATION_PARAMETER_COLO_PASSIVE_TIME],
+ params->colo_passive_time);
+
monitor_printf(mon, "\n");
}
@@ -1238,6 +1248,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict
*qdict)
bool has_compress_level = false;
bool has_compress_threads = false;
bool has_decompress_threads = false;
+ bool has_colo_passive_count = false;
+ bool has_colo_passive_limit = false;
+ bool has_colo_passive_time = false;
+
int i;
for (i = 0; i < MIGRATION_PARAMETER_MAX; i++) {
@@ -1252,10 +1266,22 @@ void hmp_migrate_set_parameter(Monitor *mon, const
QDict *qdict)
case MIGRATION_PARAMETER_DECOMPRESS_THREADS:
has_decompress_threads = true;
break;
+ case MIGRATION_PARAMETER_COLO_PASSIVE_COUNT:
+ has_colo_passive_count = true;
+ break;
+ case MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT:
+ has_colo_passive_limit = true;
+ break;
+ case MIGRATION_PARAMETER_COLO_PASSIVE_TIME:
+ has_colo_passive_time = true;
+ break;
}
qmp_migrate_set_parameters(has_compress_level, value,
has_compress_threads, value,
has_decompress_threads, value,
+ has_colo_passive_count, value,
+ has_colo_passive_limit, value,
+ has_colo_passive_time, value,
&err);
break;
}
diff --git a/migration/colo.c b/migration/colo.c
index d8ec283..37f63f2 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -21,6 +21,7 @@
#include "net/colo-nic.h"
#include "qmp-commands.h"
#include "block/block_int.h"
+#include "trace.h"
/*
* We should not do checkpoint one after another without any time interval,
@@ -66,6 +67,7 @@ typedef enum COLOCommand {
* go forward a lot when this side just receives the sync-point.
*/
COLO_CHECKPOINT_NEW,
+ COLO_CHECKPOINT_NEW_PASSIVE, /* Simple checkpoint mode, SVM doesn't run */
COLO_CHECKPOINT_SUSPENDED,
COLO_CHECKPOINT_SEND,
COLO_CHECKPOINT_RECEIVED,
@@ -294,7 +296,8 @@ static int colo_ctl_get(QEMUFile *f, uint64_t require)
return ret;
}
-static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
+static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control,
+ bool passive)
{
int colo_shutdown, ret;
size_t size;
@@ -302,7 +305,8 @@ static int colo_do_checkpoint_transaction(MigrationState
*s, QEMUFile *control)
int64_t start_time, end_time, down_time;
Error *local_err = NULL;
- ret = colo_ctl_put(s->file, COLO_CHECKPOINT_NEW);
+ ret = colo_ctl_put(s->file, passive?COLO_CHECKPOINT_NEW_PASSIVE:
^ Space
+ COLO_CHECKPOINT_NEW);
if (ret < 0) {
goto out;
}
@@ -438,6 +442,71 @@ out:
return ret;
}
+/*
+ * Counter that is reset to 'n' when we enter passive mode and
+ * is decremented once per checkpoint; when it hits zero we flip
+ * back to COLO mode.
+ */
+static unsigned int passive_count;
+
+/*
+ * Weighted average of checkpoint lengths, used to decide on mode.
+ */
+static double colo_checkpoint_time_mean;
+/* Count of checkpoints since we reset colo_checkpoint_time_mean */
+static uint64_t colo_checkpoint_time_count;
+
+/* Decides whether the checkpoint that's about to start should be
+ * a COLO type (with the secondary running and packet comparison) or
+ * a 'passive' type (with the secondary idle and running for fixed time)
+ *
+ * Returns:
+ * True: 'passive' type checkpoint
+ */
+static bool checkpoint_choice(MigrationState *s)
Confused name, maybe 'checkpoint_to_passive_mode()' is better ~
+{
+ trace_checkpoint_choice(passive_count,
+ colo_checkpoint_time_count,
+ colo_checkpoint_time_mean);
+ if (passive_count) {
+ /*
+ * The last checkpoint was passive; we stay in passive
+ * mode for a number of checkpoints before trying colo
+ * again.
+ */
+ passive_count--;
+ if (passive_count) {
+ /* Stay passive */
+ return true;
+ } else {
+ /* Transition back to COLO */
+ trace_checkpoint_choice_to_colo();
+ colo_checkpoint_time_mean = 0.0;
+ colo_checkpoint_time_count = 0;
+ return false;
+ }
+ } else {
+ /* The last checkpoint was COLO */
+ /* Could make that tunable, I'm not particularly worried about
+ * load behaviour for this, startup etc is probably more interesting.
+ */
+ if (colo_checkpoint_time_count < 5) {
+ /* Not done enough COLO cycles to evaluate times yet */
+ return false;
+ }
+ if (colo_checkpoint_time_mean <
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT]) {
+ trace_checkpoint_choice_to_passive(colo_checkpoint_time_mean);
+ /* We've had a few short checkpoints, switch to passive */
+ passive_count = s->parameters[
+ MIGRATION_PARAMETER_COLO_PASSIVE_COUNT];
passive_count =
s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_COUNT];
+ return true;
+ }
+ /* Keep going in COLO mode */
+ return false;
+ }
+}
+
/* should be calculated by bandwidth and max downtime ? */
#define THRESHOLD_PENDING_SIZE (10 * 1024 * 1024UL)
@@ -470,7 +539,7 @@ static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
QEMUFile *colo_control = NULL;
- int64_t current_time, checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
+ int64_t current_time = 0, checkpoint_time =
qemu_clock_get_ms(QEMU_CLOCK_HOST);
int i, ret;
Error *local_err = NULL;
@@ -518,8 +587,12 @@ static void *colo_thread(void *opaque)
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "run");
+ passive_count = 0;
+ colo_checkpoint_time_mean = 0.0;
+ colo_checkpoint_time_count = 0;
while (s->state == MIGRATION_STATUS_COLO) {
int proxy_checkpoint_req;
+ unsigned int checkpoint_limit;
if (failover_request_is_active()) {
error_report("failover request");
@@ -546,13 +619,17 @@ static void *colo_thread(void *opaque)
goto do_checkpoint;
}
+ checkpoint_limit = passive_count ?
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_TIME] :
+ colo_checkpoint_period;
+
/*
* No proxy checkpoint is request, wait for 100ms or
* transfer some dirty ram page,
* and then check if we need checkpoint again.
*/
current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
- if (current_time - checkpoint_time < colo_checkpoint_period) {
+ if (current_time - checkpoint_time < checkpoint_limit) {
if (colo_need_live_migrate_ram(s)) {
ret = colo_ctl_put(s->file, COLO_RAM_LIVE_MIGRATE);
if (ret < 0) {
@@ -572,8 +649,16 @@ static void *colo_thread(void *opaque)
}
do_checkpoint:
+ /* Update a weighted mean of checkpoint lengths, weighted
+ * so that an occasional short checkpoint doesn't cause a switch
+ * to passive.
+ */
+ colo_checkpoint_time_mean = colo_checkpoint_time_mean * 0.7 +
+ 0.3 * (current_time - checkpoint_time);
Why the weight value are '0.7' and '0.3'?
Are they based on some tests?
+ colo_checkpoint_time_count++;
/* start a colo checkpoint */
- if (colo_do_checkpoint_transaction(s, colo_control)) {
+ if (colo_do_checkpoint_transaction(s, colo_control,
+ checkpoint_choice(s))) {
goto out;
}
checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
@@ -638,7 +723,10 @@ void colo_init_checkpointer(MigrationState *s)
/*
* return:
- * 0: start a checkpoint
+ * COLO_CHECKPOINT_NEW: Primary requests a COLO checkpoint cycle
+ * COLO_CHECKPOINT_NEW_PASSIVE: Primary requests a basic checkpoint
+ * cycle
+ *
* -1: some error happened, exit colo restore
*/
static int colo_wait_handle_cmd(QEMUFile *f, int *checkpoint_request)
@@ -654,8 +742,9 @@ static int colo_wait_handle_cmd(QEMUFile *f, int
*checkpoint_request)
switch (cmd) {
case COLO_CHECKPOINT_NEW:
+ case COLO_CHECKPOINT_NEW_PASSIVE:
*checkpoint_request = 1;
- return 0;
+ return cmd;
case COLO_GUEST_SHUTDOWN:
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
@@ -695,6 +784,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
int fd = qemu_get_fd(f);
QEMUFile *ctl = NULL, *fb = NULL;
uint64_t total_size;
+ bool last_was_passive = false;
int i, ret;
Error *local_err = NULL;
@@ -750,9 +840,9 @@ void *colo_process_incoming_checkpoints(void *opaque)
while (mis->state == MIGRATION_STATUS_COLO) {
int request = 0;
- int ret = colo_wait_handle_cmd(f, &request);
+ int mode = colo_wait_handle_cmd(f, &request);
- if (ret < 0) {
+ if (mode < 0) {
break;
} else {
if (!request) {
@@ -765,11 +855,13 @@ void *colo_process_incoming_checkpoints(void *opaque)
goto out;
}
- /* suspend guest */
- qemu_mutex_lock_iothread();
- vm_stop_force_state(RUN_STATE_COLO);
- qemu_mutex_unlock_iothread();
- trace_colo_vm_state_change("run", "stop");
+ if (!last_was_passive) {
+ /* suspend guest */
+ qemu_mutex_lock_iothread();
+ vm_stop_force_state(RUN_STATE_COLO);
+ qemu_mutex_unlock_iothread();
+ trace_colo_vm_state_change("run", "stop");
+ }
ret = colo_ctl_put(ctl, COLO_CHECKPOINT_SUSPENDED);
if (ret < 0) {
@@ -848,10 +940,18 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
/* resume guest */
- qemu_mutex_lock_iothread();
- vm_start();
- qemu_mutex_unlock_iothread();
- trace_colo_vm_state_change("stop", "start");
+ if (mode == COLO_CHECKPOINT_NEW_PASSIVE) {
+ last_was_passive = true;
+ trace_colo_process_incoming_checkpoints_passive();
+ } else {
+ qemu_mutex_lock_iothread();
+ vm_start();
+ qemu_mutex_unlock_iothread();
+ last_was_passive = false;
+ trace_colo_vm_state_change("stop", "start");
+ trace_colo_process_incoming_checkpoints_active();
+ }
+
qemu_fclose(fb);
fb = NULL;
diff --git a/migration/migration.c b/migration/migration.c
index f7f0884..f84c676 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -49,6 +49,14 @@
/* Migration XBZRLE default cache size */
#define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
+/* COLO: The number of passive checkpoints to try before switching back
+ * to COLO */
+#define DEFAULT_MIGRATE_COLO_PASSIVE_COUNT 100
+/* COLO Checkpoint time (ms) below which we switch into passive mode */
+#define DEFAULT_MIGRATE_COLO_PASSIVE_LIMIT 400
+/* COLO passive mode checkpoint time (ms) */
+#define DEFAULT_MIGRATE_COLO_PASSIVE_TIME 250
+
static NotifierList migration_state_notifiers =
NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
@@ -72,6 +80,12 @@ MigrationState *migrate_get_current(void)
DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT,
.parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT,
+ .parameters[MIGRATION_PARAMETER_COLO_PASSIVE_COUNT] =
+ DEFAULT_MIGRATE_COLO_PASSIVE_COUNT,
+ .parameters[MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT] =
+ DEFAULT_MIGRATE_COLO_PASSIVE_LIMIT,
+ .parameters[MIGRATION_PARAMETER_COLO_PASSIVE_TIME] =
+ DEFAULT_MIGRATE_COLO_PASSIVE_TIME
.checkpoint_state.max_downtime = 0,
.checkpoint_state.min_downtime = INT64_MAX
};
@@ -389,6 +403,9 @@ MigrationParameters *qmp_query_migrate_parameters(Error
**errp)
s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS];
params->decompress_threads =
s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS];
+ params->colo_passive_count =
s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_COUNT];
+ params->colo_passive_limit =
s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT];
+ params->colo_passive_time =
s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_TIME];
return params;
}
@@ -539,7 +556,14 @@ void qmp_migrate_set_parameters(bool has_compress_level,
bool has_compress_threads,
int64_t compress_threads,
bool has_decompress_threads,
- int64_t decompress_threads, Error **errp)
+ int64_t decompress_threads,
+ bool has_colo_passive_count,
+ int64_t colo_passive_count,
+ bool has_colo_passive_limit,
+ int64_t colo_passive_limit,
+ bool has_colo_passive_time,
+ int64_t colo_passive_time,
+ Error **errp)
{
MigrationState *s = migrate_get_current();
@@ -562,6 +586,21 @@ void qmp_migrate_set_parameters(bool has_compress_level,
"is invalid, it should be in the range of 1 to 255");
return;
}
+ if (has_colo_passive_count && (colo_passive_count < 0)) {
+ error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+ "colo_passive_count",
+ "is invalid, it must be positive");
+ }
+ if (has_colo_passive_limit && (colo_passive_limit < 0)) {
+ error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+ "colo_passive_limit",
+ "is invalid, it must be positive");
+ }
+ if (has_colo_passive_time && (colo_passive_time < 0)) {
+ error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+ "colo_passive_time",
+ "is invalid, it must be positive");
+ }
if (has_compress_level) {
s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level;
@@ -573,6 +612,18 @@ void qmp_migrate_set_parameters(bool has_compress_level,
s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
decompress_threads;
}
+ if (has_colo_passive_count) {
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_COUNT] =
+ colo_passive_count;
+ }
+ if (has_colo_passive_limit) {
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT] =
+ colo_passive_limit;
+ }
+ if (has_colo_passive_time) {
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_TIME] =
+ colo_passive_time;
+ }
}
/* shared migration helpers */
@@ -689,6 +740,13 @@ static MigrationState *migrate_init(const MigrationParams
*params)
s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS];
int decompress_thread_count =
s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS];
+ int colo_passive_count = s->parameters[
+ MIGRATION_PARAMETER_COLO_PASSIVE_COUNT];
+ int colo_passive_limit = s->parameters[
+ MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT];
+ int colo_passive_time = s->parameters[
+ MIGRATION_PARAMETER_COLO_PASSIVE_TIME];
+
memcpy(enabled_capabilities, s->enabled_capabilities,
sizeof(enabled_capabilities));
@@ -704,6 +762,11 @@ static MigrationState *migrate_init(const MigrationParams
*params)
compress_thread_count;
s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
decompress_thread_count;
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_COUNT] = colo_passive_count;
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_LIMIT] =
+ colo_passive_limit;
+ s->parameters[MIGRATION_PARAMETER_COLO_PASSIVE_TIME] = colo_passive_time;
+
s->bandwidth_limit = bandwidth_limit;
migrate_set_state(&s->state, MIGRATION_STATUS_NONE,
MIGRATION_STATUS_SETUP);
diff --git a/qapi-schema.json b/qapi-schema.json
index 9f094d2..9d2b6d4 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -630,10 +630,15 @@
# compression, so set the decompress-threads to the number about 1/4
# of compress-threads is adequate.
#
+# @colo-passive-count: Time (in ms) for a COLO passive mode checkpoint
+# @colo-passive-limit: Time (in ms) below which we switch into passive mode
+# @colo-passive-time: Time (in ms) for a COLO passive mode checkpoint
+#
# Since: 2.4
##
{ 'enum': 'MigrationParameter',
- 'data': ['compress-level', 'compress-threads', 'decompress-threads'] }
+ 'data': ['compress-level', 'compress-threads', 'decompress-threads',
+ 'colo-passive-count', 'colo-passive-limit', 'colo-passive-time'] }
#
# @migrate-set-parameters
@@ -646,12 +651,19 @@
#
# @decompress-threads: decompression thread count
#
+# @colo-passive-count: Time (in ms) for a COLO passive mode checkpoint
+# @colo-passive-limit: Time (in ms) below which we switch into passive mode
+# @colo-passive-time: Time (in ms) for a COLO passive mode checkpoint
+#
# Since: 2.4
##
{ 'command': 'migrate-set-parameters',
'data': { '*compress-level': 'int',
'*compress-threads': 'int',
- '*decompress-threads': 'int'} }
+ '*decompress-threads': 'int',
+ '*colo-passive-count': 'int',
+ '*colo-passive-limit': 'int',
+ '*colo-passive-time': 'int' } }
#
# @MigrationParameters
@@ -667,7 +679,11 @@
{ 'struct': 'MigrationParameters',
'data': { 'compress-level': 'int',
'compress-threads': 'int',
- 'decompress-threads': 'int'} }
+ 'decompress-threads': 'int',
+ 'colo-passive-count': 'int',
+ 'colo-passive-limit': 'int',
+ 'colo-passive-time': 'int' } }
+
##
# @query-migrate-parameters
#
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 4fd01a7..a809710 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3501,6 +3501,10 @@ Set migration parameters
- "compress-level": set compression level during migration (json-int)
- "compress-threads": set compression thread count for migration (json-int)
- "decompress-threads": set decompression thread count for migration
(json-int)
+- "colo-passive-count": In COLO, the number of passive checkpoint cycles to
+ try before reverting to COLO mode
+- "colo-passive-limit": Time (in ms) below which we switch into passive mode
+- "colo-passive-time": Time (in ms) for a COLO passive mode checkpoint
Arguments:
@@ -3527,6 +3531,11 @@ Query current migration parameters
- "compress-level" : compression level value (json-int)
- "compress-threads" : compression thread count value (json-int)
- "decompress-threads" : decompression thread count value (json-int)
+ - "colo-passive-count": In COLO, the number of passive checkpoints
+ cycles to try before reverting to COLO mode
+ - "colo-passive-limit": Time (in ms) below which we switch into
+ passive mode
+ - "colo-passive-time": Time (in ms) for a COLO passive mode checkpoint
Arguments:
diff --git a/trace-events b/trace-events
index 03cd035..4d74166 100644
--- a/trace-events
+++ b/trace-events
@@ -1408,6 +1408,14 @@ migrate_state_too_big(void) ""
migrate_global_state_post_load(const char *state) "loaded state: %s"
migrate_global_state_pre_save(const char *state) "saved state: %s"
+# migration/colo.c
+checkpoint_choice_to_colo(void) ""
+checkpoint_choice_to_passive(double check_time) "weighted checkpoint time=%f"
+colo_process_incoming_checkpoints_new(int mode, int last_was_passive) "mode=%d
last_was_passive=%d"
+colo_process_incoming_checkpoints_passive(void) ""
+colo_process_incoming_checkpoints_active(void) ""
+checkpoint_choice(uint64_t passive_count, uint64_t colo_checkpoint_time_count, double
colo_checkpoint_time_mean) "%" PRIu64 "/%" PRIu64 " %f"
+
# migration/rdma.c
qemu_rdma_accept_incoming_migration(void) ""
qemu_rdma_accept_incoming_migration_accepted(void) ""