[Qemu-devel] [PATCH 2/6] bitmap dump code via QAPI framework

[Sanidhya Kashyap]

This patch introduces the mechanism of dumping the dirty bitmap either
in an ascii format or binary format. The implementation is almost
similar to the migration one. A separate thread is created for the
dumping process.

The bitmap is obtained with the help of ramlist blocks. I have used almost
similar functions that have been used in the migration mechanism (in
arch_init.c file). The mechanism to save the dirty bitmap is based on
RamBlock rather than MemoryRegion. After having a discussion with Juan, there
are still some issues with MemoryRegion like
* memory regions can overlap as well as
* no port of TCG to MemoryRegion.

Another point to be noted is that other instance of the bitmap dump process
will not execute if there is an already running process.

In this patch, I have introduced log-dirty-bitmap qmp interface to dump the
bitmap array.

The logging process has four variables:
1) filename - To dump the data in the file.
2) epochs - It is the number of times, the bitmaps will be logged and dumped
in the file. The default value is 3.
3) frequency - Time difference between each epochs. Currently, the default
value is 40ms.
4) readable - Its a boolean variable whose default value is false. That is
the data is save in human readable format. The first line is the total number
of pages followed by size of the bitmap and then the hex dump of the bitmap
array.

Signed-off-by: Sanidhya Kashyap <address@hidden>
---
 include/qapi/qmp/qerror.h |   3 +
 qapi-schema.json          |  10 ++
 qmp-commands.hx           |  34 +++++
 savevm.c                  | 318 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 365 insertions(+)

diff --git a/include/qapi/qmp/qerror.h b/include/qapi/qmp/qerror.h
index 902d1a7..8b5b7b3 100644
--- a/include/qapi/qmp/qerror.h
+++ b/include/qapi/qmp/qerror.h
@@ -118,6 +118,9 @@ void qerror_report_err(Error *err);
 #define QERR_MIGRATION_ACTIVE \
     ERROR_CLASS_GENERIC_ERROR, "There's a migration process in progress"
 
+#define QERR_LOG_DIRTY_BITMAP_ACTIVE \
+    ERROR_CLASS_GENERIC_ERROR, "Dirty bitmap dump already in progress"
+
 #define QERR_MISSING_PARAMETER \
     ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' is missing"
 
diff --git a/qapi-schema.json b/qapi-schema.json
index 36cb964..c1dba64 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -4700,3 +4700,13 @@
               'btn'     : 'InputBtnEvent',
               'rel'     : 'InputMoveEvent',
               'abs'     : 'InputMoveEvent' } }
+##
+# @log-dirty-bitmap
+#
+# provides the dirty bitmap for time t if specified.
+##
+{ 'command' : 'log-dirty-bitmap',
+  'data'    : { 'filename'      : 'str',
+                '*epochs'       : 'int',
+                '*frequency'    : 'int',
+                '*readable'     : 'bool' } }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index cae890e..33e7719 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -3569,3 +3569,37 @@ Example:
                    } } ] }
 
 EQMP
+
+    {
+        .name       = "log-dirty-bitmap",
+        .args_type  = "filename:s,epochs:i?,frequency:i?,readable:-r?",
+        .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap,
+    },
+
+SQMP
+log-dirty-bitmap
+--------------------
+
+start logging the memory of the VM for writable working set
+
+Arguments:
+
+- "filename": name of the file, in which the bitmap will be saved
+- "epochs": number of times, the memory will be logged
+- "frequency": time difference in milliseconds between each epoch
+- "readable": dumps the bitmap in hex format (non-binary)
+
+Examples:
+-> { "execute" : "log-dirty-bitmap",
+     "arguments" : {
+         "filename" : "/tmp/fileXXX",
+         "epochs" : 100,
+         "frequency" : 100 } }
+
+<- { "return": {} }
+
+Note: The epochs, frequency and readable are optional. epochs default
+value is 3 while that of frequency is 40 and readable defaults to false.
+
+EQMP
+
diff --git a/savevm.c b/savevm.c
index da8aa24..25d399a 100644
--- a/savevm.c
+++ b/savevm.c
@@ -41,6 +41,10 @@
 #include "qemu/iov.h"
 #include "block/snapshot.h"
 #include "block/qapi.h"
+#include "exec/address-spaces.h"
+#include "exec/cpu-all.h"
+#include "exec/ram_addr.h"
+#include "qemu/bitmap.h"
 
 #define SELF_ANNOUNCE_ROUNDS 5
 
@@ -1002,6 +1006,320 @@ void do_savevm(Monitor *mon, const QDict *qdict)
     }
 }
 
+/*
+ * Adding the functionality of continuous logging of the
+ * dirty bitmap which is almost similar to the migration
+ * thread
+ */
+
+enum {
+    LOG_BITMAP_STATE_ERROR = -1,
+    LOG_BITMAP_STATE_NONE,
+    LOG_BITMAP_STATE_SETUP,
+    LOG_BITMAP_STATE_ACTIVE,
+    LOG_BITMAP_STATE_CANCELLED,
+    LOG_BITMAP_STATE_CANCELING,
+    LOG_BITMAP_STATE_COMPLETED
+};
+
+typedef struct BitmapLogState BitmapLogState;
+static unsigned long *logging_bitmap;
+static int64_t LOG_SIZE_MAX = 100000;
+
+struct BitmapLogState {
+    int state;
+    int64_t current_frequency;
+    int64_t total_epochs;
+    bool readable;
+    QemuThread thread;
+    FILE *bitmap_dump_fp;
+};
+
+/*
+ * helper functions
+ */
+
+static inline void logging_lock(void)
+{
+    qemu_mutex_lock_iothread();
+    qemu_mutex_lock_ramlist();
+}
+
+static inline void logging_unlock(void)
+{
+    qemu_mutex_unlock_ramlist();
+    qemu_mutex_unlock_iothread();
+}
+
+static inline void logging_bitmap_set_dirty(ram_addr_t addr)
+{
+    int nr  = addr >> TARGET_PAGE_BITS;
+    set_bit(nr, logging_bitmap);
+}
+
+static bool logging_state_set_status(BitmapLogState *b,
+                                     int old_state,
+                                     int new_state)
+{
+    return atomic_cmpxchg(&b->state, old_state, new_state);
+}
+
+static inline void check_frequency_value(int64_t *frequency, Error **errp)
+{
+    if (*frequency < 10) {
+        error_setg(errp, "frequency should be greater "
+                        "than 10 milliseconds\n");
+        *frequency = 10;
+    }
+
+    if (*frequency > LOG_SIZE_MAX) {
+        error_setg(errp, "frequency greater than %ld (LOG_SIZE_MAX)\n",
+                   LOG_SIZE_MAX);
+        *frequency = LOG_SIZE_MAX;
+    }
+}
+
+static inline void check_epochs_value(int64_t *epochs, Error **errp)
+{
+    if (*epochs <= 0) {
+        error_setg(errp, "epoch size must be greater than 0, setting"
+                        " a value of 3\n");
+        *epochs = 3;
+    }
+
+    if (*epochs > LOG_SIZE_MAX) {
+        error_setg(errp, "epoch size greater than %ld (LOG_SIZE_MAX)\n",
+                   LOG_SIZE_MAX);
+        *epochs = LOG_SIZE_MAX;
+    }
+}
+
+static inline long sizeof_bitmap(long nbits)
+{
+    return BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+}
+
+/*
+ * inspired from migration mechanism
+ */
+
+static BitmapLogState *logging_current_state(void)
+{
+    static BitmapLogState current_bitmaplogstate = {
+        .state = LOG_BITMAP_STATE_NONE,
+        .readable = false,
+    };
+
+    return &current_bitmaplogstate;
+}
+
+/*
+ * syncing the logging_bitmap with the ram_list dirty bitmap
+ */
+
+/*
+ * copied from arch_init.c file (below function)
+ */
+
+static void dirty_bitmap_logging_sync_range(ram_addr_t start,
+                                            ram_addr_t length)
+{
+    ram_addr_t addr;
+    unsigned long long page = BIT_WORD(start >> TARGET_PAGE_BITS);
+
+    if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
+        int k;
+        int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
+        unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_LOG_BITMAP];
+        for (k = page; k < page + nr; k++) {
+            if (src[k]) {
+                unsigned long new_dirty;
+                new_dirty = ~logging_bitmap[k];
+                logging_bitmap[k] |= src[k];
+                new_dirty &= src[k];
+                src[k] = 0;
+            }
+        }
+    } else {
+        for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
+            if (cpu_physical_memory_get_dirty(start + addr,
+                        TARGET_PAGE_SIZE,
+                        DIRTY_MEMORY_LOG_BITMAP)) {
+                cpu_physical_memory_reset_dirty(start + addr,
+                        TARGET_PAGE_SIZE,
+                        DIRTY_MEMORY_LOG_BITMAP);
+                logging_bitmap_set_dirty(start + addr);
+            }
+        }
+    }
+}
+
+static void dirty_bitmap_sync(void)
+{
+    RAMBlock *block;
+    address_space_sync_dirty_bitmap(&address_space_memory);
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+        dirty_bitmap_logging_sync_range(block->mr->ram_addr, block->length);
+    }
+}
+
+static void write_file(FILE *dump_fp, int64_t ram_bitmap_pages, bool readable)
+{
+    int64_t i = 0;
+    int64_t bitmap_length = BITS_TO_LONGS(ram_bitmap_pages);
+
+    if (readable) {
+        for (; i < bitmap_length - 1; i++) {
+            fprintf(dump_fp, "%lx ", logging_bitmap[i]);
+        }
+        fprintf(dump_fp, "%lx\n", logging_bitmap[i]);
+    } else {
+        fwrite(logging_bitmap, sizeof(unsigned long), bitmap_length, dump_fp);
+    }
+}
+
+static inline void logging_bitmap_close(BitmapLogState *b)
+{
+    logging_lock();
+    memory_global_dirty_log_stop();
+    logging_unlock();
+
+    g_free(logging_bitmap);
+    fclose(b->bitmap_dump_fp);
+    b->bitmap_dump_fp = NULL;
+    logging_bitmap = NULL;
+    b = NULL;
+}
+
+static void *bitmap_logging_thread(void *opaque)
+{
+    /*
+     * setup basic structures
+     */
+
+    BitmapLogState *b = opaque;
+    int64_t curr_epoch = 0;
+    int64_t total_epochs = b->total_epochs;
+    FILE *dump_fp = b->bitmap_dump_fp;
+    bool readable = b->readable;
+    int64_t ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+    int64_t bitmap_length = BITS_TO_LONGS(ram_bitmap_pages);
+
+    logging_state_set_status(b, LOG_BITMAP_STATE_NONE,
+                                LOG_BITMAP_STATE_SETUP);
+
+    logging_bitmap = bitmap_new(ram_bitmap_pages);
+
+    if (logging_bitmap == NULL) {
+        b->state = LOG_BITMAP_STATE_ERROR;
+        goto log_thread_end;
+    }
+
+    logging_state_set_status(b, LOG_BITMAP_STATE_SETUP,
+                                LOG_BITMAP_STATE_ACTIVE);
+    /*
+     *  start the logging period
+     */
+    logging_lock();
+    memory_global_dirty_log_start();
+    dirty_bitmap_sync();
+    bitmap_zero(logging_bitmap, ram_bitmap_pages);
+    logging_unlock();
+
+    if (readable) {
+        fprintf(dump_fp, "Total RAM pages: %ld\n"
+                         "Bitmap length: %ld\n",
+                         ram_bitmap_pages, bitmap_length);
+    } else {
+        fwrite(&ram_bitmap_pages, sizeof(int64_t), 1, dump_fp);
+        fwrite(&bitmap_length, sizeof(int64_t), 1, dump_fp);
+    }
+
+    /*
+     * sync the dirty bitmap along with saving it
+     * using the FILE pointer f.
+     */
+    while (curr_epoch < total_epochs) {
+        if (!runstate_is_running() || b->state != LOG_BITMAP_STATE_ACTIVE) {
+            goto log_thread_end;
+        }
+        bitmap_zero(logging_bitmap, ram_bitmap_pages);
+        logging_lock();
+        dirty_bitmap_sync();
+        logging_unlock();
+        write_file(dump_fp, ram_bitmap_pages, readable);
+        g_usleep(b->current_frequency * 1000);
+        curr_epoch++;
+    }
+
+    /*
+     * stop the logging period.
+     */
+ log_thread_end:
+    logging_bitmap_close(b);
+    if (b->state == LOG_BITMAP_STATE_ACTIVE) {
+        logging_state_set_status(b, LOG_BITMAP_STATE_ACTIVE,
+                                    LOG_BITMAP_STATE_COMPLETED);
+    } else if (b->state == LOG_BITMAP_STATE_CANCELING) {
+        logging_state_set_status(b, LOG_BITMAP_STATE_CANCELING,
+                                    LOG_BITMAP_STATE_CANCELLED);
+    }
+    return NULL;
+}
+
+void qmp_log_dirty_bitmap(const char *filename, bool has_epochs,
+                          int64_t epochs, bool has_frequency,
+                          int64_t frequency, bool has_readable,
+                          bool readable, Error **errp)
+{
+    FILE *f;
+    BitmapLogState *b = logging_current_state();
+
+    if (b->state == LOG_BITMAP_STATE_ACTIVE ||
+            b->state == LOG_BITMAP_STATE_SETUP ||
+            b->state == LOG_BITMAP_STATE_CANCELING) {
+        error_set(errp, QERR_LOG_DIRTY_BITMAP_ACTIVE);
+        return;
+    }
+
+    if (b->state == LOG_BITMAP_STATE_COMPLETED) {
+        logging_state_set_status(b, LOG_BITMAP_STATE_COMPLETED,
+                                    LOG_BITMAP_STATE_NONE);
+    } else if (b->state == LOG_BITMAP_STATE_CANCELLED) {
+        logging_state_set_status(b, LOG_BITMAP_STATE_CANCELLED,
+                                    LOG_BITMAP_STATE_NONE);
+    }
+
+    if (readable) {
+        f = fopen(filename, "w");
+    } else {
+        f = fopen(filename, "wb");
+    }
+    if (!f) {
+        error_setg_file_open(errp, errno, filename);
+        goto no_bitmap_end;
+    }
+
+    if (!has_epochs) {
+        epochs = 3;
+    }
+    if (!has_frequency) {
+        frequency = 40;
+    }
+    check_epochs_value(&epochs, errp);
+    check_frequency_value(&frequency, errp);
+    b->total_epochs = epochs;
+    b->current_frequency = frequency;
+    b->bitmap_dump_fp = f;
+    b->readable = readable;
+    qemu_thread_create(&b->thread, "dirty-bitmap-logging",
+                       bitmap_logging_thread, b,
+                       QEMU_THREAD_JOINABLE);
+
+ no_bitmap_end:
+    return;
+}
+
 void qmp_xen_save_devices_state(const char *filename, Error **errp)
 {
     QEMUFile *f;
-- 
1.8.3.1