[Qemu-devel] [PATCH v6 09/11] dump: Add API to write dump pages

[Qiao Nuohan]

functions are used to write page to vmcore. vmcore is written page by page.
page desc is used to store the information of a page, including a page's size,
offset, compression format, etc.

Signed-off-by: Qiao Nuohan <address@hidden>
---
 dump.c                |  258 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/sysemu/dump.h |    9 ++
 2 files changed, 267 insertions(+), 0 deletions(-)

diff --git a/dump.c b/dump.c
index b4c40f2..848957c 100644
--- a/dump.c
+++ b/dump.c
@@ -25,6 +25,14 @@
 #include "qapi/error.h"
 #include "qmp-commands.h"
 
+#include <zlib.h>
+#ifdef CONFIG_LZO
+#include <lzo/lzo1x.h>
+#endif
+#ifdef CONFIG_SNAPPY
+#include <snappy-c.h>
+#endif
+
 static uint16_t cpu_convert_to_target16(uint16_t val, int endian)
 {
     if (endian == ELFDATA2LSB) {
@@ -1140,6 +1148,256 @@ static void free_data_cache(DataCache *data_cache)
     g_free(data_cache->buf);
 }
 
+static size_t get_len_buf_out(size_t page_size, uint32_t flag_compress)
+{
+    size_t len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy;
+    size_t len_buf_out;
+
+    /* init buf_out */
+    len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = 0;
+
+    /* buf size for zlib */
+    len_buf_out_zlib = compressBound(page_size);
+
+    /* buf size for lzo */
+#ifdef CONFIG_LZO
+    if (flag_compress & DUMP_DH_COMPRESSED_LZO) {
+        if (lzo_init() != LZO_E_OK) {
+            /* return 0 to indicate lzo is unavailable */
+            return 0;
+        }
+    }
+
+    /*
+     * LZO will expand incompressible data by a little amount. please check the
+     * following URL to see the expansion calculation:
+     * http://www.oberhumer.com/opensource/lzo/lzofaq.php
+     */
+    len_buf_out_lzo = page_size + page_size / 16 + 64 + 3;
+#endif
+
+#ifdef CONFIG_SNAPPY
+    /* buf size for snappy */
+    len_buf_out_snappy = snappy_max_compressed_length(page_size);
+#endif
+
+    /* get the biggest that can store all kinds of compressed page */
+    len_buf_out = MAX(len_buf_out_zlib,
+                      MAX(len_buf_out_lzo, len_buf_out_snappy));
+
+    return len_buf_out;
+}
+
+/*
+ * search memory blocks to find the exact place of the specified page, then
+ * dump the page into buf. memory should be read page by page, or it may exceed
+ * the boundary and fail to read
+ */
+static int readmem(void *bufptr, ram_addr_t addr, size_t size, DumpState *s)
+{
+    GuestPhysBlock *block;
+
+    QTAILQ_FOREACH(block, &s->guest_phys_blocks.head, next) {
+        if ((addr >= block->target_start) &&
+            (addr + size <= block->target_end)) {
+            memcpy(bufptr, block->host_addr + (addr - block->target_start),
+                    size);
+            return 0;
+        }
+    }
+
+    return -1;
+}
+
+/*
+ * check if the page is all 0
+ */
+static inline bool is_zero_page(unsigned char *buf, long page_size)
+{
+    return buffer_is_zero(buf, page_size);
+}
+
+static int write_dump_pages(DumpState *s)
+{
+    int ret = 0;
+    DataCache page_desc, page_data;
+    size_t len_buf_out, size_out;
+#ifdef CONFIG_LZO
+    lzo_bytep wrkmem = NULL;
+#endif
+    unsigned char *buf_out = NULL;
+    off_t offset_desc, offset_data;
+    PageDesc pd, pd_zero;
+    uint64_t pfn_start, pfn_end, pfn;
+    unsigned char buf[s->page_size];
+    MemoryMapping *memory_mapping;
+    bool zero_page;
+
+    prepare_data_cache(&page_desc, s);
+    prepare_data_cache(&page_data, s);
+
+    /* prepare buffer to store compressed data */
+    len_buf_out = get_len_buf_out(s->page_size, s->flag_compress);
+    if (len_buf_out == 0) {
+        dump_error(s, "dump: failed to get length of output buffer.\n");
+        goto out;
+    }
+
+#ifdef CONFIG_LZO
+    wrkmem = g_malloc(LZO1X_1_MEM_COMPRESS);
+#endif
+
+    buf_out = g_malloc(len_buf_out);
+
+    /* get offset of page_desc and page_data in dump file */
+    offset_desc = s->offset_page;
+    offset_data = offset_desc + sizeof(PageDesc) * s->num_dumpable;
+    page_desc.offset = offset_desc;
+    page_data.offset = offset_data;
+
+    /*
+     * init zero page's page_desc and page_data, because every zero page
+     * uses the same page_data
+     */
+    pd_zero.size = s->page_size;
+    pd_zero.flags = 0;
+    pd_zero.offset = offset_data;
+    pd_zero.page_flags = 0;
+    memset(buf, 0, pd_zero.size);
+    ret = write_cache(&page_data, s->flag_flatten, buf, pd_zero.size);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write page data(zero page).\n");
+        goto out;
+    }
+
+    offset_data += pd_zero.size;
+
+    /* dump memory to vmcore page by page */
+    QTAILQ_FOREACH(memory_mapping, &s->list.head, next) {
+        pfn_start = paddr_to_pfn(memory_mapping->phys_addr, s->page_shift);
+        pfn_end = paddr_to_pfn(memory_mapping->phys_addr +
+                               memory_mapping->length, s->page_shift);
+
+        for (pfn = pfn_start; pfn < pfn_end; pfn++) {
+            memset(buf, 0, s->page_size);
+            ret = readmem(buf, pfn_to_paddr(pfn, s->page_shift), s->page_size,
+                          s);
+            if (ret < 0) {
+                dump_error(s, "dump: failed to read memory.\n");
+                goto out;
+            }
+
+            /* check zero page */
+            zero_page = is_zero_page(buf, s->page_size);
+            if (zero_page) {
+                ret = write_cache(&page_desc, s->flag_flatten, &pd_zero,
+                                  sizeof(PageDesc));
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page desc.\n");
+                    goto out;
+                }
+            } else {
+                /*
+                 * not zero page, then:
+                 * 1. compress the page
+                 * 2. write the compressed page into the cache of page_data
+                 * 3. get page desc of the compressed page and write it into 
the
+                 *    cache of page_desc
+                 */
+                size_out = len_buf_out;
+                if ((s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) &&
+                        (compress2(buf_out, &size_out, buf, s->page_size,
+                        Z_BEST_SPEED) == Z_OK) && (size_out < s->page_size)) {
+                    pd.flags = DUMP_DH_COMPRESSED_ZLIB;
+                    pd.size  = size_out;
+
+                    ret = write_cache(&page_data, s->flag_flatten, buf_out,
+                                      pd.size);
+                    if (ret < 0) {
+                        dump_error(s, "dump: failed to write page data.\n");
+                        goto out;
+                    }
+#ifdef CONFIG_LZO
+                } else if ((s->flag_compress & DUMP_DH_COMPRESSED_LZO) &&
+                        (lzo1x_1_compress(buf, s->page_size, buf_out,
+                        &size_out, wrkmem) == LZO_E_OK) &&
+                        (size_out < s->page_size)) {
+                    pd.flags = DUMP_DH_COMPRESSED_LZO;
+                    pd.size  = size_out;
+
+                    ret = write_cache(&page_data, s->flag_flatten, buf_out,
+                                      pd.size);
+                    if (ret < 0) {
+                        dump_error(s, "dump: failed to write page data.\n");
+                        goto out;
+                    }
+#endif
+#ifdef CONFIG_SNAPPY
+                } else if ((s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) &&
+                        (snappy_compress((char *)buf, s->page_size,
+                        (char *)buf_out, (size_t *)&size_out) == SNAPPY_OK) &&
+                        (size_out < s->page_size)) {
+                    pd.flags = DUMP_DH_COMPRESSED_SNAPPY;
+                    pd.size  = size_out;
+
+                    ret = write_cache(&page_data, s->flag_flatten, buf_out,
+                                      pd.size);
+                    if (ret < 0) {
+                        dump_error(s, "dump: failed to write page data.\n");
+                        goto out;
+                    }
+#endif
+                } else {
+                    pd.flags = 0;
+                    pd.size = s->page_size;
+
+                    ret = write_cache(&page_data, s->flag_flatten, buf,
+                                      pd.size);
+                    if (ret < 0) {
+                        dump_error(s, "dump: failed to write page data.\n");
+                        goto out;
+                    }
+                }
+
+                /* get and write page desc here */
+                pd.page_flags = 0;
+                pd.offset = offset_data;
+                offset_data += pd.size;
+
+                ret = write_cache(&page_desc, s->flag_flatten, &pd,
+                                  sizeof(PageDesc));
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page desc.\n");
+                    goto out;
+                }
+            }
+        }
+    }
+
+    ret = sync_data_cache(&page_desc, s->flag_flatten);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_desc.\n");
+        goto out;
+    }
+    ret = sync_data_cache(&page_data, s->flag_flatten);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_data.\n");
+        goto out;
+    }
+
+out:
+    free_data_cache(&page_desc);
+    free_data_cache(&page_data);
+
+#ifdef CONFIG_LZO
+    g_free(wrkmem);
+#endif
+
+    g_free(buf_out);
+
+    return ret;
+}
+
 static ram_addr_t get_start_block(DumpState *s)
 {
     GuestPhysBlock *block;
diff --git a/include/sysemu/dump.h b/include/sysemu/dump.h
index ab44af8..382a3c3 100644
--- a/include/sysemu/dump.h
+++ b/include/sysemu/dump.h
@@ -40,6 +40,8 @@
 
 #define paddr_to_pfn(X, page_shift) \
     (((unsigned long long)(X) >> (page_shift)) - ARCH_PFN_OFFSET)
+#define pfn_to_paddr(X, page_shift) \
+    (((unsigned long long)(X) + ARCH_PFN_OFFSET) << (page_shift))
 
 typedef struct ArchDumpInfo {
     int d_machine;  /* Architecture */
@@ -149,6 +151,13 @@ typedef struct DataCache {
     off_t offset;       /* offset of the file */
 } DataCache;
 
+typedef struct PageDesc {
+    uint64_t offset;                   /* the offset of the page data*/
+    uint32_t size;                  /* the size of this dump page */
+    uint32_t flags;                 /* flags */
+    uint64_t page_flags;            /* page flags */
+} PageDesc;
+
 struct GuestPhysBlockList; /* memory_mapping.h */
 int cpu_get_dump_info(ArchDumpInfo *info,
                       const struct GuestPhysBlockList *guest_phys_blocks);
-- 
1.7.1