[Qemu-devel] [Patch 4/4] [RFC] Zero Cluster Dedup, Offline dedup, qemu-img extentions

[Shahar Frank]

General dedup verb (qemu-img, qcow2)

Signed-off-by: Shahar Frank <address@hidden>

diff --git a/block-qcow2.c b/block-qcow2.c
index 0a5d4c0..f337f23 100644
--- a/block-qcow2.c
+++ b/block-qcow2.c
@@ -1235,6 +1235,44 @@ static int dedup_clusters(BlockDriverState *bs,
uint64_t loffset, uint64_t poffs
 }
 
 /*
+ * dedup: remap logical offset(s) starting at 'src' up to 'src' +
'nclusters'
+ * (not including) to share storage with the logical cluster at offset
'dest' (many to one).
+ * The original cluster(s) storage space will be freed if it is not the
same as the new one.
+ * Returns the number of remapped clusters, or -1 in case of errors.
+ * Notes:
+ *  If 'dst' is not mapped, it will be allocated and filled with zeros.
+ *  'Src' clusters may or may not be mapped before the call.
+ *  Mapping a cluster to itself is legal. It will do nothing if the
cluster is COW, and it
+ *  will turn it to COW otherwise.
+ *  A similar effect can be achieved if using nclustesr=0.
+ */
+static int qcow_dedup(BlockDriverState *bs, uint64_t src, uint64_t dst,
int nclusters)
+{
+    uint64_t l2_offset, *l2_table, poffset;
+    BDRVQcowState *s = bs->opaque;
+    int l2_index;
+
+    if (get_cluster_table(bs, dst, &l2_table, &l2_offset, &l2_index) ==
0)
+        return 0;
+
+    poffset = be64_to_cpu(l2_table[l2_index]);
+    DEBUG("%"PRIx64" to %"PRIx64" (%"PRIx64") (%d)", src, dst, poffset,
nclusters);
+
+    /* if the destination cluster is copied, we have to turn it first
into COW */
+    if (poffset & QCOW_OFLAG_COPIED) {
+        poffset &= ~QCOW_OFLAG_COPIED;
+        l2_table[l2_index] = cpu_to_be64(poffset);
+
+        if (bdrv_pwrite(s->hd,
+                    l2_offset + l2_index * sizeof(uint64_t),
+                    l2_table + l2_index, sizeof(uint64_t)) !=
sizeof(uint64_t))
+        return -1;
+    }
+
+    return dedup_clusters(bs, src, poffset, nclusters);
+}
+
+/*
  * Attempt to optimize data writing. Currently only zero cluster dedup
is supported.
  * 'sector_num' is the loggical offset of the data in sectors.
  * 'nb_sectors' (in param) is the maximux number of sectors to dedup.
@@ -2825,5 +2863,6 @@ BlockDriver bdrv_qcow2 = {
     .bdrv_snapshot_list = qcow_snapshot_list,
     .bdrv_get_info = qcow_get_info,
     .bdrv_map = qcow_map,
+    .bdrv_dedup = qcow_dedup,
     .bdrv_check = check_refcounts,
 };
diff --git a/block_int.h b/block_int.h
index 6195fe3..f280d9a 100644
--- a/block_int.h
+++ b/block_int.h
@@ -87,6 +87,7 @@ struct BlockDriver {
 
     /* misc */
     int64_t (*bdrv_map)(BlockDriverState *bs, int64_t offset, int
local, BlockDriverState **dst);
+    int (*bdrv_dedup)(BlockDriverState *bs, uint64_t src, uint64_t
dest, int nclusters);
     int (*bdrv_check)(BlockDriverState *bs);
     BlockDriverAIOCB *free_aiocb;
     struct BlockDriver *next;
diff --git a/qemu-img.c b/qemu-img.c
index 8cda447..dd70193 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -59,6 +59,7 @@ static void help(void)
            "  convert [-c] [-e] [-6] [-f fmt] [-O output_fmt] [-B
output_base_image] filename [filename2 [...]] output_filename\n"
            "  info [-f fmt] [ -r ]filename\n"
            "  map [ -f fmt] [ -r ] [ -s ] filename\n"
+           "  dedup [ -f fmt] filename src-offset dest-offset [ count
]\n"
            "\n"
            "Command parameters:\n"
            "  'filename' is a disk image filename\n"
@@ -73,6 +74,8 @@ static void help(void)
            "    and 'G' (gigabyte) are supported\n"
            "  'output_filename' is the destination disk image
filename\n"
            "  'output_fmt' is the destination format\n"
+           "  'src-offset' is the source cluster logical offset (guest
side offset)\n"
+           "  'dest-offset' is the destination cluster logical offset
(guest side offset)\n"
            "  '-c' indicates that target image must be compressed (qcow
format only)\n"
            "  '-e' indicates that the target image must be encrypted
(qcow format only)\n"
            "  '-6' indicates that the target image must use
compatibility level 6 (vmdk format only)\n"
@@ -551,6 +554,88 @@ static int img_map(int argc, char **argv)
     return 0;
 }
 
+static int img_dedup(int argc, char **argv)
+{
+    int c, ret = 0, cluster_size, cluster_sectors;
+    const char *filename, *fmt;
+    BlockDriver *drv;
+    BlockDriverState *bs;
+    int64_t loffset, poffset, e;
+    uint64_t bs_sectors;
+    BlockDriverInfo bdi;
+    int count = 1;
+    
+    fmt = NULL;
+    for(;;) {
+        c = getopt(argc, argv, "f:h");
+        if (c == -1)
+            break;
+        switch(c) {
+        case 'h':
+            help();
+            break;
+        case 'f':
+            fmt = optarg;
+            break;
+        }
+    }
+    if (optind + 3 > argc)
+        help();
+    filename = argv[optind++];
+    loffset = strtoull(argv[optind++], 0, 0);
+    poffset = strtoull(argv[optind++], 0, 0);
+    if (optind < argc)
+        count = strtoul(argv[optind++], 0, 0);
+    if (count <= 0)
+        error("bad count %d", count);
+
+    bs = bdrv_new("");
+    if (!bs)
+        error("Not enough memory");
+    if (fmt) {
+        drv = bdrv_find_format(fmt);
+        if (!drv)
+            error("Unknown file format '%s'", fmt);
+    } else {
+        drv = NULL;
+    }
+    if (bdrv_open2(bs, filename, 0, drv) < 0) {
+        error("Could not open '%s'", filename);
+    }
+    if (!bs->drv->bdrv_dedup)
+        error("image format does not support remap");
+
+    bdrv_get_geometry(bs, &bs_sectors);
+
+    if (bdrv_get_info(bs, &bdi) < 0)
+        error("could not get block driver info");
+    cluster_size = bdi.cluster_size;
+    if (cluster_size <= 0 || cluster_size > IO_BUF_SIZE)
+        error("invalid cluster size");
+    cluster_sectors = cluster_size >> 9;
+    e = bs_sectors << 9;
+
+    if (loffset >= e)
+        error("logicall offset %"PRIx64" is out of image size range",
loffset);
+
+    if ((ret = bs->drv->bdrv_dedup(bs, loffset, poffset, count)) > 0) {
+        for (; count; count--, loffset += cluster_size)
+            printf("%s 0x%016llx 0x%016llx\n", filename, (long long
unsigned int)loffset, (long long unsigned int)poffset);
+    }
+
+    switch(ret) {
+    case 0:
+        printf("remap failed\n");
+        break;
+    case -1:
+        error("IO error");
+        break;
+    }
+
+    bdrv_delete(bs);
+    return 0;
+}
+
 static int img_check(int argc, char **argv)
 {
     int c, ret = 0, cluster_size, cluster_sectors;
@@ -1026,6 +1111,8 @@ int main(int argc, char **argv)
         img_info(argc, argv);
     } else if (!strcmp(cmd, "map")) {
         img_map(argc, argv);
+    } else if (!strcmp(cmd, "dedup")) {
+        img_dedup(argc, argv);
     } else if (!strcmp(cmd, "check")) {
         ret = img_check(argc, argv);
     } else {

4-dedup.patch
Description: 4-dedup.patch