[Qemu-devel] [RFC V8 16/24] block: Add qcow2_dedup format and image creation code.

[Benoît Canet]

Also modify qemu-io-test.

Signed-off-by: Benoit Canet <address@hidden>
---
 block/qcow2.c                |  176 ++++++++++++++++++++++++++++++++++++++++--
 include/block/block_int.h    |    1 +
 tests/qemu-iotests/common.rc |    3 +-
 3 files changed, 173 insertions(+), 7 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index 58e6236..e1265a2 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1313,7 +1313,8 @@ static int preallocate(BlockDriverState *bs)
 static int qcow2_create2(const char *filename, int64_t total_size,
                          const char *backing_file, const char *backing_format,
                          int flags, size_t cluster_size, int prealloc,
-                         QEMUOptionParameter *options, int version)
+                         QEMUOptionParameter *options, int version,
+                         bool dedup, uint8_t hash_algo)
 {
     /* Calculate cluster_bits */
     int cluster_bits;
@@ -1417,11 +1418,39 @@ static int qcow2_create2(const char *filename, int64_t 
total_size,
     }
 
     /* Okay, now that we have a valid image, let's give it the right size */
+    BDRVQcowState *s = bs->opaque;
     ret = bdrv_truncate(bs, total_size * BDRV_SECTOR_SIZE);
     if (ret < 0) {
         goto out;
     }
 
+    if (dedup) {
+        s->has_dedup = true;
+        /* fill cluster size for qcow2_store_create */
+        s->cluster_size = cluster_size;
+        /* unlimited incarnation number */
+        s->dedup_max_incarnations = 0;
+        s->dedup_hash_algo = hash_algo;
+
+        ret = qcow2_set_incompat_feature(bs, QCOW2_INCOMPAT_DEDUP);
+        if (ret < 0) {
+            goto out;
+        }
+
+        /* this will create the journal and save the conf to disk then
+         * save header extension
+         */
+        ret = qcow2_store_create(bs,
+                                 &s->key_value_store,
+                                 total_size * BDRV_SECTOR_SIZE);
+
+        if (ret < 0) {
+            goto out;
+        }
+
+        qcow2_store_cleanup(bs, &s->key_value_store);
+    }
+
     /* Want a backing file? There you go.*/
     if (backing_file) {
         ret = bdrv_change_backing_file(bs, backing_file, backing_format);
@@ -1447,15 +1476,41 @@ out:
     return ret;
 }
 
+static int qcow2_warn_if_version_3_is_needed(int version,
+                                             bool has_feature,
+                                             const char *feature)
+{
+    if (version < 3 && has_feature) {
+        fprintf(stderr, "%s only supported with compatibility "
+                "level 1.1 and above (use compat=1.1 or greater)\n",
+                feature);
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static int8_t qcow2_get_dedup_hash_algo(char *value)
+{
+    if (!value || !strcmp(value, "sha256")) {
+        return QCOW_HASH_SHA256;
+    }
+
+    error_printf("Unsupported deduplication hash algorithm.\n");
+    return -EINVAL;
+}
+
 static int qcow2_create(const char *filename, QEMUOptionParameter *options)
 {
     const char *backing_file = NULL;
     const char *backing_fmt = NULL;
     uint64_t sectors = 0;
     int flags = 0;
+    int ret;
     size_t cluster_size = DEFAULT_CLUSTER_SIZE;
     int prealloc = 0;
     int version = 2;
+    bool dedup = false;
+    int8_t hash_algo = 0;
 
     /* Read out options */
     while (options && options->name) {
@@ -1493,6 +1548,13 @@ static int qcow2_create(const char *filename, 
QEMUOptionParameter *options)
             }
         } else if (!strcmp(options->name, BLOCK_OPT_LAZY_REFCOUNTS)) {
             flags |= options->value.n ? BLOCK_FLAG_LAZY_REFCOUNTS : 0;
+        } else if (!strcmp(options->name, BLOCK_OPT_DEDUP)) {
+            hash_algo = qcow2_get_dedup_hash_algo(options->value.s);
+            if (hash_algo < 0) {
+                return hash_algo;
+            }
+            dedup = true;
+            version = 3;
         }
         options++;
     }
@@ -1503,14 +1565,22 @@ static int qcow2_create(const char *filename, 
QEMUOptionParameter *options)
         return -EINVAL;
     }
 
-    if (version < 3 && (flags & BLOCK_FLAG_LAZY_REFCOUNTS)) {
-        fprintf(stderr, "Lazy refcounts only supported with compatibility "
-                "level 1.1 and above (use compat=1.1 or greater)\n");
-        return -EINVAL;
+    ret = qcow2_warn_if_version_3_is_needed(version,
+                                            flags & BLOCK_FLAG_LAZY_REFCOUNTS,
+                                            "Lazy refcounts");
+    if (ret < 0) {
+        return ret;
+    }
+    ret = qcow2_warn_if_version_3_is_needed(version,
+                                            dedup,
+                                            "Deduplication");
+    if (ret < 0) {
+        return ret;
     }
 
     return qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
-                         cluster_size, prealloc, options, version);
+                         cluster_size, prealloc, options, version,
+                         dedup, hash_algo);
 }
 
 static int qcow2_make_empty(BlockDriverState *bs)
@@ -1829,6 +1899,51 @@ static QEMUOptionParameter qcow2_create_options[] = {
     { NULL }
 };
 
+static QEMUOptionParameter qcow2_dedup_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    {
+        .name = BLOCK_OPT_BACKING_FILE,
+        .type = OPT_STRING,
+        .help = "File name of a base image"
+    },
+    {
+        .name = BLOCK_OPT_BACKING_FMT,
+        .type = OPT_STRING,
+        .help = "Image format of the base image"
+    },
+    {
+        .name = BLOCK_OPT_ENCRYPT,
+        .type = OPT_FLAG,
+        .help = "Encrypt the image"
+    },
+    {
+        .name = BLOCK_OPT_CLUSTER_SIZE,
+        .type = OPT_SIZE,
+        .help = "qcow2 cluster size",
+        .value = { .n = DEFAULT_DEDUP_CLUSTER_SIZE },
+    },
+    {
+        .name = BLOCK_OPT_PREALLOC,
+        .type = OPT_STRING,
+        .help = "Preallocation mode (allowed values: off, metadata)"
+    },
+    {
+        .name = BLOCK_OPT_LAZY_REFCOUNTS,
+        .type = OPT_FLAG,
+        .help = "Postpone refcount updates",
+    },
+    {
+        .name = BLOCK_OPT_DEDUP,
+        .type = OPT_STRING,
+        .help = "Deduplication",
+    },
+    { NULL }
+};
+
 static BlockDriver bdrv_qcow2 = {
     .format_name        = "qcow2",
     .instance_size      = sizeof(BDRVQcowState),
@@ -1868,9 +1983,58 @@ static BlockDriver bdrv_qcow2 = {
     .bdrv_check = qcow2_check,
 };
 
+/* As all the defined .create_options are passed to qcow2_create() even if
+ * the user does not specify them it's not possible to have a default 4KB
+ * cluster size for deduplication.
+ * For example it's impossible to make the difference between the 64KB cluster
+ * size default create option of qcow2 or a 64KB user specified cluster size.
+ * So we declare the qcow2_dedup format in order to be able to define
+ * deduplication specific create options.
+ * It will also help for qemu-io-test integration.
+ */
+static BlockDriver bdrv_qcow2_dedup = {
+    .format_name        = "qcow2_dedup",
+    .instance_size      = sizeof(BDRVQcowState),
+    .bdrv_probe         = qcow2_probe,
+    .bdrv_open          = qcow2_open,
+    .bdrv_close         = qcow2_close,
+    .bdrv_reopen_prepare  = qcow2_reopen_prepare,
+    .bdrv_create        = qcow2_create,
+    .bdrv_co_is_allocated = qcow2_co_is_allocated,
+    .bdrv_set_key       = qcow2_set_key,
+    .bdrv_make_empty    = qcow2_make_empty,
+
+    .bdrv_co_readv          = qcow2_co_readv,
+    .bdrv_co_writev         = qcow2_co_writev,
+    .bdrv_co_flush_to_os    = qcow2_co_flush_to_os,
+
+    .bdrv_co_write_zeroes   = qcow2_co_write_zeroes,
+    .bdrv_co_discard        = qcow2_co_discard,
+    .bdrv_truncate          = qcow2_truncate,
+    .bdrv_write_compressed  = qcow2_write_compressed,
+
+    .bdrv_snapshot_create   = qcow2_snapshot_create,
+    .bdrv_snapshot_goto     = qcow2_snapshot_goto,
+    .bdrv_snapshot_delete   = qcow2_snapshot_delete,
+    .bdrv_snapshot_list     = qcow2_snapshot_list,
+    .bdrv_snapshot_load_tmp     = qcow2_snapshot_load_tmp,
+    .bdrv_get_info      = qcow2_get_info,
+
+    .bdrv_save_vmstate    = qcow2_save_vmstate,
+    .bdrv_load_vmstate    = qcow2_load_vmstate,
+
+    .bdrv_change_backing_file   = qcow2_change_backing_file,
+
+    .bdrv_invalidate_cache      = qcow2_invalidate_cache,
+
+    .create_options = qcow2_dedup_create_options,
+    .bdrv_check = qcow2_check,
+};
+
 static void bdrv_qcow2_init(void)
 {
     bdrv_register(&bdrv_qcow2);
+    bdrv_register(&bdrv_qcow2_dedup);
 }
 
 block_init(bdrv_qcow2_init);
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 6078dd3..ef4d5d5 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -57,6 +57,7 @@
 #define BLOCK_OPT_COMPAT_LEVEL      "compat"
 #define BLOCK_OPT_LAZY_REFCOUNTS    "lazy_refcounts"
 #define BLOCK_OPT_ADAPTER_TYPE      "adapter_type"
+#define BLOCK_OPT_DEDUP             "dedup"
 
 typedef struct BdrvTrackedRequest BdrvTrackedRequest;
 
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index 31eb62b..71b67f7 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -130,7 +130,8 @@ _make_test_img()
             -e "s# zeroed_grain=\\(on\\|off\\)##g" \
             -e "s# subformat='[^']*'##g" \
             -e "s# adapter_type='[^']*'##g" \
-            -e "s# lazy_refcounts=\\(on\\|off\\)##g"
+            -e "s# lazy_refcounts=\\(on\\|off\\)##g" \
+            -e "s# dedup=\\('sha256'\\|'skein'\\|'sha3'\\)##g"
 
     # Start an NBD server on the image file, which is what we'll be talking to
     if [ $IMGPROTO = "nbd" ]; then
-- 
1.7.10.4