Re: [Qemu-block] [PATCH v2] qemu-img: add the 'dd' subcommand

[Fam Zheng]

On Mon, 07/18 12:00, Reda Sallahi wrote:
> This patch adds a basic dd subcommand analogous to dd(1) to qemu-img.

Please submit subsequent revisions as top level email. I.e. no In-Reply-To to
previous version.

> 
> For the start, this implements the bs, if, of and count options and requires
> both if and of to be specified (no stdin/stdout if not specified) and doesn't
> support tty, pipes, etc.
> 
> The image format must be specified with -O for the output if the raw format
> is not the intended one.
> 
> Two tests are added to test qemu-img dd.
> 

Looks very close, see some minor comments below.

> Signed-off-by: Reda Sallahi <address@hidden>
> ---
> Changes from v1:
> * Removal of dead code.
> * Fix a memory leak.
> * Complete the cleanup function in the test cases.
> 
>  qemu-img-cmds.hx           |   6 +
>  qemu-img.c                 | 366 
> ++++++++++++++++++++++++++++++++++++++++++++-
>  tests/qemu-iotests/158     |  54 +++++++
>  tests/qemu-iotests/158.out |  15 ++
>  tests/qemu-iotests/159     |  57 +++++++
>  tests/qemu-iotests/159.out |  87 +++++++++++
>  tests/qemu-iotests/group   |   2 +
>  7 files changed, 586 insertions(+), 1 deletion(-)
>  create mode 100755 tests/qemu-iotests/158
>  create mode 100644 tests/qemu-iotests/158.out
>  create mode 100755 tests/qemu-iotests/159
>  create mode 100644 tests/qemu-iotests/159.out
> 
> diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
> index 7e95b2d..03bdd7a 100644
> --- a/qemu-img-cmds.hx
> +++ b/qemu-img-cmds.hx
> @@ -45,6 +45,12 @@ STEXI
>  @item convert [--object @var{objectdef}] [--image-opts] [-c] [-p] [-q] [-n] 
> [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] 
> [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] 
> [-S @var{sparse_size}] @var{filename} address@hidden [...]] 
> @var{output_filename}
>  ETEXI
>  
> +DEF("dd", img_dd,
> +    "dd [--image-opts] [-f fmt] [-O output_fmt] [bs=block_size] 
> [count=blocks] if=input of=output")
> +STEXI
> address@hidden dd [--image-opts] [-f @var{fmt}] [-O @var{output_fmt}] 
> address@hidden address@hidden address@hidden address@hidden
> +ETEXI
> +
>  DEF("info", img_info,
>      "info [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] 
> [--backing-chain] filename")
>  STEXI
> diff --git a/qemu-img.c b/qemu-img.c
> index 2e40e1f..329f01c 100644
> --- a/qemu-img.c
> +++ b/qemu-img.c
> @@ -166,7 +166,14 @@ static void QEMU_NORETURN help(void)
>             "Parameters to compare subcommand:\n"
>             "  '-f' first image format\n"
>             "  '-F' second image format\n"
> -           "  '-s' run in Strict mode - fail on different image size or 
> sector allocation\n";
> +           "  '-s' run in Strict mode - fail on different image size or 
> sector allocation\n"
> +           "\n"
> +           "Parameters to dd subcommand:\n"
> +           "  'bs=BYTES' read and write up to BYTES bytes at a time "
> +           "(default: 512)\n"
> +           "  'count=N' copy only N input blocks\n"
> +           "  'if=FILE' read from FILE\n"
> +           "  'of=FILE' write to FILE\n";
>  
>      printf("%s\nSupported formats:", help_msg);
>      bdrv_iterate_format(format_print, NULL);
> @@ -3794,6 +3801,363 @@ out:
>      return 0;
>  }
>  
> +#define C_BS      01
> +#define C_COUNT   02
> +#define C_IF      04
> +#define C_OF      010
> +
> +struct DdInfo {
> +    unsigned int flags;
> +    size_t count;
> +};
> +
> +struct DdIo {
> +    size_t bsz;    /* Block size */
> +    off_t offset;
> +    const char *filename;
> +    uint8_t *buf;
> +};
> +
> +struct DdOpts {
> +    const char *name;
> +    int (*f)(const char *, struct DdIo *, struct DdIo *, struct DdInfo *);
> +    unsigned int flag;
> +};
> +
> +/*
> + * get_size() was needed for the size syntax dd(1) supports which is
> + * different from qemu_strtosz_suffix()
> + *
> + */
> +

Blank line is superfluous between a function and its document comment.

> +static size_t get_size(const char *str)
> +{
> +    /* XXX: handle {k,m,g}B notations */
> +    unsigned long num;
> +    size_t res = 0;
> +    const char *buf;
> +    int ret;
> +
> +    errno = 0;
> +    if (strchr(str, '-')) {
> +        error_report("invalid number: '%s'", str);
> +        errno = EINVAL;
> +        return res;
> +    }
> +    ret = qemu_strtoul(str, &buf, 0, &num);
> +
> +    if (ret < 0) {
> +        error_report("invalid number: '%s'", str);
> +        return res;
> +    }
> +
> +    switch (*buf) {
> +    case '\0':
> +    case 'c':
> +        res = num;
> +        break;
> +    case 'w':
> +        res = num * 2;
> +        break;
> +    case 'b':
> +        res = num * 512;
> +        break;
> +    case 'K':
> +        res = num * 1024;
> +        break;
> +    case 'M':
> +        res = num * 1024 * 1024;
> +        break;
> +    case 'G':
> +        res = num * 1024 * 1024 * 1024;
> +        break;
> +    case 'T':
> +        res = num * 1024 * 1024 * 1024 * 1024;
> +        break;
> +    case 'P':
> +        res = num * 1024 * 1024 * 1024 * 1024 * 1024;
> +        break;
> +    case 'E':
> +        res = num * 1024 * 1024 * 1024 * 1024 * 1024 * 1024;
> +        break;
> +    case 'Z':
> +        res = num * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024;
> +        break;
> +    case 'Y':
> +        res = num * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024;
> +        break;
> +    default:
> +        error_report("invalid number: '%s'", str);
> +        errno = EINVAL;
> +    }
> +
> +    return res;
> +}
> +
> +static int img_dd_bs(const char *arg,
> +                     struct DdIo *in, struct DdIo *out,
> +                     struct DdInfo *dd)
> +{
> +    in->bsz = out->bsz = get_size(arg);
> +
> +    if (in->bsz == 0 && (errno == EINVAL || errno == ERANGE)) {
> +        return 1;
> +    }
> +    if (in->bsz == 0) {
> +        error_report("invalid number: '%s'", arg);
> +        return 1;
> +    }
> +
> +    return 0;
> +}
> +
> +static int img_dd_count(const char *arg,
> +                        struct DdIo *in, struct DdIo *out,
> +                        struct DdInfo *dd)
> +{
> +    dd->count = get_size(arg);
> +
> +    if (dd->count == 0 && (errno == EINVAL || errno == ERANGE)) {
> +        return 1;
> +    }
> +
> +    return 0;
> +}
> +
> +static int img_dd_if(const char *arg,
> +                     struct DdIo *in, struct DdIo *out,
> +                     struct DdInfo *dd)
> +{
> +    in->filename = g_strdup(arg);
> +
> +    return 0;
> +}
> +
> +static int img_dd_of(const char *arg,
> +                     struct DdIo *in, struct DdIo *out,
> +                     struct DdInfo *dd)
> +{
> +    out->filename = g_strdup(arg);
> +
> +    return 0;
> +}
> +
> +static int img_dd(int argc, char **argv)
> +{
> +    int ret = 0;
> +    char *arg = NULL;
> +    char *tmp;
> +    BlockDriver *drv = NULL, *proto_drv = NULL;
> +    BlockBackend *blk1 = NULL, *blk2 = NULL;
> +    QemuOpts *opts = NULL;
> +    QemuOptsList *create_opts = NULL;
> +    Error *local_err = NULL;
> +    bool image_opts = false;
> +    int c;
> +    const char *out_fmt = "raw";
> +    const char *fmt = NULL;
> +    int64_t size = 0;
> +    int64_t block_count = 0, incount = 0, outcount = 0;
> +    struct DdInfo dd = {
> +        .flags = 0,
> +        .count = 0,
> +    };
> +    struct DdIo in = {
> +        .bsz = 512, /* Block size is by default 512 bytes */
> +        .offset = 0,
> +        .filename = NULL,
> +        .buf = NULL
> +    };
> +    struct DdIo out = {
> +        .bsz = 512,
> +        .offset = 0,
> +        .filename = NULL,
> +        .buf = NULL
> +    };
> +
> +    const struct DdOpts options[] = {
> +        { "bs", img_dd_bs, C_BS },
> +        { "count", img_dd_count, C_COUNT },
> +        { "if", img_dd_if, C_IF },
> +        { "of", img_dd_of, C_OF },
> +        { NULL, NULL, 0 }
> +    };
> +    const struct option long_options[] = {
> +        { "help", no_argument, 0, 'h'},
> +        { "image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
> +        { 0, 0, 0, 0 }
> +    };
> +
> +    while ((c = getopt_long(argc, argv, "hf:O:", long_options, NULL))) {
> +        if (c == EOF) {
> +            break;
> +        }
> +        switch (c) {
> +        case 'O':
> +            out_fmt = optarg;
> +            break;
> +        case 'f':
> +            fmt = optarg;
> +            break;
> +        case '?':
> +            error_report("Try 'qemu-img --help' for more information.");
> +            ret = -1;
> +            goto out;
> +            break;
> +        case 'h':
> +            help();
> +            break;
> +        case OPTION_IMAGE_OPTS:
> +            image_opts = true;
> +            break;
> +        }
> +    }
> +
> +    for (int i = optind; i < argc; i++) {
> +        int j;
> +        arg = g_strdup(argv[i]);
> +
> +        tmp = strchr(arg, '=');
> +        if (tmp == NULL) {
> +            error_report("unrecognized operand %s", arg);
> +            ret = -1;
> +            goto out;
> +        }
> +
> +        *tmp++ = '\0';
> +
> +        for (j = 0; options[j].name != NULL; j++) {
> +            if (!strcmp(arg, options[j].name)) {
> +                break;
> +            }
> +        }
> +        if (options[j].name == NULL) {
> +            error_report("unrecognized operand %s", arg);
> +            ret = -1;
> +            goto out;
> +        }
> +
> +        if (options[j].f(tmp, &in, &out, &dd) != 0) {
> +            ret = -1;
> +            goto out;
> +        }
> +        dd.flags |= options[j].flag;
> +        g_free(arg);
> +        arg = NULL;
> +    }
> +
> +    if (dd.flags & C_IF && dd.flags & C_OF) {
> +        blk1 = img_open(image_opts, in.filename, fmt, 0, false, true);
> +
> +        if (!blk1) {
> +            ret = -1;
> +            goto out;
> +        }
> +
> +        drv = bdrv_find_format(out_fmt);
> +        if (!drv) {
> +            error_report("Unknown file format");
> +            ret = -1;
> +            goto out;
> +        }
> +        proto_drv = bdrv_find_protocol(out.filename, true, &local_err);
> +
> +        if (!proto_drv) {
> +            error_report_err(local_err);
> +            ret = -1;
> +            goto out;
> +        }
> +        if (!drv->create_opts) {
> +            error_report("Format driver '%s' does not support image 
> creation",
> +                         drv->format_name);
> +            ret = -1;
> +            goto out;
> +        }
> +        if (!proto_drv->create_opts) {
> +            error_report("Protocol driver '%s' does not support image 
> creation",
> +                       proto_drv->format_name);
> +            ret = -1;
> +            goto out;
> +        }
> +        create_opts = qemu_opts_append(create_opts, drv->create_opts);
> +        create_opts = qemu_opts_append(create_opts, proto_drv->create_opts);
> +
> +        opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
> +
> +        size =  blk_getlength(blk1);
> +
> +        if (dd.flags & C_COUNT && dd.count * in.bsz < size) {
> +            size = dd.count * in.bsz;
> +        }
> +
> +        qemu_opt_set_number(opts, BLOCK_OPT_SIZE, size, &error_abort);
> +
> +        ret = bdrv_create(drv, out.filename, opts, &local_err);
> +        if (ret < 0) {
> +            error_reportf_err(local_err,
> +                              "%s: error while copying and converting: ",

Shouldn't this be "error while creating output image"?

> +                              out.filename);
> +            ret = -1;
> +            goto out;
> +        }
> +
> +        blk2 = img_open(image_opts, out.filename, out_fmt, BDRV_O_RDWR,
> +                        false, true);
> +
> +        if (!blk2) {
> +            ret = -1;
> +            goto out;
> +        }
> +
> +        in.buf = g_new(uint8_t, in.bsz);
> +
> +        for (; incount < size; incount += ret, block_count++) {
> +            int out_ret;
> +
> +            if (in.bsz + incount > size) {
> +                ret = blk_pread(blk1, incount, in.buf, size - incount);
> +            } else {
> +                ret = blk_pread(blk1, incount, in.buf, in.bsz);
> +            }
> +            if (ret < 0) {
> +                error_report("error while reading from input image file: %s",
> +                             strerror(-ret));
> +                ret = -1;
> +                goto out;
> +            }
> +            out_ret = blk_pwrite(blk2, outcount, in.buf, ret, 0);
> +
> +            if (out_ret < 0) {
> +                error_report("error while writing to output image file: %s",
> +                             strerror(-out_ret));
> +                ret = -1;
> +                goto out;
> +            }
> +            outcount += out_ret;
> +        }
> +    }
> +    if (!(dd.flags & C_IF && dd.flags & C_OF)) {
> +        error_report("Must specify both input and output files");
> +        ret = -1;
> +        goto out;
> +    }

If you move this if stanza above the "if (dd.flags & C_IF && dd.flags & C_OF)"
line, the later will not be necessary any more (I think that is what Stefan
meant) in v1.

> +out:
> +    g_free(arg);
> +    qemu_opts_del(opts);
> +    qemu_opts_free(create_opts);
> +    blk_unref(blk1);
> +    blk_unref(blk2);
> +    g_free((char *) in.filename);
> +    g_free((char *) out.filename);

No need to cast to (char *) here.

> +    g_free(in.buf);
> +    g_free(out.buf);
> +
> +    if (ret) {
> +        return 1;
> +    }
> +    return 0;
> +}
> +
>  
>  static const img_cmd_t img_cmds[] = {
>  #define DEF(option, callback, arg_string)        \
> diff --git a/tests/qemu-iotests/158 b/tests/qemu-iotests/158
> new file mode 100755
> index 0000000..a56661b
> --- /dev/null
> +++ b/tests/qemu-iotests/158
> @@ -0,0 +1,54 @@
> +#! /bin/bash
> +#
> +# qemu-img dd test

A copyright header should go here, please refer to other test cases. Each
source file should have its own copyright and license header.

> +
> address@hidden
> +
> +seq="$(basename $0)"
> +echo "QA output created by $seq"
> +
> +here="$PWD"
> +status=1
> +
> +_cleanup()
> +{
> +    _cleanup_test_img
> +    rm -f "$TEST_IMG.out"
> +}
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +. ./common.rc
> +. ./common.filter
> +. ./common.pattern
> +
> +_supported_fmt generic
> +_supported_proto file
> +_supported_os Linux
> +
> +echo
> +echo "== Creating image =="
> +
> +size=1M
> +_make_test_img $size
> +_check_test_img
> +
> +$QEMU_IO -c "write -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
> +
> +echo
> +echo "== Converting the image with dd =="
> +
> +$QEMU_IMG dd if="$TEST_IMG" of="$TEST_IMG.out" -O "$IMGFMT"
> +$QEMU_IMG check "$TEST_IMG.out" -f "$IMGFMT" 2>&1  | _filter_testdir | \
> +    sed -e '/allocated.*fragmented.*compressed clusters/d' \
> +        -e 's/qemu-img: This image format does not support checks/No errors 
> were found on the image./' \
> +        -e '/Image end offset: [0-9]\+/d'

I think it's better to either introduce a _filter_qemu_img_check in
tests/qemu-iotests/common.filter that does these output replacements, or simply
test the exit code and redirect all output to /dev/null.

As you can see below, these lines are already repeated in two scripts in this
patch.

> +
> +echo
> +echo "== Compare the images with qemu-img compare =="
> +
> +$QEMU_IMG compare "$TEST_IMG" "$TEST_IMG.out"
> +
> +echo
> +echo "*** done"
> +rm -f "$seq.full"
> +status=0
> diff --git a/tests/qemu-iotests/158.out b/tests/qemu-iotests/158.out
> new file mode 100644
> index 0000000..58bfd85
> --- /dev/null
> +++ b/tests/qemu-iotests/158.out
> @@ -0,0 +1,15 @@
> +QA output created by 158
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +*** done
> diff --git a/tests/qemu-iotests/159 b/tests/qemu-iotests/159
> new file mode 100755
> index 0000000..d56ea69
> --- /dev/null
> +++ b/tests/qemu-iotests/159
> @@ -0,0 +1,57 @@
> +#! /bin/bash
> +#
> +# qemu-img dd test with different block sizes

A copyright header is needed here too.

> +
> address@hidden
> +
> +seq="$(basename $0)"
> +echo "QA output created by $seq"
> +
> +here="$PWD"
> +status=1
> +
> +_cleanup()
> +{
> +    _cleanup_test_img
> +    rm -f "$TEST_IMG.out"
> +}
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +. ./common.rc
> +. ./common.filter
> +. ./common.pattern
> +
> +_supported_fmt generic
> +_supported_proto file
> +_supported_os Linux
> +
> +TEST_SIZES="5 512 1024 1999 1K 64K 1M"
> +
> +for bs in $TEST_SIZES; do
> +    echo
> +    echo "== Creating image =="
> +
> +    size=1M
> +    _make_test_img $size
> +    _check_test_img
> +    $QEMU_IO -c "write -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
> +
> +    echo
> +    echo "== Converting the image with dd with a block size of $bs =="
> +
> +    $QEMU_IMG dd if="$TEST_IMG" of="$TEST_IMG.out" bs=$bs -O "$IMGFMT"
> +    $QEMU_IMG check "$TEST_IMG.out" -f "$IMGFMT" 2>&1  | _filter_testdir | \
> +     sed -e '/allocated.*fragmented.*compressed clusters/d' \

A tabspace sneaking in? Use whitespace, please. But as said above, the sed can
go away completely.

> +            -e 's/qemu-img: This image format does not support checks/No 
> errors were found on the image./' \
> +            -e '/Image end offset: [0-9]\+/d'
> +
> +    echo
> +    echo "== Compare the images with qemu-img compare =="
> +
> +    $QEMU_IMG compare "$TEST_IMG" "$TEST_IMG.out"
> +done
> +
> +echo
> +echo "*** done"
> +rm -f "$seq.full"
> +status=0
> diff --git a/tests/qemu-iotests/159.out b/tests/qemu-iotests/159.out
> new file mode 100644
> index 0000000..b86b63a
> --- /dev/null
> +++ b/tests/qemu-iotests/159.out
> @@ -0,0 +1,87 @@
> +QA output created by 159
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 5 ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 512 ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 1024 ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 1999 ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 1K ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 64K ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +== Creating image ==
> +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
> +No errors were found on the image.
> +wrote 1048576/1048576 bytes at offset 0
> +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +
> +== Converting the image with dd with a block size of 1M ==
> +No errors were found on the image.
> +
> +== Compare the images with qemu-img compare ==
> +Images are identical.
> +
> +*** done
> diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
> index 3a3973e..ec712bc 100644
> --- a/tests/qemu-iotests/group
> +++ b/tests/qemu-iotests/group
> @@ -157,3 +157,5 @@
>  155 rw auto
>  156 rw auto quick
>  157 auto
> +158 rw auto quick
> +159 rw auto quick
> -- 
> 2.9.0
> 

Fam