[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
bug#6131: [PATCH]: fiemap support for efficient sparse file copy
|
From: |
Jim Meyering |
|
Subject: |
bug#6131: [PATCH]: fiemap support for efficient sparse file copy |
|
Date: |
Tue, 08 Jun 2010 13:33:47 +0200 |
Jim Meyering wrote:
> Jim Meyering wrote:
> ...
>>>> Do you know of a tool other than filefrag that I could use?
>>> nope.
>>>>
>>>> It looks like a small script could filter filefrag -v output, detect
>>>> split extents and rewrite to make the output match what's expected.
>>>> Probably not worth it, though, since this is already a very fragile test.
>>
>> I went ahead and did it, after all.
>> Here's the script, filefrag-extent-compare.
>> With it, this test should pass when run on any of those four
>> file system types.
>
> Not quite.
> Parsing filefrag -v output is part of what is fragile.
>
> Here are two examples:
>
> ==> ff1 <==
> Filesystem type is: ef53
> File size of j1 is 49152 (12 blocks, blocksize 4096)
> ext logical physical expected length flags
> 0 3 10256380 3
> 1 9 10740733 10256382 3 eof
>
> ==> ff2 <==
> Filesystem type is: ef53
> File size of j2 is 49152 (12 blocks, blocksize 4096)
> ext logical physical expected length flags
> 0 3 10283520 3
> 1 9 10283523 3 eof
> ^^^^^^
> Note the missing "expected" number.
> That doesn't happen often, but it's enough to cause occasional
> false-positive failures, since the awk filter counts fields.
> I don't dare try to count columns, because those physical block
> numbers are likely to have width greater than 8 some of the time.
>
> Instead, I adjusted the filter to remove the "eof"
> and let the existing awk code handle the rest:
>
> # Extract logical block number and length pairs from filefrag -v output.
> # The initial sed is to remove the "eof" from the normally-empty "flags"
> field.
> # That is required when that final extent has no number in the "expected"
> field.
> f()
> {
> sed 's/ eof$//' $@ \
> | awk '/^ *[0-9]/ {printf "%d %d ", $2 ,NF < 5 ? $NF : $5 } END {print
> ""}'
> }
>
> I'll post a new patch soon.
Slowly but surely...
I'll squash the two most recent changes (at the end, below)
into yours, Jeff.
>From 9f7a9882944455bfc2b6aec9c9f5431ac429b88b Mon Sep 17 00:00:00 2001
From: Jie Liu <address@hidden>
Date: Thu, 13 May 2010 22:09:30 +0800
Subject: [PATCH 01/10] cp: Add FIEMAP support for efficient sparse file copy
* src/fiemap.h: Add fiemap.h for fiemap ioctl(2) support.
Copied from linux's include/linux/fiemap.h, with minor formatting changes.
* src/copy.c (copy_reg): Now, when `cp' invoked with --sparse=[WHEN] option, we
will try to do FIEMAP-copy if the underlaying file system support it, fall back
to a normal copy if it fails.
---
src/copy.c | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/fiemap.h | 102 +++++++++++++++++++++++++++++++++++++
2 files changed, 261 insertions(+), 0 deletions(-)
create mode 100644 src/fiemap.h
diff --git a/src/copy.c b/src/copy.c
index 171499c..2c15ca0 100644
--- a/src/copy.c
+++ b/src/copy.c
@@ -63,6 +63,10 @@
#include <sys/ioctl.h>
+#ifndef HAVE_FIEMAP
+# include "fiemap.h"
+#endif
+
#ifndef HAVE_FCHOWN
# define HAVE_FCHOWN false
# define fchown(fd, uid, gid) (-1)
@@ -149,6 +153,141 @@ clone_file (int dest_fd, int src_fd)
#endif
}
+#ifdef __linux__
+# ifndef FS_IOC_FIEMAP
+# define FS_IOC_FIEMAP _IOWR ('f', 11, struct fiemap)
+# endif
+/* Perform FIEMAP(available in mainline 2.6.27) copy if possible.
+ Call ioctl(2) with FS_IOC_FIEMAP to efficiently map file allocation
+ excepts holes. So the overhead to deal with holes with lseek(2) in
+ normal copy could be saved. This would result in much faster backups
+ for any kind of sparse file. */
+static bool
+fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
+ off_t src_total_size, char const *src_name,
+ char const *dst_name, bool *normal_copy_required)
+{
+ bool fail = false;
+ bool last = false;
+ char fiemap_buf[4096];
+ struct fiemap *fiemap = (struct fiemap *)fiemap_buf;
+ struct fiemap_extent *fm_ext = &fiemap->fm_extents[0];
+ uint32_t count = (sizeof (fiemap_buf) - sizeof (*fiemap)) /
+ sizeof (struct fiemap_extent);
+ off_t last_ext_logical = 0;
+ uint64_t last_ext_len = 0;
+ uint64_t last_read_size = 0;
+ unsigned int i = 0;
+
+ /* This is required at least to initialize fiemap->fm_start,
+ but also serves (in May 2010) to appease valgrind, which
+ appears not to know the semantics of the FIEMAP ioctl. */
+ memset (fiemap_buf, 0, sizeof fiemap_buf);
+
+ do
+ {
+ fiemap->fm_length = FIEMAP_MAX_OFFSET;
+ fiemap->fm_extent_count = count;
+
+ /* When ioctl(2) fails, fall back to the normal copy only if it
+ is the first time we met. */
+ if (ioctl (src_fd, FS_IOC_FIEMAP, fiemap) < 0)
+ {
+ /* If `i > 0', then at least one ioctl(2) has been performed before.
*/
+ if (i == 0)
+ *normal_copy_required = true;
+ return false;
+ }
+
+ /* If 0 extents are returned, then more ioctls are not needed. */
+ if (fiemap->fm_mapped_extents == 0)
+ break;
+
+ for (i = 0; i < fiemap->fm_mapped_extents; i++)
+ {
+ assert (fm_ext[i].fe_logical <= OFF_T_MAX);
+
+ off_t ext_logical = fm_ext[i].fe_logical;
+ uint64_t ext_len = fm_ext[i].fe_length;
+
+ if (lseek (src_fd, ext_logical, SEEK_SET) < 0LL)
+ {
+ error (0, errno, _("cannot lseek %s"), quote (src_name));
+ return fail;
+ }
+
+ if (lseek (dest_fd, ext_logical, SEEK_SET) < 0LL)
+ {
+ error (0, errno, _("cannot lseek %s"), quote (dst_name));
+ return fail;
+ }
+
+ if (fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST)
+ {
+ last_ext_logical = ext_logical;
+ last_ext_len = ext_len;
+ last = true;
+ }
+
+ while (0 < ext_len)
+ {
+ char buf[buf_size];
+
+ /* Avoid reading into the holes if the left extent
+ length is shorter than the buffer size. */
+ if (ext_len < buf_size)
+ buf_size = ext_len;
+
+ ssize_t n_read = read (src_fd, buf, buf_size);
+ if (n_read < 0)
+ {
+#ifdef EINTR
+ if (errno == EINTR)
+ continue;
+#endif
+ error (0, errno, _("reading %s"), quote (src_name));
+ return fail;
+ }
+
+ if (n_read == 0)
+ {
+ /* Figure out how many bytes read from the last extent. */
+ last_read_size = last_ext_len - ext_len;
+ break;
+ }
+
+ if (full_write (dest_fd, buf, n_read) != n_read)
+ {
+ error (0, errno, _("writing %s"), quote (dst_name));
+ return fail;
+ }
+
+ ext_len -= n_read;
+ }
+ }
+
+ fiemap->fm_start = fm_ext[i - 1].fe_logical + fm_ext[i - 1].fe_length;
+
+ } while (! last);
+
+ /* If a file ends up with holes, the sum of the last extent logical offset
+ and the read-returned size will be shorter than the actual size of the
+ file. Use ftruncate to extend the length of the destination file. */
+ if (last_ext_logical + last_read_size < src_total_size)
+ {
+ if (ftruncate (dest_fd, src_total_size) < 0)
+ {
+ error (0, errno, _("extending %s"), quote (dst_name));
+ return fail;
+ }
+ }
+
+ return ! fail;
+}
+#else
+static bool fiemap_copy_ok (ignored) { errno == ENOTSUP; return false; }
+#endif
+
/* FIXME: describe */
/* FIXME: rewrite this to use a hash table so we avoid the quadratic
performance hit that's probably noticeable only on trees deeper
@@ -679,6 +818,25 @@ copy_reg (char const *src_name, char const *dst_name,
#endif
}
+ if (make_holes)
+ {
+ bool require_normal_copy = false;
+ /* Perform efficient FIEMAP copy for sparse files, fall back to the
+ standard copy only if the ioctl(2) fails. */
+ if (fiemap_copy_ok (source_desc, dest_desc, buf_size,
+ src_open_sb.st_size, src_name,
+ dst_name, &require_normal_copy))
+ goto preserve_metadata;
+ else
+ {
+ if (! require_normal_copy)
+ {
+ return_val = false;
+ goto close_src_and_dst_desc;
+ }
+ }
+ }
+
/* If not making a sparse file, try to use a more-efficient
buffer size. */
if (! make_holes)
@@ -807,6 +965,7 @@ copy_reg (char const *src_name, char const *dst_name,
}
}
+preserve_metadata:
if (x->preserve_timestamps)
{
struct timespec timespec[2];
diff --git a/src/fiemap.h b/src/fiemap.h
new file mode 100644
index 0000000..d33293b
--- /dev/null
+++ b/src/fiemap.h
@@ -0,0 +1,102 @@
+/* FS_IOC_FIEMAP ioctl infrastructure.
+ Some portions copyright (C) 2007 Cluster File Systems, Inc
+ Authors: Mark Fasheh <address@hidden>
+ Kalpak Shah <address@hidden>
+ Andreas Dilger <address@hidden>. */
+
+/* Copy from kernel, modified to respect GNU code style by Jie Liu. */
+
+#ifndef _LINUX_FIEMAP_H
+# define _LINUX_FIEMAP_H
+
+# include <linux/types.h>
+
+struct fiemap_extent
+{
+ /* Logical offset in bytes for the start of the extent
+ from the beginning of the file. */
+ uint64_t fe_logical;
+
+ /* Physical offset in bytes for the start of the extent
+ from the beginning of the disk. */
+ uint64_t fe_physical;
+
+ /* Length in bytes for this extent. */
+ uint64_t fe_length;
+
+ uint64_t fe_reserved64[2];
+
+ /* FIEMAP_EXTENT_* flags for this extent. */
+ uint32_t fe_flags;
+
+ uint32_t fe_reserved[3];
+};
+
+struct fiemap
+{
+ /* Logical offset(inclusive) at which to start mapping(in). */
+ uint64_t fm_start;
+
+ /* Logical length of mapping which userspace wants(in). */
+ uint64_t fm_length;
+
+ /* FIEMAP_FLAG_* flags for request(in/out). */
+ uint32_t fm_flags;
+
+ /* Number of extents that were mapped(out). */
+ uint32_t fm_mapped_extents;
+
+ /* Size of fm_extents array(in). */
+ uint32_t fm_extent_count;
+
+ uint32_t fm_reserved;
+
+ /* Array of mapped extents(out). */
+ struct fiemap_extent fm_extents[0];
+};
+
+/* The maximum offset can be mapped for a file. */
+# define FIEMAP_MAX_OFFSET (~0ULL)
+
+/* Sync file data before map. */
+# define FIEMAP_FLAG_SYNC 0x00000001
+
+/* Map extented attribute tree. */
+# define FIEMAP_FLAG_XATTR 0x00000002
+
+# define FIEMAP_FLAGS_COMPAT (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR)
+
+/* Last extent in file. */
+# define FIEMAP_EXTENT_LAST 0x00000001
+
+/* Data location unknown. */
+# define FIEMAP_EXTENT_UNKNOWN 0x00000002
+
+/* Location still pending, Sets EXTENT_UNKNOWN. */
+# define FIEMAP_EXTENT_DELALLOC 0x00000004
+
+/* Data can not be read while fs is unmounted. */
+# define FIEMAP_EXTENT_ENCODED 0x00000008
+
+/* Data is encrypted by fs. Sets EXTENT_NO_BYPASS. */
+# define FIEMAP_EXTENT_DATA_ENCRYPTED 0x00000080
+
+/* Extent offsets may not be block aligned. */
+# define FIEMAP_EXTENT_NOT_ALIGNED 0x00000100
+
+/* Data mixed with metadata. Sets EXTENT_NOT_ALIGNED. */
+# define FIEMAP_EXTENT_DATA_INLINE 0x00000200
+
+/* Multiple files in block. Set EXTENT_NOT_ALIGNED. */
+# define FIEMAP_EXTENT_DATA_TAIL 0x00000400
+
+/* Space allocated, but not data (i.e. zero). */
+# define FIEMAP_EXTENT_UNWRITTEN 0x00000800
+
+/* File does not natively support extents. Result merged for efficiency. */
+# define FIEMAP_EXTENT_MERGED 0x00001000
+
+/* Space shared with other files. */
+# define FIEMAP_EXTENT_SHARED 0x00002000
+
+#endif
--
1.7.1.501.g23b46
>From dc98ac611dbb141275613fe30f5a8d2486817a3b Mon Sep 17 00:00:00 2001
From: Jie Liu <address@hidden>
Date: Thu, 13 May 2010 22:17:53 +0800
Subject: [PATCH 02/10] tests: add a new test for FIEMAP-copy
* tests/cp/sparse-fiemap: Add a new test for FIEMAP-copy against a
loopbacked ext4 partition.
* tests/Makefile.am (sparse-fiemap): Reference the new test.
---
tests/Makefile.am | 1 +
tests/cp/sparse-fiemap | 56 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 57 insertions(+), 0 deletions(-)
create mode 100755 tests/cp/sparse-fiemap
diff --git a/tests/Makefile.am b/tests/Makefile.am
index c458574..f7840c8 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -25,6 +25,7 @@ root_tests = \
cp/special-bits \
cp/cp-mv-enotsup-xattr \
cp/capability \
+ cp/sparse-fiemap \
dd/skip-seek-past-dev \
install/install-C-root \
ls/capability \
diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap
new file mode 100755
index 0000000..945c94b
--- /dev/null
+++ b/tests/cp/sparse-fiemap
@@ -0,0 +1,56 @@
+#!/bin/sh
+# Test cp --sparse=always through fiemap copy
+
+# Copyright (C) 2006-2010 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+if test "$VERBOSE" = yes; then
+ set -x
+ cp --version
+fi
+
+. $srcdir/test-lib.sh
+require_root_
+
+cwd=`pwd`
+cleanup_() { cd /; umount "$cwd/mnt"; }
+
+skip=0
+# Create an ext4 loopback file system
+dd if=/dev/zero of=blob bs=8192 count=1000 || skip=1
+mkdir mnt
+mkfs -t ext4 -F blob ||
+ skip_test_ "failed to create ext4 file system"
+mount -oloop blob mnt || skip=1
+echo test > mnt/f || skip=1
+test -s mnt/f || skip=1
+
+test $skip = 1 &&
+ skip_test_ "insufficient mount/ext4 support"
+
+# Create a 1TiB sparse file
+dd if=/dev/zero of=mnt/sparse bs=1k count=1 seek=1G || framework_failure
+
+cd mnt || fail=1
+
+# It takes many minutes to copy this sparse file using the old method.
+# By contrast, it takes far less than 1 second using FIEMAP-copy.
+timeout 10 cp --sparse=always sparse fiemap || fail=1
+
+# Ensure that the sparse file copied through fiemap has the same size
+# in bytes as the original.
+test $(stat --printf %s sparse) = $(stat --printf %s fiemap) || fail=1
+
+Exit $fail
--
1.7.1.501.g23b46
>From af8047bc22c93e72c38cfd9cd7e1fdd455ce81e0 Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Fri, 28 May 2010 09:24:15 +0200
Subject: [PATCH 03/10] tests: sparse-fiemap: factor out some set-up
* tests/cp/sparse-fiemap: Cd into test directory sooner.
---
tests/cp/sparse-fiemap | 8 ++++----
1 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap
index 945c94b..21b02ac 100755
--- a/tests/cp/sparse-fiemap
+++ b/tests/cp/sparse-fiemap
@@ -33,9 +33,10 @@ dd if=/dev/zero of=blob bs=8192 count=1000 || skip=1
mkdir mnt
mkfs -t ext4 -F blob ||
skip_test_ "failed to create ext4 file system"
-mount -oloop blob mnt || skip=1
-echo test > mnt/f || skip=1
-test -s mnt/f || skip=1
+mount -oloop blob mnt || skip=1
+cd mnt || skip=1
+echo test > f || skip=1
+test -s f || skip=1
test $skip = 1 &&
skip_test_ "insufficient mount/ext4 support"
@@ -43,7 +44,6 @@ test $skip = 1 &&
# Create a 1TiB sparse file
dd if=/dev/zero of=mnt/sparse bs=1k count=1 seek=1G || framework_failure
-cd mnt || fail=1
# It takes many minutes to copy this sparse file using the old method.
# By contrast, it takes far less than 1 second using FIEMAP-copy.
--
1.7.1.501.g23b46
>From 0a6df6c17b2043515f7719491b48eb15d2812456 Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Fri, 21 May 2010 18:28:42 +0200
Subject: [PATCH 04/10] tests: exercise more of the new FIEMAP copying code
* tests/cp/sparse-fiemap: Ensure that a file with many extents (more
than fit in copy.c's internal 4KiB buffer) is copied properly.
---
tests/cp/sparse-fiemap | 38 ++++++++++++++++++++++++++++++++++++++
1 files changed, 38 insertions(+), 0 deletions(-)
diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap
index 21b02ac..3608db3 100755
--- a/tests/cp/sparse-fiemap
+++ b/tests/cp/sparse-fiemap
@@ -53,4 +53,42 @@ timeout 10 cp --sparse=always sparse fiemap || fail=1
# in bytes as the original.
test $(stat --printf %s sparse) = $(stat --printf %s fiemap) || fail=1
+# =================================================
+# Ensure that we exercise the FIEMAP-copying code enough
+# to provoke at least two iterations of the do...while loop
+# in which it calls ioctl (fd, FS_IOC_FIEMAP,...
+# This also verifies that non-trivial extents are preserved.
+
+$PERL -e 1 || skip_test_ 'skipping part of this test; you lack perl'
+
+$PERL -e 'BEGIN { $n = 16 * 1024; *F = *STDOUT }' \
+ -e 'for (1..100) { sysseek (*F, $n, 1)' \
+ -e '&& syswrite (*F, "."x$n) or die "$!"}' > j1 || fail=1
+
+cp --sparse=always j1 j2 || fail=1
+cmp j1 j2 || fail=1
+
+filefrag j1 | grep extent \
+ || skip_test_ 'skipping part of this test; you lack filefrag'
+
+# Here is sample filefrag output:
+# $ perl -e 'BEGIN{$n=16*1024; *F=*STDOUT}' \
+# -e 'for (1..5) { sysseek(*F,$n,1)' \
+# -e '&& syswrite *F,"."x$n or die "$!"}' > j
+# $ filefrag -v j
+# File system type is: ef53
+# File size of j is 163840 (40 blocks, blocksize 4096)
+# ext logical physical expected length flags
+# 0 4 6258884 4
+# 1 12 6258892 6258887 4
+# 2 20 6258900 6258895 4
+# 3 28 6258908 6258903 4
+# 4 36 6258916 6258911 4 eof
+# j: 6 extents found
+
+# exclude the physical block numbers; they always differ
+filefrag -v j1 | awk '/^ / {print $1,$2,$NF}' > ff1 || fail=1
+filefrag -v j2 | awk '/^ / {print $1,$2,$NF}' > ff2 || fail=1
+compare ff1 ff2 || fail=1
+
Exit $fail
--
1.7.1.501.g23b46
>From b257d65c6d923567af5d4b757810dd62e1be91bb Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Sat, 22 May 2010 10:22:58 +0200
Subject: [PATCH 05/10] tests: require root only if current partition is neither
btrfs nor xfs
* tests/cp/sparse-fiemap: Don't require root access if current
partition is btrfs or xfs.
---
tests/cp/sparse-fiemap | 43 ++++++++++++++++++++++++-------------------
1 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap
index 3608db3..17ed4f9 100755
--- a/tests/cp/sparse-fiemap
+++ b/tests/cp/sparse-fiemap
@@ -22,28 +22,33 @@ if test "$VERBOSE" = yes; then
fi
. $srcdir/test-lib.sh
-require_root_
-cwd=`pwd`
-cleanup_() { cd /; umount "$cwd/mnt"; }
-
-skip=0
-# Create an ext4 loopback file system
-dd if=/dev/zero of=blob bs=8192 count=1000 || skip=1
-mkdir mnt
-mkfs -t ext4 -F blob ||
- skip_test_ "failed to create ext4 file system"
-mount -oloop blob mnt || skip=1
-cd mnt || skip=1
-echo test > f || skip=1
-test -s f || skip=1
-
-test $skip = 1 &&
- skip_test_ "insufficient mount/ext4 support"
+if df -T -t btrfs -t xfs . ; then
+ : # Current dir is on a partition with working extents. Good!
+else
+ # It's not; we need to create one, hence we need root access.
+ require_root_
+
+ cwd=$PWD
+ cleanup_() { cd /; umount "$cwd/mnt"; }
+
+ skip=0
+ # Create an XFS loopback file system
+ dd if=/dev/zero of=blob bs=32k count=1000 || skip=1
+ mkdir mnt
+ mkfs -t xfs blob ||
+ skip_test_ "failed to create XFS file system"
+ mount -oloop blob mnt || skip=1
+ cd mnt || skip=1
+ echo test > f || skip=1
+ test -s f || skip=1
+
+ test $skip = 1 &&
+ skip_test_ "insufficient mount/XFS support"
+fi
# Create a 1TiB sparse file
-dd if=/dev/zero of=mnt/sparse bs=1k count=1 seek=1G || framework_failure
-
+dd if=/dev/zero of=sparse bs=1k count=1 seek=1G || framework_failure
# It takes many minutes to copy this sparse file using the old method.
# By contrast, it takes far less than 1 second using FIEMAP-copy.
--
1.7.1.501.g23b46
>From 71b4c7df63795345781bc2012f79695fc6933c7d Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Sat, 22 May 2010 10:21:46 +0200
Subject: [PATCH 06/10] tests: test fiemap-enabled cp more thoroughly
* tests/cp/sparse-fiemap: More tests.
---
tests/cp/sparse-fiemap | 61 +++++++++++++++++++++++++----------------------
1 files changed, 32 insertions(+), 29 deletions(-)
diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap
index 17ed4f9..6d0bd83 100755
--- a/tests/cp/sparse-fiemap
+++ b/tests/cp/sparse-fiemap
@@ -66,34 +66,37 @@ test $(stat --printf %s sparse) = $(stat --printf %s
fiemap) || fail=1
$PERL -e 1 || skip_test_ 'skipping part of this test; you lack perl'
-$PERL -e 'BEGIN { $n = 16 * 1024; *F = *STDOUT }' \
- -e 'for (1..100) { sysseek (*F, $n, 1)' \
- -e '&& syswrite (*F, "."x$n) or die "$!"}' > j1 || fail=1
-
-cp --sparse=always j1 j2 || fail=1
-cmp j1 j2 || fail=1
-
-filefrag j1 | grep extent \
- || skip_test_ 'skipping part of this test; you lack filefrag'
-
-# Here is sample filefrag output:
-# $ perl -e 'BEGIN{$n=16*1024; *F=*STDOUT}' \
-# -e 'for (1..5) { sysseek(*F,$n,1)' \
-# -e '&& syswrite *F,"."x$n or die "$!"}' > j
-# $ filefrag -v j
-# File system type is: ef53
-# File size of j is 163840 (40 blocks, blocksize 4096)
-# ext logical physical expected length flags
-# 0 4 6258884 4
-# 1 12 6258892 6258887 4
-# 2 20 6258900 6258895 4
-# 3 28 6258908 6258903 4
-# 4 36 6258916 6258911 4 eof
-# j: 6 extents found
-
-# exclude the physical block numbers; they always differ
-filefrag -v j1 | awk '/^ / {print $1,$2,$NF}' > ff1 || fail=1
-filefrag -v j2 | awk '/^ / {print $1,$2,$NF}' > ff2 || fail=1
-compare ff1 ff2 || fail=1
+for i in $(seq 20); do
+ for j in 1 2 31 100; do
+ $PERL -e 'BEGIN { $n = '$i' * 1024; *F = *STDOUT }' \
+ -e 'for (1..'$j') { sysseek (*F, $n, 1)' \
+ -e '&& syswrite (*F, "."x$n) or die "$!"}' > j1 || fail=1
+
+ cp --sparse=always j1 j2 || fail=1
+ cmp j1 j2 || fail=1
+ filefrag -v j1 | grep extent \
+ || skip_test_ 'skipping part of this test; you lack filefrag'
+
+ # Here is sample filefrag output:
+ # $ perl -e 'BEGIN{$n=16*1024; *F=*STDOUT}' \
+ # -e 'for (1..5) { sysseek(*F,$n,1)' \
+ # -e '&& syswrite *F,"."x$n or die "$!"}' > j
+ # $ filefrag -v j
+ # File system type is: ef53
+ # File size of j is 163840 (40 blocks, blocksize 4096)
+ # ext logical physical expected length flags
+ # 0 4 6258884 4
+ # 1 12 6258892 6258887 4
+ # 2 20 6258900 6258895 4
+ # 3 28 6258908 6258903 4
+ # 4 36 6258916 6258911 4 eof
+ # j: 6 extents found
+
+ # exclude the physical block numbers; they always differ
+ filefrag -v j1 | awk '/^ / {print $1,$2}' > ff1 || fail=1
+ filefrag -v j2 | awk '/^ / {print $1,$2}' > ff2 || fail=1
+ compare ff1 ff2 || fail=1
+ done
+done
Exit $fail
--
1.7.1.501.g23b46
>From f674f49bfd315143ffb4ab376e30546fd021f719 Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Sat, 29 May 2010 21:22:40 +0200
Subject: [PATCH 07/10] tests: relax the root-tests cross-check
* cfg.mk (sc_root_tests): Allow spaces before "require_root_",
now that tests/cp/sparse-fiemap has a conditional use.
---
cfg.mk | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/cfg.mk b/cfg.mk
index dff5de5..17267cf 100644
--- a/cfg.mk
+++ b/cfg.mk
@@ -80,7 +80,7 @@ sc_root_tests:
@if test -d tests \
&& grep check-root tests/Makefile.am>/dev/null 2>&1; then \
t1=sc-root.expected; t2=sc-root.actual; \
- grep -nl '^require_root_$$' \
+ grep -nl '^ *require_root_$$' \
$$($(VC_LIST) tests) |sed s,tests/,, |sort > $$t1; \
sed -n '/^root_tests =[ ]*\\$$/,/[^\]$$/p' \
$(srcdir)/tests/Makefile.am \
--
1.7.1.501.g23b46
>From 034c56d9a9a408ce336d039dba072c4ec97d7d04 Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Sun, 30 May 2010 21:20:30 +0200
Subject: [PATCH 08/10] tests: improve fiemap test to work with 4 FS types; fall
back on ext4
* tests/cp/sparse-fiemap: Improve.
* tests/filefrag-extent-compare: New file.
---
tests/cp/sparse-fiemap | 38 ++++++++++++++++------
tests/filefrag-extent-compare | 68 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 95 insertions(+), 11 deletions(-)
create mode 100644 tests/filefrag-extent-compare
diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap
index 6d0bd83..3e7c11f 100755
--- a/tests/cp/sparse-fiemap
+++ b/tests/cp/sparse-fiemap
@@ -23,7 +23,7 @@ fi
. $srcdir/test-lib.sh
-if df -T -t btrfs -t xfs . ; then
+if df -T -t btrfs -t xfs -t ext4 -t ocfs2 . ; then
: # Current dir is on a partition with working extents. Good!
else
# It's not; we need to create one, hence we need root access.
@@ -33,18 +33,18 @@ else
cleanup_() { cd /; umount "$cwd/mnt"; }
skip=0
- # Create an XFS loopback file system
+ # Create an ext4 loopback file system
dd if=/dev/zero of=blob bs=32k count=1000 || skip=1
mkdir mnt
- mkfs -t xfs blob ||
- skip_test_ "failed to create XFS file system"
+ mkfs -t ext4 -F blob ||
+ skip_test_ "failed to create ext4 file system"
mount -oloop blob mnt || skip=1
cd mnt || skip=1
echo test > f || skip=1
test -s f || skip=1
test $skip = 1 &&
- skip_test_ "insufficient mount/XFS support"
+ skip_test_ "insufficient mount/ext4 support"
fi
# Create a 1TiB sparse file
@@ -66,13 +66,26 @@ test $(stat --printf %s sparse) = $(stat --printf %s
fiemap) || fail=1
$PERL -e 1 || skip_test_ 'skipping part of this test; you lack perl'
-for i in $(seq 20); do
+# Extract logical block number and length pairs from filefrag -v output.
+# The initial sed is to remove the "eof" from the normally-empty "flags" field.
+# That is required when that final extent has no number in the "expected"
field.
+f()
+{
+ sed 's/ eof$//' $@ \
+ | awk '/^ *[0-9]/ {printf "%d %d ", $2 ,NF < 5 ? $NF : $5 } END {print ""}'
+}
+
+for i in $(seq 1 2 21); do
for j in 1 2 31 100; do
$PERL -e 'BEGIN { $n = '$i' * 1024; *F = *STDOUT }' \
-e 'for (1..'$j') { sysseek (*F, $n, 1)' \
- -e '&& syswrite (*F, "."x$n) or die "$!"}' > j1 || fail=1
-
+ -e '&& syswrite (*F, chr($_)x$n) or die "$!"}' > j1 || fail=1
+ # sync
cp --sparse=always j1 j2 || fail=1
+ # sync
+ # Technically we may need the 'sync' uses above, but
+ # uncommenting them makes this test take much longer.
+
cmp j1 j2 || fail=1
filefrag -v j1 | grep extent \
|| skip_test_ 'skipping part of this test; you lack filefrag'
@@ -93,10 +106,13 @@ for i in $(seq 20); do
# j: 6 extents found
# exclude the physical block numbers; they always differ
- filefrag -v j1 | awk '/^ / {print $1,$2}' > ff1 || fail=1
- filefrag -v j2 | awk '/^ / {print $1,$2}' > ff2 || fail=1
- compare ff1 ff2 || fail=1
+ filefrag -v j1 > ff1 || fail=1
+ filefrag -v j2 > ff2 || fail=1
+ { f ff1; f ff2; } \
+ | $PERL $abs_top_srcdir/tests/filefrag-extent-compare \
+ || { fail=1; break; }
done
+ test $fail = 1 && break
done
Exit $fail
diff --git a/tests/filefrag-extent-compare b/tests/filefrag-extent-compare
new file mode 100644
index 0000000..3c095d5
--- /dev/null
+++ b/tests/filefrag-extent-compare
@@ -0,0 +1,68 @@
+eval '(exit $?0)' && eval 'exec perl -wS "$0" ${1+"$@"}'
+ & eval 'exec perl -wS "$0" $argv:q'
+ if 0;
+# Determine whether two files have the same extents by comparing
+# the logical block numbers and lengths from filefrag -v for each.
+
+# Invoke like this:
+# This helper function, f, extracts logical block number and lengths.
+# f() { awk '/^ *[0-9]/ {printf "%d %d ",$2,NF<5?$NF:$5} END {print ""}'; }
+# { filefrag -v j1 | f; filefrag -v j2 | f; } | ./filefrag-extent-compare
+
+use warnings;
+use strict;
+(my $ME = $0) =~ s|.*/||;
+
+my @line = <>;
+my $n_lines = @line;
+$n_lines == 2
+ or die "$ME: expected exactly two input lines; got $n_lines\n";
+
+my @A = split ' ', $line[0];
+my @B = split ' ', $line[1];
address@hidden % 2 || @B % 2
+ and die "$ME: unexpected input: odd number of numbers; expected even\n";
+
+my @a;
+my @b;
+foreach my $i (address@hidden/2-1) { $a[$i] = { L_BLK => $A[2*$i], LEN =>
$A[2*$i+1] } };
+foreach my $i (address@hidden/2-1) { $b[$i] = { L_BLK => $B[2*$i], LEN =>
$B[2*$i+1] } };
+
+my $i = 0;
+my $j = 0;
+while (1)
+ {
+ !defined $a[$i] && !defined $b[$j]
+ and exit 0;
+ defined $a[$i] && defined $b[$j]
+ or die "address@hidden and address@hidden have different lengths, even
after adjustment\n";
+ ($a[$i]->{L_BLK} == $b[$j]->{L_BLK}
+ && $a[$i]->{LEN} == $b[$j]->{LEN})
+ and next;
+ ($a[$i]->{LEN} < $b[$j]->{LEN}
+ && exists $a[$i+1] && $a[$i]->{LEN} + $a[$i+1]->{LEN} == $b[$j]->{LEN})
+ and ++$i, next;
+ exists $b[$j+1] && $a[$i]->{LEN} == $b[$i]->{LEN} + $b[$i+1]->{LEN}
+ and ++$j, next;
+ die "differing extent:\n"
+ . " [$i]=$a[$i]->{L_BLK} $a[$i]->{LEN}\n"
+ . " [$j]=$b[$j]->{L_BLK} $b[$j]->{LEN}\n"
+ }
+continue
+ {
+ ++$i;
+ ++$j;
+ }
+
+### Setup "GNU" style for perl-mode and cperl-mode.
+## Local Variables:
+## mode: perl
+## perl-indent-level: 2
+## perl-continued-statement-offset: 2
+## perl-continued-brace-offset: 0
+## perl-brace-offset: 0
+## perl-brace-imaginary-offset: 0
+## perl-label-offset: -2
+## perl-extra-newline-before-brace: t
+## perl-merge-trailing-else: nil
+## End:
--
1.7.1.501.g23b46
>From a7a188a4fd4f58404b825f463f3670193232a037 Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Sat, 5 Jun 2010 10:17:48 +0200
Subject: [PATCH 09/10] cleanup
* src/copy.c (fiemap_copy): Rename from fiemap_copy_ok.
Add/improve comments.
Remove local, "fail".
---
src/copy.c | 42 +++++++++++++++++++++---------------------
1 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/src/copy.c b/src/copy.c
index 2c15ca0..6a4631a 100644
--- a/src/copy.c
+++ b/src/copy.c
@@ -157,17 +157,17 @@ clone_file (int dest_fd, int src_fd)
# ifndef FS_IOC_FIEMAP
# define FS_IOC_FIEMAP _IOWR ('f', 11, struct fiemap)
# endif
-/* Perform FIEMAP(available in mainline 2.6.27) copy if possible.
- Call ioctl(2) with FS_IOC_FIEMAP to efficiently map file allocation
- excepts holes. So the overhead to deal with holes with lseek(2) in
- normal copy could be saved. This would result in much faster backups
- for any kind of sparse file. */
+/* Perform a FIEMAP copy, if possible.
+ Call ioctl(2) with FS_IOC_FIEMAP (available in linux 2.6.27) to
+ obtain a map of file extents excluding holes. This avoids the
+ overhead of detecting holes in a hole-introducing/preserving copy,
+ and thus makes copying sparse files much more efficient.
+ Upon a successful copy, return true. Otherwise, return false. */
static bool
-fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
- off_t src_total_size, char const *src_name,
- char const *dst_name, bool *normal_copy_required)
+fiemap_copy (int src_fd, int dest_fd, size_t buf_size,
+ off_t src_total_size, char const *src_name,
+ char const *dst_name, bool *normal_copy_required)
{
- bool fail = false;
bool last = false;
char fiemap_buf[4096];
struct fiemap *fiemap = (struct fiemap *)fiemap_buf;
@@ -180,7 +180,7 @@ fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
unsigned int i = 0;
/* This is required at least to initialize fiemap->fm_start,
- but also serves (in May 2010) to appease valgrind, which
+ but also serves (in mid 2010) to appease valgrind, which
appears not to know the semantics of the FIEMAP ioctl. */
memset (fiemap_buf, 0, sizeof fiemap_buf);
@@ -213,13 +213,13 @@ fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
if (lseek (src_fd, ext_logical, SEEK_SET) < 0LL)
{
error (0, errno, _("cannot lseek %s"), quote (src_name));
- return fail;
+ return false;
}
if (lseek (dest_fd, ext_logical, SEEK_SET) < 0LL)
{
error (0, errno, _("cannot lseek %s"), quote (dst_name));
- return fail;
+ return false;
}
if (fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST)
@@ -246,7 +246,7 @@ fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
continue;
#endif
error (0, errno, _("reading %s"), quote (src_name));
- return fail;
+ return false;
}
if (n_read == 0)
@@ -259,7 +259,7 @@ fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
if (full_write (dest_fd, buf, n_read) != n_read)
{
error (0, errno, _("writing %s"), quote (dst_name));
- return fail;
+ return false;
}
ext_len -= n_read;
@@ -277,15 +277,15 @@ fiemap_copy_ok (int src_fd, int dest_fd, size_t buf_size,
{
if (ftruncate (dest_fd, src_total_size) < 0)
{
- error (0, errno, _("extending %s"), quote (dst_name));
- return fail;
+ error (0, errno, _("failed to extend %s"), quote (dst_name));
+ return false;
}
}
- return ! fail;
+ return true;
}
#else
-static bool fiemap_copy_ok (ignored) { errno == ENOTSUP; return false; }
+static bool fiemap_copy (ignored) { errno == ENOTSUP; return false; }
#endif
/* FIXME: describe */
@@ -823,9 +823,9 @@ copy_reg (char const *src_name, char const *dst_name,
bool require_normal_copy = false;
/* Perform efficient FIEMAP copy for sparse files, fall back to the
standard copy only if the ioctl(2) fails. */
- if (fiemap_copy_ok (source_desc, dest_desc, buf_size,
- src_open_sb.st_size, src_name,
- dst_name, &require_normal_copy))
+ if (fiemap_copy (source_desc, dest_desc, buf_size,
+ src_open_sb.st_size, src_name,
+ dst_name, &require_normal_copy))
goto preserve_metadata;
else
{
--
1.7.1.501.g23b46
>From 7d10de7a2ba4161bc2b910f30ff4841f8c91512d Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Sat, 5 Jun 2010 10:56:36 +0200
Subject: [PATCH 10/10] adjust normal_copy_required semantics
* src/copy.c (fiemap_copy): Do not require caller to set
"normal_copy_required" before calling fiemap_copy.
---
src/copy.c | 18 +++++++++++-------
1 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/src/copy.c b/src/copy.c
index 6a4631a..eb67700 100644
--- a/src/copy.c
+++ b/src/copy.c
@@ -161,8 +161,10 @@ clone_file (int dest_fd, int src_fd)
Call ioctl(2) with FS_IOC_FIEMAP (available in linux 2.6.27) to
obtain a map of file extents excluding holes. This avoids the
overhead of detecting holes in a hole-introducing/preserving copy,
- and thus makes copying sparse files much more efficient.
- Upon a successful copy, return true. Otherwise, return false. */
+ and thus makes copying sparse files much more efficient. Upon a
+ successful copy, return true. If the initial ioctl fails, set
+ *NORMAL_COPY_REQUIRED to true and return false. Upon any other
+ failure, set *NORMAL_COPY_REQUIRED to false and return false. */
static bool
fiemap_copy (int src_fd, int dest_fd, size_t buf_size,
off_t src_total_size, char const *src_name,
@@ -170,14 +172,15 @@ fiemap_copy (int src_fd, int dest_fd, size_t buf_size,
{
bool last = false;
char fiemap_buf[4096];
- struct fiemap *fiemap = (struct fiemap *)fiemap_buf;
+ struct fiemap *fiemap = (struct fiemap *) fiemap_buf;
struct fiemap_extent *fm_ext = &fiemap->fm_extents[0];
- uint32_t count = (sizeof (fiemap_buf) - sizeof (*fiemap)) /
- sizeof (struct fiemap_extent);
+ uint32_t count = ((sizeof fiemap_buf - sizeof (*fiemap))
+ / sizeof (struct fiemap_extent));
off_t last_ext_logical = 0;
uint64_t last_ext_len = 0;
uint64_t last_read_size = 0;
unsigned int i = 0;
+ *normal_copy_required = false;
/* This is required at least to initialize fiemap->fm_start,
but also serves (in mid 2010) to appease valgrind, which
@@ -193,7 +196,8 @@ fiemap_copy (int src_fd, int dest_fd, size_t buf_size,
is the first time we met. */
if (ioctl (src_fd, FS_IOC_FIEMAP, fiemap) < 0)
{
- /* If `i > 0', then at least one ioctl(2) has been performed before.
*/
+ /* If the first ioctl fails, tell the caller that it is
+ ok to proceed with a normal copy. */
if (i == 0)
*normal_copy_required = true;
return false;
@@ -820,7 +824,7 @@ copy_reg (char const *src_name, char const *dst_name,
if (make_holes)
{
- bool require_normal_copy = false;
+ bool require_normal_copy;
/* Perform efficient FIEMAP copy for sparse files, fall back to the
standard copy only if the ioctl(2) fails. */
if (fiemap_copy (source_desc, dest_desc, buf_size,
--
1.7.1.501.g23b46
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Jim Meyering, 2010/06/02
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy,
Jim Meyering <=
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Eric Blake, 2010/06/08
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Jim Meyering, 2010/06/08
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, jeff.liu, 2010/06/09
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Jim Meyering, 2010/06/09
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, jeff.liu, 2010/06/10
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Paul Eggert, 2010/06/10
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Tao Ma, 2010/06/10
- bug#6131: [PATCH]: fiemap support for efficient sparse file copy, Paul Eggert, 2010/06/15