ODP: [Bug-tar] GNU tar, star and BSD tar speed comparision +new script

[Jan Psota]

Latest TCP script at the bottom (3180 bytes).
4 tests: 64bit dual core Athlon tmpfs / disk (reiserfs) - 60MB/s,
        32bit Athlon tmpfs / disk (reiserfs) - 55MB/s
Both machines were idle -- used for testing only.
Tarball and extracted files were on different physical devices.
Test data: linux 2.6.22/3 kernel sources for memory operations,
for the other data average file size should bring enough info.

2 x [...] processor means 1 processor with 2 cores (2 entries in cpuinfo).
Archive format is set to pax (Joerg).
Let's end with it. I only wanted to send You a new version of TCP script :-).

--
Jan Psota

TCP, version 2007-10-25
Linux 2.6.22-suspend2-r2 / Gentoo Base System release 2.0.0_rc5
2012MB of memory, 2 x AMD Athlon(tm) 64 X2 Dual Core Processor 4200+ 2211.348 512 KB 4426.24 bmips
gcc (GCC) 4.2.2 (Gentoo 4.2.2 p1.0)
CFLAGS="-O2 -march=k8 -pipe"

bsdtar: bsdtar 2.3.4 - libarchive 2.3.4
gnutar: tar (GNU tar) 1.19
star:   star: star 1.5a85 (x86_64-unknown-linux-gnu)

best time of 5 repetitions,
        src="" 291M in 23867 files, avg 13KB/file,
        archive=/tmp/tcp.tar, extract to /tmp/tcptmp
program operation       real    user    system  %CPU         speed
bsdtar  create          0.764   0.232   0.532   99.96       370308 KB/s
gnutar  create          0.743   0.200   0.512   95.87       380775 KB/s
star    create          0.587   0.040   0.820   100.00      441247 KB/s

bsdtar  list            0.164   0.096   0.068   99.84      1579341 KB/s
gnutar  list            0.218   0.064   0.152   98.92      1188128 KB/s
star    list            0.359   0.044   0.240   79.09       721481 KB/s

bsdtar  extract         0.733   0.200   0.504   96.02       353358 KB/s
gnutar  extract         0.625   0.092   0.508   96.02       414419 KB/s
star    extract         0.875   0.096   0.980   100.00      296013 KB/s

bsdtar  compare         0.001   0.000   0.000   0.00     259012000 KB/s
gnutar  compare         0.719   0.288   0.400   95.66       360239 KB/s
star    compare         0.695   0.224   0.636   100.00      372679 KB/s

[...]
best time of 3 repetitions,
        src="" 3.2G in 7447 files, avg 554KB/file,
        archive=/var/tcp.tar, extract to /mnt/a/tcptmp
program operation       real    user    system  %CPU         speed
bsdtar  create          184.680 0.552   13.365  7.53         17958 KB/s
gnutar  create          159.240 0.256   12.417  7.95         20827 KB/s
star    create          181.779 0.140   14.789  8.21         18203 KB/s

bsdtar  list            0.053   0.032   0.016   91.41     62435471 KB/s
gnutar  list            56.535  0.136   3.764   6.89         58531 KB/s
star    list            56.652  0.080   5.236   9.38         58410 KB/s

bsdtar  extract         78.914  0.820   15.149  20.23        41932 KB/s
gnutar  extract         78.480  0.196   14.197  18.33        42164 KB/s
star    extract         79.439  0.132   12.973  16.49        41655 KB/s

bsdtar  compare         0.001   0.000   0.000   0.00    3309080000 KB/s
gnutar  compare         61.771  3.464   8.905   20.02        53570 KB/s
star    compare         57.561  1.728   9.897   20.19        57488 KB/s


Linux 2.6.22-suspend2-smp / Gentoo Base System release 2.0.0_rc5
504MB of memory, 1 x AMD Athlon(tm) Processor 1500.033 256 KB 3002.55 bmips
gcc (GCC) 4.2.2 (Gentoo 4.2.2 p1.0)
CFLAGS="-O2 -march=athlon-xp -mfpmath=sse -frename-registers -pipe"

bsdtar: bsdtar 2.3.4 - libarchive 2.3.4
gnutar: tar (GNU tar) 1.19
star:   star: star 1.5a85 (i686-pc-linux-gnu)

best time of 3 repetitions,
        src="" 119M in 5900 files,
        avg 21KB/file, archive=/tmp/tcp.tar, extract to /tmp/tcptmp
program operation       real    user    system  %CPU         speed
bsdtar  create          1.329   0.192   1.132   99.63        89784 KB/s
gnutar  create          1.223   0.124   1.092   99.46        97566 KB/s
star    create          1.848   0.036   1.708   94.36        61372 KB/s

bsdtar  list            0.167   0.060   0.108   100.00      679137 KB/s
gnutar  list            0.161   0.040   0.124   100.00      704447 KB/s
star    list            0.859   0.044   0.716   88.51       132032 KB/s

bsdtar  extract         1.186   0.172   1.012   99.87        95629 KB/s
gnutar  extract         1.064   0.056   1.004   99.63       106593 KB/s
star    extract         1.920   0.088   1.724   94.40        59070 KB/s

bsdtar  compare         0.002   0.000   0.000   0.00      56708000 KB/s
gnutar  compare         0.925   0.232   0.692   99.90       122611 KB/s
star    compare         1.569   0.376   1.096   93.79        72285 KB/s

[...]
best time of 3 repetitions,
        src="" 2.1G in 8416 files, avg 277KB/file,
        archive=/home/j2/tcp.tar, extract to /mnt/a/tar/tcptmp
program operation       real    user    system  %CPU         speed
bsdtar  create          182.171 1.692   29.130  16.91        11584 KB/s
gnutar  create          174.999 0.632   27.450  16.04        12059 KB/s
star    create          180.004 0.360   41.795  23.41        11677 KB/s

bsdtar  list            0.214   0.076   0.136   99.04      9822294 KB/s
gnutar  list            0.210   0.076   0.136   100.00    10009385 KB/s
star    list            43.462  0.148   18.109  42.00        48363 KB/s

bsdtar  extract         94.912  4.476   31.574  37.98        22146 KB/s
gnutar  extract         94.657  0.396   29.462  31.54        22206 KB/s
star    extract         100.814 0.400   39.906  39.98        20849 KB/s

bsdtar  compare         0.003   0.000   0.004   100.00   700657000 KB/s
gnutar  compare         80.174  3.932   20.365  30.30        26217 KB/s
star    compare         73.911  8.341   27.670  48.72        28439 KB/s

-----------------------------------------------------------------------------
#!/bin/sh
# tar comparision program
# 2007-10-25 Jan Psota

n=3                                     # number of repetitions
TAR=(bsdtar gnutar star)                # TApeArchivers to compare
OPT=("" "--seek" "-no-fsync")
pax="--format=pax"                      # comment out for defaults
OPN=(create list extract compare)       # operations
version="2007-10-25"
TIMEFORMAT=$'%R\t%U\t%S\t%P'
LC_ALL=C

test $# -ge 2 || {
        echo -e "usage:\t$0 source_dir where_to_place_archive [where_to_extract_it]

TCP, version $version
TCP stands for Tar Comparision Program here.
It currently compares: BSD tar (bsdtar), GNU tar (gnutar) and star in archive
creation, listing, extraction and archive-to-extracted comparision.
Tcp prints out best time of n=$n repetitions.

Tcp creates temporary archive named tcp.tar with $pax and some native
(--seek/-no-fsync) options and extracts it to [\$3]/tcptmp/.
If unset, third argument defaults to [\$2].
After normal exit tcp removes tarball and extracted files.
Tcp does not check filesystems destination directories are on for free space,
so make sure there is enough space (a bit more than source_dir uses) for both:
archive and extracted files.
Do not use white space in arguments.
        Jan Psota, $version"
        exit 0
}
src=""> dst=$2/tcp.tar
dst_path=${3:-$2}/tcptmp
test -e $dst -o -e /tmp/tcp \
        && { echo "$dst or /tmp/tcp exists, exiting"; exit 1; }
mkdir $dst_path || exit 2

use_times ()
{
        awk -F"\t" -vN=$n -vL="`du -k $dst`" -vOFS="\t" -vORS="" '
                { if (NF==4) { printf "\t%s\t%10.1d KB/s\n", $0, ($1+0>0 ? (L+0)/($1+0) : 0) } }' \
                /tmp/tcp | sort | head -1
        > /tmp/tcp
}

test -d $src || { echo "'$src' is not a directory"; exit 3; }

# system information: type, release, memory, cpu(s), compiler and flags
echo -e "TCP, version $version\n"`uname -sr`" / "`head -1 /etc/*-release`
free -m | awk '/^Mem/ { printf "%dMB of memory, ", $2 }'
test -e /proc/cpuinfo \
        && awk -F: '/name|cache size|MHz|mips/ { if (!a) b=b $2 }
        /^$/ { a++ } END { print a" x"b" bmips" }' /proc/cpuinfo
test -e /etc/gentoo-release \
        && gcc --version | head -1 && grep ^CFLAGS /etc/make.conf

# tar versions
echo
for tar in address@hidden; do echo -ne "$tar:\t"; $tar --version | head -1; done

echo -e "\nbest time of $n repetitions,\n"\
"       src="" "\
`du -sh $src | awk '{print $1}'`" in "`find $src | wc -l`" files, "\
"avg "$((`du -sk $src | awk '{print $1}'`/`find $src -type f | wc -l`))"KB/file,\n"\
"       archive=$dst, extract to $dst_path"

echo -e "program\toperation\treal\tuser\tsystem\t%CPU\t     speed"
> /tmp/tcp
let op_num=0
for op in "cf $dst $pax -C $src ." "tf $dst" "xf $dst -C $dst_path" \
        "f $dst -C $dst_path --diff"; do
        let tar_num=0
        for tar in address@hidden; do
                echo -en "$tar\t${OPN[op_num]}\t"
                for ((i=1; i<=$n; i++)); do
                        echo $op | grep -q ^cf && rm -f $dst
                        echo $op | grep -q ^xf &&
                                { chmod -R u+w $dst_path
                                rm -rf $dst_path; mkdir $dst_path; }
                        sync
                        if echo $op | grep -q ^f; then  # op == compare
                                time $tar $op ${OPT[$tar_num]} > /dev/null
                        else    # op in (create | list | extract)
                                time $tar $op ${OPT[$tar_num]} > /dev/null \
                                        || break 3
                        fi 2>> /tmp/tcp
                done
                use_times
                let tar_num++
        done
        let op_num++
        echo
done
rm -rf $dst_path $dst
echo
cat /tmp/tcp
rm -f /tmp/tcp