[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Quilt-dev] [PATCH 2/2 RFC] setup: Add --fast option
|
From: |
Jean Delvare |
|
Subject: |
[Quilt-dev] [PATCH 2/2 RFC] setup: Add --fast option |
|
Date: |
Tue, 18 Nov 2014 11:51:16 +0100 |
This is an alternative implementation of "quilt setup" for rpm spec
files, which is much faster than the original implementation. The
idea is to have rpmbuild generate our working tree directly, instead
of taking note of everything it does and then replaying that record
from scratch.
In general, the generated working tree should be the same with and
without --fast. There are 3 known exceptions though:
* The header of the series files is incomplete when using --fast,
which means that you can't reuse these series files to feed "quilt
setup" later.
* If the spec file generates files other than with the tar, unzip and
patch commands, you wouldn't get these files in your working tree
before, but now with --fast you will.
* With --fast, all patches are added to the series file, regardless
of whether they apply or not. This also means that patches which
fail to apply are not reported as such until you attempt to push
them.
On large packages, this alternative implementation was found to be
up to 4 times faster than the original implementation.
---
This is my SUSE hackweek 11 project, more information at:
https://hackweek.suse.com/11/projects/194
Note that this is only a RFC, I do not intend to commit this right now
as I first want to investigate ways to minimize the differences between
legacy mode and fast mode code paths in order to reduce the complexity
and code redundancy.
Makefile.in | 6
quilt/scripts/inspect-wrapper-fast.in | 208 ++++++++++++++++++++++++++++++++++
quilt/scripts/inspect.in | 38 ++++--
quilt/setup.in | 31 ++++-
4 files changed, 267 insertions(+), 16 deletions(-)
--- a/Makefile.in
+++ b/Makefile.in
@@ -78,8 +78,8 @@ QUILT := $(QUILT_IN)
SRC += $(QUILT_SRC:%=quilt/%)
DIRT += $(QUILT_IN:%=quilt/%)
-SCRIPTS_IN := patchfns inspect inspect-wrapper dependency-graph \
- edmail remove-trailing-ws backup-files
+SCRIPTS_IN := patchfns inspect inspect-wrapper inspect-wrapper-fast \
+ dependency-graph edmail remove-trailing-ws backup-files
SCRIPTS_SRC := $(SCRIPTS_IN:%=%.in)
SCRIPTS := $(SCRIPTS_IN)
@@ -390,7 +390,7 @@ test/.depend : Makefile $(TESTS)
-e 's:quilt/graph:quilt/graph quilt/scripts/dependency-graph:' \
-e 's:quilt/mail:quilt/mail quilt/scripts/edmail:' \
-e 's:quilt/refresh:quilt/refresh
quilt/scripts/remove-trailing-ws:' \
- -e 's:quilt/setup:quilt/setup quilt/scripts/inspect
quilt/scripts/inspect-wrapper:' \
+ -e 's:quilt/setup:quilt/setup quilt/scripts/inspect
quilt/scripts/inspect-wrapper quilt/scripts/inspect-wrapper-fast:' \
> $@
ifneq ($(shell . $(QUILTRC) ; echo $$QUILT_PATCHES_PREFIX),)
--- /dev/null
+++ b/quilt/scripts/inspect-wrapper-fast.in
@@ -0,0 +1,208 @@
+#! @BASH@
+
+# find original data file by md5sum
+original_file()
+{
+ local file=$1 md5sum
+
+ set -- $(md5sum < $file)
+ md5sum=$1
+ while read md5sum_ file_
+ do
+ if [ "$md5sum" = "$md5sum_" ]
+ then
+ echo ${file_#\*}
+ return 0
+ fi
+ done < $tmpdir/md5sums
+
+ # Try harder
+ if ! [ -e $tmpdir/more-md5sums ]
+ then
+ ( cd $RPM_BUILD_DIR
+ find . -type f \
+ | sed -e 's:^.\/::' \
+ | xargs md5sum \
+ ) > $tmpdir/more-md5sums
+ fi
+
+ while read md5sum_ file_
+ do
+ if [ "$md5sum" = "$md5sum_" ]
+ then
+ echo $QUILT_SETUP_PREFIX${file_#\*}
+ return 0
+ fi
+ done < $tmpdir/more-md5sums
+
+ return 1
+}
+
+# Extract a command line option with or without argument
+cmdline_option()
+{
+ local letter=$1 no_arg=$2
+ shift
+
+ while [ $# -ne 0 ]
+ do
+ if [ "${1:0:2}" = -$letter ]
+ then
+ if [ -z "$no_arg" ]
+ then
+ [ "$1" = -$letter ] && set -- "$1$2"
+ fi
+ echo $1
+ break
+ fi
+ shift
+ done
+}
+
+# Extract the -p option from the command line
+strip_option()
+{
+ set -- $(cmdline_option p "$@")
+ [ "$1" != -p1 ] && echo $1
+}
+
+# Extract the -R option from the command line
+reverse_option()
+{
+ set -- $(cmdline_option R no_arg "$@")
+ echo $1
+}
+
+patch_opt_d()
+{
+ local subdir=$(cmdline_option d "$@")
+ [ -z "$subdir" ] || echo "${subdir:2}"
+
+}
+
+patch_input_file()
+{
+ while [ $# -gt 0 ]
+ do
+ case "$1" in
+ -i|--input)
+ if [ $# -ge 2 ]
+ then
+ echo "$2"
+ return
+ fi
+ ;;
+ -i*)
+ echo "${1#-i}"
+ return
+ ;;
+ --input=*)
+ echo "${1#--input=}"
+ return
+ ;;
+ esac
+ shift
+ done
+ return 1
+}
+
+pwd_to_dir()
+{
+ local subdir=$1 dir
+
+ if [ -n "$subdir" ]
+ then
+ dir=$(cd "$subdir" && echo $PWD)
+ else
+ dir=$PWD
+ fi
+ dir=${dir/$RPM_BUILD_DIR}
+ dir=${dir##/}
+ dir=${dir// /\\ }
+
+ echo "$dir"
+}
+
+create_patches_symlink()
+{
+ local target=$(normalize_path "$1") link=$(normalize_path "$2")
+
+ # Two cases here: either the target is a substring of the
+ # link, and we want a relative path; or it is not and we want
+ # an absolute path.
+ if [ "${link:0:${#target}}" = "$target" ]
+ then
+ target="${link:${#target}}"
+ target=$(dirname "${target#/}" | sed -e
'address@hidden/][^/address@hidden@g')
+ fi
+
+ ln -s "$target" "$link"
+}
+
+PATH=${PATH#*:}
+# If we are called too early, pass through without processing
+[ -n "$RPM_BUILD_DIR" ] || exec ${0##*/} "$@"
+
+tmpdir=${RPM_BUILD_DIR%/*}
+case "${0##*/}" in
+patch)
+ inputfile=$(patch_input_file "$@")
+ ;;
+esac
+if [ -n "$inputfile" ]
+then
+ unpackfile=$(original_file "$inputfile")
+else
+ # put data from stdin into tmpfile
+ cat > $tmpdir/data
+ unpackfile=$(original_file $tmpdir/data)
+fi
+
+if [ -n "$unpackfile" ]
+then
+ case "${0##*/}" in
+ patch)
+ echo -n p >&4
+ subdir=$(patch_opt_d "$@")
+ dir=$(pwd_to_dir $subdir)
+
+ # For the first patch, we need to setup the quilt
+ # environment first. For subsequent patches, we must
+ # reuse the same setup.
+ if [ -e "$QUILT_PC" ]
+ then
+ QUILT_SERIES=$(< $QUILT_PC/.quilt_series)
+ else
+ # Check for potential collisions, use alternative
+ # names if needed
+ [ -e "$QUILT_PATCHES" ] && QUILT_PATCHES=quilt_patches
+ [ -e "$QUILT_SERIES" ] && QUILT_SERIES=quilt_series
+ if [ -e "$QUILT_PATCHES" -o -e "$QUILT_SERIES" ]
+ then
+ exit 1
+ fi
+
+ targetdir=$(readlink "$RPM_BUILD_DIR")
+ create_patches_symlink "$RPM_SOURCE_DIR"
"$targetdir$dir/$QUILT_PATCHES"
+ ( echo "# Patch series file for quilt," \
+ "created by quilt setup"
+ echo "# Patchdir: ${dir:-.}"
+ echo "#"
+ ) > "$QUILT_SERIES"
+
+ # Stripped down version of create_db, could be replaced
+ # by a call to "quilt init" when it is implemented
+ mkdir -p $QUILT_PC
+ echo $DB_VERSION > $QUILT_PC/.version
+ echo "$QUILT_PATCHES" > $QUILT_PC/.quilt_patches
+ echo "$QUILT_SERIES" > $QUILT_PC/.quilt_series
+ fi
+
+ # Don't let rpmbuild apply the patch, instead add it
+ # to the series file and let the user apply it later
+ # using quilt
+ echo "$unpackfile" $(strip_option "$@") \
+ $(reverse_option "$@") >> "$QUILT_SERIES"
+ ;;
+ esac
+fi
--- a/quilt/scripts/inspect.in
+++ b/quilt/scripts/inspect.in
@@ -21,11 +21,11 @@ cd ${SUBDIR:-.}
usage()
{
- echo "Usage: ${0##*/} [-v] [--sourcedir dir] [--fuzz=N] specfile"
+ echo "Usage: ${0##*/} [-v] [--sourcedir dir] [--targetdir dir]
[--fuzz=N] specfile"
exit 1
}
-options=$(getopt -o v --long sourcedir:,fuzz: -n "${0##*/}" -- "$@") || exit
+options=$(getopt -o v --long sourcedir:,targetdir:,fuzz: -n "${0##*/}" --
"$@") || exit
eval set -- "$options"
@@ -40,6 +40,9 @@ do
--sourcedir)
sourcedir=${2%/}/
shift 2 ;;
+ --targetdir)
+ targetdir=$2
+ shift 2 ;;
--fuzz)
# Only works with rpm 4.6 and later
DEFINE_FUZZ="%define _default_patch_fuzz $2"
@@ -68,8 +71,22 @@ fi
tmpdir=$(gen_tempfile -d ${VARTMPDIR:-/var/tmp}/${0##*/})
mkdir -p $tmpdir || exit 1
add_exit_handler "rm -rf $tmpdir"
-mkdir -p $tmpdir/build
mkdir -p $tmpdir/bin
+if [ -n "$targetdir" ]
+then
+ # Fast mode
+ [ -d "$targetdir" ] || mkdir -p "$targetdir" || exit 1
+ ln -s "$targetdir" $tmpdir/build
+ ln -s $QUILT_DIR/scripts/inspect-wrapper-fast $tmpdir/bin/patch
+else
+ # Standard mode
+ mkdir -p $tmpdir/build
+ ln -s $QUILT_DIR/scripts/inspect-wrapper $tmpdir/bin/patch
+ ln -s $QUILT_DIR/scripts/inspect-wrapper $tmpdir/bin/tar
+ ln -s $QUILT_DIR/scripts/inspect-wrapper $tmpdir/bin/unzip
+ DEFINE_TAR="%define __tar $tmpdir/bin/tar"
+ DEFINE_UNZIP="%define __unzip $tmpdir/bin/unzip"
+fi
# Redirect file descriptors
# 5 is used in verbose mode, 4 in non-verbose mode, and 2 for both (real
errors)
@@ -90,6 +107,11 @@ do
ready|bigpack|_constraints|_service|baselibs.conf|MD5SUMS|MD5SUMS.meta|*.spec|*.changes|*.sig|*.sign|*rpmlintrc)
continue
;;
+ # In fast mode, we are only interested in patches, so filter out
+ # archives
+ *.tar|*.tar.Z|*.tar.gz|*.tgz|*.tar.bz2|*.tar.xz|*.zip)
+ [ -n "$targetdir" ] && continue
+ ;;
esac
[ -f "$file" ] || continue
echo -n "." >&4
@@ -142,20 +164,18 @@ done > $tmpdir/md5sums
echo >&4
shopt -u nullglob
-ln -s $QUILT_DIR/scripts/inspect-wrapper $tmpdir/bin/patch
-ln -s $QUILT_DIR/scripts/inspect-wrapper $tmpdir/bin/tar
-ln -s $QUILT_DIR/scripts/inspect-wrapper $tmpdir/bin/unzip
-
# let rpm do all the dirty specfile stuff ...
echo -n "### rpmbuild: " >&4
export PATH="$tmpdir/bin:$PATH"
+export QUILT_PC QUILT_PATCHES QUILT_SERIES DB_VERSION
+export -f normalize_path
rpmbuild --eval "%define _sourcedir $sourcedir" \
--eval "%define _specdir $specdir" \
--eval "%define _builddir $tmpdir/build" \
--eval "%define __patch $tmpdir/bin/patch" \
- --eval "%define __tar $tmpdir/bin/tar" \
- --eval "%define __unzip $tmpdir/bin/unzip" \
+ --eval "$DEFINE_TAR" \
+ --eval "$DEFINE_UNZIP" \
--eval "$DEFINE_FUZZ" \
--nodeps \
-bp "$specdir/$specfile" < /dev/null >&5 2>&5
--- a/quilt/setup.in
+++ b/quilt/setup.in
@@ -115,7 +115,7 @@ dir_to_dir()
usage()
{
- printf $"Usage: quilt setup [-d path-prefix] [-v] [--sourcedir dir]
[--fuzz=N] {specfile|seriesfile}\n"
+ printf $"Usage: quilt setup [-d path-prefix] [-v] [--sourcedir dir]
[--fuzz=N] [--fast] {specfile|seriesfile}\n"
if [ x$1 = x-h ]
then
printf $"
@@ -130,6 +130,8 @@ Initializes a source tree from an rpm sp
--fuzz=N
Set the maximum fuzz factor (needs rpm 4.6 or later).
+
+--fast Use an alternative, faster method to process the spec file.
"
exit 0
else
@@ -137,7 +139,7 @@ Initializes a source tree from an rpm sp
fi
}
-options=`getopt -o d:vh --long sourcedir:,fuzz: -- "$@"`
+options=`getopt -o d:vh --long sourcedir:,fuzz:,fast -- "$@"`
if [ $? -ne 0 ]
then
@@ -166,6 +168,9 @@ do
--fuzz)
opt_fuzz="--fuzz $2"
shift 2 ;;
+ --fast)
+ opt_fast=1
+ shift ;;
--)
shift
break ;;
@@ -194,8 +199,24 @@ case "$1" in
# check if rpmbuild is installed before running inspect
check_external_tool rpmbuild rpm-build
- if ! $QUILT_DIR/scripts/inspect $verbose $opt_sourcedir $opt_fuzz \
- "$spec_file" 2>&1 > $tmpfile
+ if [ -n "$opt_fast" ]
+ then
+ if [ "${prefix:0:1}" = / ]
+ then
+ targetdir=$prefix
+ else
+ targetdir=$PWD/$prefix
+ fi
+
+ $QUILT_DIR/scripts/inspect $verbose $opt_sourcedir $opt_fuzz \
+ --targetdir "$targetdir" \
+ "$spec_file" 2>&1
+ else
+ $QUILT_DIR/scripts/inspect $verbose $opt_sourcedir $opt_fuzz \
+ "$spec_file" 2>&1 > $tmpfile
+ fi
+
+ if [ $? -ne 0 ]
then
printf $"The %%prep section of %s failed; results may be
incomplete\n" "$spec_file"
if [ -z "$verbose" ]
@@ -203,6 +224,8 @@ case "$1" in
printf $"The -v option will show rpm's output\n"
fi
fi
+
+ [ -n "$opt_fast" ] && exit
;;
*)
series_file=$1
--
Jean Delvare
SUSE L3 Support