>From 5ba2828e72f6d276fc349f69824cd6cb626053a4 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 15:41:17 -0600
Subject: [PATCH 00/14] *** SUBJECT HERE ***
*** BLURB HERE ***
Assaf Gordon (14):
shuf: add --repetition to generate random numbers
shuf: add tests for --repetition option
shuf: mention new --repetition option in NEWS
shuf: document new --repetition option
shuf: enable --repetition on stdin/FILE/-e input
shuf: add tests for --repetition with STDIN
shuf: document new --repetitions option
shuf: code-cleanup
shuf: add more tests
shuf: refactor --repetition with stdin
shuf: refactor write_permuted_output()
shuf: code cleanup
shuf: code clean-up
shuf: add tests for more erroneous usage
NEWS | 3 +
doc/coreutils.texi | 37 +++++++++++
src/shuf.c | 192 +++++++++++++++++++++++++++++++++++++----------------
tests/misc/shuf.sh | 92 +++++++++++++++++++++++++
4 files changed, 268 insertions(+), 56 deletions(-)
--
1.8.3.2
>From c41160016ed36fe5b4e2b3d03cde34e0dcec84b6 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Thu, 4 Jul 2013 13:26:45 -0600
Subject: [PATCH 01/14] shuf: add --repetition to generate random numbers
* src/shuf.c: new option (-r,--repetition), generate random numbers.
main(): process new option.
usage(): mention new option.
write_random_numbers(): generate random numbers.
---
src/shuf.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 46 insertions(+), 4 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index 0fabb0b..cdc3151 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -76,6 +76,9 @@ Write a random permutation of the input lines to standard output.\n\
-n, --head-count=COUNT output at most COUNT lines\n\
-o, --output=FILE write result to FILE instead of standard output\n\
--random-source=FILE get random bytes from FILE\n\
+ -r, --repetition used with -iLO-HI, output COUNT random numbers\n\
+ between LO and HI, with repetitions.\n\
+ count defaults to 1 if -n COUNT is not used.\n\
-z, --zero-terminated end lines with 0 byte, not newline\n\
"), stdout);
fputs (HELP_OPTION_DESCRIPTION, stdout);
@@ -104,6 +107,7 @@ static struct option const long_opts[] =
{"head-count", required_argument, NULL, 'n'},
{"output", required_argument, NULL, 'o'},
{"random-source", required_argument, NULL, RANDOM_SOURCE_OPTION},
+ {"repetition", no_argument, NULL, 'r'},
{"zero-terminated", no_argument, NULL, 'z'},
{GETOPT_HELP_OPTION_DECL},
{GETOPT_VERSION_OPTION_DECL},
@@ -328,6 +332,23 @@ write_permuted_output (size_t n_lines, char *const *line, size_t lo_input,
return 0;
}
+static int
+write_random_numbers (struct randint_source *s, size_t count,
+ size_t lo_input, size_t hi_input, char eolbyte)
+{
+ size_t i;
+ const randint range = hi_input - lo_input + 1;
+
+ for (i = 0; i < count; i++)
+ {
+ randint j = lo_input + randint_choose (s, range);
+ if (printf ("%lu%c", j, eolbyte) < 0)
+ return -1;
+ }
+
+ return 0;
+}
+
int
main (int argc, char **argv)
{
@@ -340,6 +361,7 @@ main (int argc, char **argv)
char eolbyte = '\n';
char **input_lines = NULL;
bool use_reservoir_sampling = false;
+ bool repetition = false;
int optc;
int n_operands;
@@ -348,7 +370,7 @@ main (int argc, char **argv)
char **line = NULL;
struct linebuffer *reservoir = NULL;
struct randint_source *randint_source;
- size_t *permutation;
+ size_t *permutation = NULL;
int i;
initialize_main (&argc, &argv);
@@ -424,6 +446,10 @@ main (int argc, char **argv)
random_source = optarg;
break;
+ case 'r':
+ repetition = true;
+ break;
+
case 'z':
eolbyte = '\0';
break;
@@ -454,9 +480,19 @@ main (int argc, char **argv)
}
n_lines = hi_input - lo_input + 1;
line = NULL;
+
+ /* When generating random numbers with repetitions,
+ the default count is one, unless specified by the user */
+ if (repetition && head_lines == SIZE_MAX)
+ head_lines = 1 ;
}
else
{
+ if (repetition)
+ {
+ error (0, 0, _("--repetition requires --input-range"));
+ usage (EXIT_FAILURE);
+ }
switch (n_operands)
{
case 0:
@@ -488,10 +524,12 @@ main (int argc, char **argv)
}
}
- head_lines = MIN (head_lines, n_lines);
+ if (!repetition)
+ head_lines = MIN (head_lines, n_lines);
randint_source = randint_all_new (random_source,
- use_reservoir_sampling ? SIZE_MAX :
+ (use_reservoir_sampling||repetition)?
+ SIZE_MAX:
randperm_bound (head_lines, n_lines));
if (! randint_source)
error (EXIT_FAILURE, errno, "%s", quotearg_colon (random_source));
@@ -512,13 +550,17 @@ main (int argc, char **argv)
&& (fclose (stdin) != 0))
error (EXIT_FAILURE, errno, _("read error"));
- permutation = randperm_new (randint_source, head_lines, n_lines);
+ if (!repetition)
+ permutation = randperm_new (randint_source, head_lines, n_lines);
if (outfile && ! freopen (outfile, "w", stdout))
error (EXIT_FAILURE, errno, "%s", quotearg_colon (outfile));
if (use_reservoir_sampling)
i = write_permuted_output_reservoir (n_lines, reservoir, permutation);
+ else if (repetition)
+ i = write_random_numbers (randint_source, head_lines,
+ lo_input, hi_input, eolbyte);
else
i = write_permuted_output (head_lines, line, lo_input,
permutation, eolbyte);
--
1.8.3.2
>From 545e164dd89a785ce28b867e3c25c55142dea204 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Thu, 4 Jul 2013 13:54:04 -0600
Subject: [PATCH 02/14] shuf: add tests for --repetition option
* tests/misc/shuf.sh: add tests for --repetition option.
---
tests/misc/shuf.sh | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/tests/misc/shuf.sh b/tests/misc/shuf.sh
index 3e33b61..3cf39c3 100755
--- a/tests/misc/shuf.sh
+++ b/tests/misc/shuf.sh
@@ -65,4 +65,33 @@ if ! test -r unreadable; then
shuf -n1 unreadable && fail=1
fi
+# Test --repetition option
+
+# --repetition should fail without --input-range
+shuf --repetition &&
+ { fail=1; echo "--repetition should fail without range">&2 ; }
+
+# --repetition without count should return one line
+shuf --rep -i0-10 > exp || framework_failure_
+c=$(cat exp | wc -l) || framework_failure_
+test "$c" -eq 1 || { fail=1; echo "--repetition default count is not 1">&2 ; }
+
+# --repetition can output more values than the input range
+shuf --rep -i0-9 -n1000 > exp || framework_failure_
+c=$(cat exp | wc -l) || framework_failure_
+test "$c" -eq 1000 || { fail=1; echo "--repetition with --count failed">&2 ; }
+
+# Check output values (this is not bullet-proof, but drawing 1000 values
+# between 0 and 9 should produce all values, unless there's a bug in shuf
+# or a very poor random source
+c=$(cat exp | sort -nu | fmt ) || framework_failure_
+test "$c" = "0 1 2 3 4 5 6 7 8 9" ||
+ { fail=1; echo "--repetition produced bad output">&2 ; }
+
+# check --repetition with non-zero low value
+shuf --rep -i222-233 -n2000 > exp || framework_failure_
+c=$(cat exp | sort -nu | fmt ) || framework_failure_
+test "$c" = "222 223 224 225 226 227 228 229 230 231 232 233" ||
+ { fail=1; echo "--repetition produced bad output with non-zero low">&2 ; }
+
Exit $fail
--
1.8.3.2
>From f737c5a4bfc1d4a8d1b2c39c09c332a9b0c3aaf4 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Thu, 4 Jul 2013 13:55:59 -0600
Subject: [PATCH 03/14] shuf: mention new --repetition option in NEWS
* NEWS: mention new shuf option.
---
NEWS | 3 +++
1 file changed, 3 insertions(+)
diff --git a/NEWS b/NEWS
index 75ec253..9736690 100644
--- a/NEWS
+++ b/NEWS
@@ -42,6 +42,9 @@ GNU coreutils NEWS -*- outline -*-
csplit accepts a new option: --suppressed-matched, to elide the lines
used to identify the split points.
+ shuf accepts a new option: --repetition (-r). Output random numbers in
+ a given range, with repetitions.
+
** Changes in behavior
stdbuf now requires at least one buffering mode option to be specified,
--
1.8.3.2
>From bd97294c36dbbc4e0775a92c69bd53cbd0cdcef8 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Thu, 4 Jul 2013 14:31:06 -0600
Subject: [PATCH 04/14] shuf: document new --repetition option
* doc/coreutils.texi: mention --repetition, add examples.
---
doc/coreutils.texi | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index b3233f6..817cd7e 100644
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -4945,6 +4945,15 @@ commands like @code{shuf -o F From 2445fe3a9db16f87a56f5f04f2afc62d27feb37e Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 11:24:54 -0600
Subject: [PATCH 05/14] shuf: enable --repetition on stdin/FILE/-e input
* src/shuf.c:
usage(): update usage information.
main(): enable repetition on input data from stdin.
randint_choose_new(): allocate N random values.
TODO: move this function to gnulib's randint.c.
* tests/misc/shuf.sh: remove obsolete test.
---
src/shuf.c | 44 +++++++++++++++++++++++++++-----------------
tests/misc/shuf.sh | 4 ----
2 files changed, 27 insertions(+), 21 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index cdc3151..26d7da1 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -76,8 +76,9 @@ Write a random permutation of the input lines to standard output.\n\
-n, --head-count=COUNT output at most COUNT lines\n\
-o, --output=FILE write result to FILE instead of standard output\n\
--random-source=FILE get random bytes from FILE\n\
- -r, --repetition used with -iLO-HI, output COUNT random numbers\n\
- between LO and HI, with repetitions.\n\
+ -r, --repetitions output COUNT values, with repetitions.\n\
+ with -iLO-HI, output random numbers.\n\
+ with -e, stdin or FILE, output random lines.\n\
count defaults to 1 if -n COUNT is not used.\n\
-z, --zero-terminated end lines with 0 byte, not newline\n\
"), stdout);
@@ -107,13 +108,25 @@ static struct option const long_opts[] =
{"head-count", required_argument, NULL, 'n'},
{"output", required_argument, NULL, 'o'},
{"random-source", required_argument, NULL, RANDOM_SOURCE_OPTION},
- {"repetition", no_argument, NULL, 'r'},
+ {"repetitions", no_argument, NULL, 'r'},
{"zero-terminated", no_argument, NULL, 'z'},
{GETOPT_HELP_OPTION_DECL},
{GETOPT_VERSION_OPTION_DECL},
{0, 0, 0, 0},
};
+/* Returns a newly allocated array of COUNT random elements,
+ between 0-(CHOICES-1) */
+static size_t *
+randint_choose_new (struct randint_source *src, size_t choices, size_t count)
+{
+ size_t i;
+ size_t *v = xnmalloc (count, sizeof *v);
+ for (i=0; i RESERVOIR_MIN_INPUT))
+ if (!repetition && head_lines != SIZE_MAX
+ && (! head_lines || input_size () > RESERVOIR_MIN_INPUT))
{
use_reservoir_sampling = true;
n_lines = SIZE_MAX; /* unknown number of input lines, for now. */
@@ -524,6 +527,11 @@ main (int argc, char **argv)
}
}
+ /* When generating random numbers with repetitions,
+ the default count is one, unless specified by the user */
+ if (repetition && head_lines == SIZE_MAX)
+ head_lines = 1 ;
+
if (!repetition)
head_lines = MIN (head_lines, n_lines);
@@ -552,13 +560,15 @@ main (int argc, char **argv)
if (!repetition)
permutation = randperm_new (randint_source, head_lines, n_lines);
+ if (repetition && !input_numbers_option_used (lo_input, hi_input))
+ permutation = randint_choose_new (randint_source, n_lines, head_lines);
if (outfile && ! freopen (outfile, "w", stdout))
error (EXIT_FAILURE, errno, "%s", quotearg_colon (outfile));
if (use_reservoir_sampling)
i = write_permuted_output_reservoir (n_lines, reservoir, permutation);
- else if (repetition)
+ else if (repetition && input_numbers_option_used (lo_input, hi_input))
i = write_random_numbers (randint_source, head_lines,
lo_input, hi_input, eolbyte);
else
diff --git a/tests/misc/shuf.sh b/tests/misc/shuf.sh
index 3cf39c3..bdf6645 100755
--- a/tests/misc/shuf.sh
+++ b/tests/misc/shuf.sh
@@ -67,10 +67,6 @@ fi
# Test --repetition option
-# --repetition should fail without --input-range
-shuf --repetition &&
- { fail=1; echo "--repetition should fail without range">&2 ; }
-
# --repetition without count should return one line
shuf --rep -i0-10 > exp || framework_failure_
c=$(cat exp | wc -l) || framework_failure_
--
1.8.3.2
>From 77ad196b0ba203bdec9a2ef33f5f8a10b5643f1b Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 11:43:16 -0600
Subject: [PATCH 06/14] shuf: add tests for --repetition with STDIN
* tests/misc/shuf.sh: add more tests.
---
tests/misc/shuf.sh | 40 +++++++++++++++++++++++++++++++++++++++-
1 file changed, 39 insertions(+), 1 deletion(-)
diff --git a/tests/misc/shuf.sh b/tests/misc/shuf.sh
index bdf6645..63dc764 100755
--- a/tests/misc/shuf.sh
+++ b/tests/misc/shuf.sh
@@ -79,7 +79,7 @@ test "$c" -eq 1000 || { fail=1; echo "--repetition with --count failed">&2 ; }
# Check output values (this is not bullet-proof, but drawing 1000 values
# between 0 and 9 should produce all values, unless there's a bug in shuf
-# or a very poor random source
+# or a very poor random source, or extremely bad luck)
c=$(cat exp | sort -nu | fmt ) || framework_failure_
test "$c" = "0 1 2 3 4 5 6 7 8 9" ||
{ fail=1; echo "--repetition produced bad output">&2 ; }
@@ -90,4 +90,42 @@ c=$(cat exp | sort -nu | fmt ) || framework_failure_
test "$c" = "222 223 224 225 226 227 228 229 230 231 232 233" ||
{ fail=1; echo "--repetition produced bad output with non-zero low">&2 ; }
+# --repetition,-i,count=0 should not fail and produce no output
+shuf --rep -i0-9 -n0 > exp || framework_failure_
+# file size should be zero (no output from shuf)
+test \! -s exp ||
+ { fail=1; echo "--repetition,-i0-9,-n0 produced bad output">&2 ; }
+
+# --repetition with -e, without count, should return one line
+shuf --rep -e A B C D > exp || framework_failure_
+c=$(cat exp | wc -l) || framework_failure_
+test "$c" -eq 1 ||
+ { fail=1; echo "--repetition,-e default count is not 1">&2 ; }
+
+# --repetition with STDIN, without count, should return one line
+printf "A\nB\nC\nD\nE\n" | shuf --rep > exp || framework_failure_
+c=$(cat exp | wc -l) || framework_failure_
+test "$c" -eq 1 ||
+ { fail=1; echo "--repetition,STDIN default count is not 1">&2 ; }
+
+# --repetition with STDIN,count - can return move values than input lines
+printf "A\nB\nC\nD\nE\n" | shuf --rep -n2000 > exp || framework_failure_
+c=$(cat exp | wc -l) || framework_failure_
+test "$c" -eq 2000 ||
+ { fail=1; echo "--repetition,STDIN,count failed">&2 ; }
+
+# Check output values (this is not bullet-proof, but drawing 2000 values
+# between A and E should produce all values, unless there's a bug in shuf
+# or a very poor random source, or extremely bad luck)
+c=$(cat exp | sort -u | fmt ) || framework_failure_
+test "$c" = "A B C D E" ||
+ { fail=1; echo "--repetition,STDIN,count produced bad output">&2 ; }
+
+# --repetition,stdin,count=0 should not fail and produce no output
+printf "A\nB\nC\nD\nE\n" | shuf --rep -n0 > exp || framework_failure_
+# file size should be zero (no output from shuf)
+test \! -s exp ||
+ { fail=1; echo "--repetition,STDIN,-n0 produced bad output">&2 ; }
+
+
Exit $fail
--
1.8.3.2
>From 9e14bf963eb27faed847a979677fb5f344c27362 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 11:55:57 -0600
Subject: [PATCH 07/14] shuf: document new --repetitions option
* NEWS: update wording.
* doc/coreutils.texi: update wording and add examples.
---
NEWS | 4 ++--
doc/coreutils.texi | 30 ++++++++++++++++++++++--------
2 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/NEWS b/NEWS
index 9736690..542f5c5 100644
--- a/NEWS
+++ b/NEWS
@@ -42,8 +42,8 @@ GNU coreutils NEWS -*- outline -*-
csplit accepts a new option: --suppressed-matched, to elide the lines
used to identify the split points.
- shuf accepts a new option: --repetition (-r). Output random numbers in
- a given range, with repetitions.
+ shuf accepts a new option: --repetitions (-r). Output random values,
+ with repetitions.
** Changes in behavior
diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index 817cd7e..1ca4b95 100644
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -4946,13 +4946,14 @@ Use @var{file} as a source of random data used to determine which
permutation to generate. @xref{Random sources}.
@item -r
-@itemx --repetition
+@itemx --repetitions
@opindex -r
-@opindex --repetition
-@cindex writing random numbers with repetitions
-output random numbers with repetitions. Use @option{--input-range} to specify
-the range of output values. Use @option{--had-count} to specify count of output
-numbers (defaults to 1 if not specified).
+@opindex --repetitions
+@cindex allowing repetitions in output values
+Changes the default behaviour of @command{shuf}, allowing repetition in
+output values (in which case, @option{--head-count} can be larger
+than the number of input values). If @option{--head-count} is not
+specified, outputs a single random value.
@zeroTerminatedOption
@@ -5017,14 +5018,27 @@ output permutations.
To output 50 random numbers between 0 and 9, use:
@example
-shuf --repetition --input-range 0-9 --head-count 50
+shuf --repetitions --input-range 0-9 --head-count 50
@end example
@noindent
or (using short options):
@example
-shuf --rep -i0-9 -n50
+shuf -r -i0-9 -n50
+@end example
+
+@noindent
+To simulate 100 coin flips, use:
+
+@example
+shuf -r -n100 -e Head Tail
+@end example
+
+@noindent
+or
+@example
+printf "Head\nTail\n" | shuf -r -n100
@end example
@exitstatus
--
1.8.3.2
>From 99db4abc3cf14c0bba89f32a4e14887b7ff3bb25 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 12:55:31 -0600
Subject: [PATCH 08/14] shuf: code-cleanup
* src/shuf.c: replace input_numbers_option_used() with a local variable.
---
src/shuf.c | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index 26d7da1..fc1a483 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -127,12 +127,6 @@ randint_choose_new (struct randint_source *src, size_t choices, size_t count)
return v;
}
-static bool
-input_numbers_option_used (size_t lo_input, size_t hi_input)
-{
- return ! (lo_input == SIZE_MAX && hi_input == 0);
-}
-
static void
input_from_argv (char **operand, int n_operands, char eolbyte)
{
@@ -366,6 +360,7 @@ int
main (int argc, char **argv)
{
bool echo = false;
+ bool input_range = false;
size_t lo_input = SIZE_MAX;
size_t hi_input = 0;
size_t head_lines = SIZE_MAX;
@@ -408,8 +403,9 @@ main (int argc, char **argv)
char const *hi_optarg = optarg;
bool invalid = !p;
- if (input_numbers_option_used (lo_input, hi_input))
+ if (input_range)
error (EXIT_FAILURE, 0, _("multiple -i options specified"));
+ input_range = true;
if (p)
{
@@ -478,13 +474,13 @@ main (int argc, char **argv)
if (echo)
{
- if (input_numbers_option_used (lo_input, hi_input))
+ if (input_range)
error (EXIT_FAILURE, 0, _("cannot combine -e and -i options"));
input_from_argv (operand, n_operands, eolbyte);
n_lines = n_operands;
line = operand;
}
- else if (input_numbers_option_used (lo_input, hi_input))
+ else if (input_range)
{
if (n_operands)
{
@@ -554,13 +550,13 @@ main (int argc, char **argv)
/* Close stdin now, rather than earlier, so that randint_all_new
doesn't have to worry about opening something other than
stdin. */
- if (! (echo || input_numbers_option_used (lo_input, hi_input))
+ if (! (echo || input_range)
&& (fclose (stdin) != 0))
error (EXIT_FAILURE, errno, _("read error"));
if (!repetition)
permutation = randperm_new (randint_source, head_lines, n_lines);
- if (repetition && !input_numbers_option_used (lo_input, hi_input))
+ if (repetition && !input_range)
permutation = randint_choose_new (randint_source, n_lines, head_lines);
if (outfile && ! freopen (outfile, "w", stdout))
@@ -568,7 +564,7 @@ main (int argc, char **argv)
if (use_reservoir_sampling)
i = write_permuted_output_reservoir (n_lines, reservoir, permutation);
- else if (repetition && input_numbers_option_used (lo_input, hi_input))
+ else if (repetition && input_range)
i = write_random_numbers (randint_source, head_lines,
lo_input, hi_input, eolbyte);
else
--
1.8.3.2
>From 784fbd09cda531f6b820216f24a612b6db379500 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 13:06:50 -0600
Subject: [PATCH 09/14] shuf: add more tests
* tests/misc/shuf.sh: add tests for erroneous conditions.
---
tests/misc/shuf.sh | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/tests/misc/shuf.sh b/tests/misc/shuf.sh
index 63dc764..5c70a73 100755
--- a/tests/misc/shuf.sh
+++ b/tests/misc/shuf.sh
@@ -65,6 +65,29 @@ if ! test -r unreadable; then
shuf -n1 unreadable && fail=1
fi
+# Multiple -n is accepted, should use the smallest value
+shuf -n10 -i0-9 -n3 -n20 > exp || framework_failure_
+c=$(cat exp | wc -l) || framework_failure_
+test "$c" -eq 3 || { fail=1; echo "Multiple -n failed">&2 ; }
+
+# Test error conditions
+
+# -i and -e must not be used together
+printf "" | shuf -i -e A B &&
+ { fail=1; echo "shuf did not detect erroneous -e and -i usage.">&2 ; }
+# Test invalid value for -n
+printf "" | shuf -nA &&
+ { fail=1; echo "shuf did not detect erroneous -n usage.">&2 ; }
+# Test multiple -i
+shuf -i0-9 -n10 -i8-90 &&
+ { fail=1; echo "shuf did not detect multiple -i usage.">&2 ; }
+# Test invalid range
+for ARG in "1" "A" "1-" "1-A" ;
+do
+ shuf -i$ARG &&
+ { fail=1; echo "shuf did not detect erroneous -i$ARG usage.">&2 ; }
+done
+
# Test --repetition option
# --repetition without count should return one line
--
1.8.3.2
>From 65ffa3b599210d7875f6375b55caea94f59b3cf4 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 13:29:17 -0600
Subject: [PATCH 10/14] shuf: refactor --repetition with stdin
* src/shuf.c: instead of pre-allocating random numbers, print lines
on-the-go.
randint_choose_new(): removed.
write_random_lines(): draw a random number, print the line.
---
src/shuf.c | 34 ++++++++++++++++++++--------------
1 file changed, 20 insertions(+), 14 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index fc1a483..59012df 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -115,18 +115,6 @@ static struct option const long_opts[] =
{0, 0, 0, 0},
};
-/* Returns a newly allocated array of COUNT random elements,
- between 0-(CHOICES-1) */
-static size_t *
-randint_choose_new (struct randint_source *src, size_t choices, size_t count)
-{
- size_t i;
- size_t *v = xnmalloc (count, sizeof *v);
- for (i=0; iFrom ec42051663a8f3064d50f7c8dc90dd7c90c91dc4 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 13:34:06 -0600
Subject: [PATCH 11/14] shuf: refactor write_permuted_output()
* src/shuf.c: split write_permuted_output() into two smaller functions:
write_permuted_numbers() and write_permuted_lines().
Update main() accordingly.
---
src/shuf.c | 48 +++++++++++++++++++++++++++++-------------------
1 file changed, 29 insertions(+), 19 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index 59012df..e357dce 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -303,26 +303,34 @@ read_input (FILE *in, char eolbyte, char ***pline)
}
static int
-write_permuted_output (size_t n_lines, char *const *line, size_t lo_input,
- size_t const *permutation, char eolbyte)
+write_permuted_lines (size_t n_lines, char *const *line,
+ size_t const *permutation)
{
size_t i;
- if (line)
- for (i = 0; i < n_lines; i++)
- {
- char *const *p = line + permutation[i];
- size_t len = p[1] - p[0];
- if (fwrite (p[0], sizeof *p[0], len, stdout) != len)
- return -1;
- }
- else
- for (i = 0; i < n_lines; i++)
- {
- unsigned long int n = lo_input + permutation[i];
- if (printf ("%lu%c", n, eolbyte) < 0)
- return -1;
- }
+ for (i = 0; i < n_lines; i++)
+ {
+ char *const *p = line + permutation[i];
+ size_t len = p[1] - p[0];
+ if (fwrite (p[0], sizeof *p[0], len, stdout) != len)
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+write_permuted_numbers (size_t n_lines, size_t lo_input,
+ size_t const *permutation, char eolbyte)
+{
+ size_t i;
+
+ for (i = 0; i < n_lines; i++)
+ {
+ unsigned long int n = lo_input + permutation[i];
+ if (printf ("%lu%c", n, eolbyte) < 0)
+ return -1;
+ }
return 0;
}
@@ -573,9 +581,11 @@ main (int argc, char **argv)
lo_input, hi_input, eolbyte);
else if (repetition && !input_range)
i = write_random_lines (randint_source, head_lines, line, n_lines);
+ else if (input_range)
+ i = write_permuted_numbers (head_lines, lo_input, permutation, eolbyte);
else
- i = write_permuted_output (head_lines, line, lo_input,
- permutation, eolbyte);
+ i = write_permuted_lines (head_lines, line, permutation);
+
if (i != 0)
error (EXIT_FAILURE, errno, _("write error"));
--
1.8.3.2
>From 64febd3eafb497b1dfc1ba61834c256a69741893 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 14:20:08 -0600
Subject: [PATCH 12/14] shuf: code cleanup
* src/shuf.c: add comments,
restructure main() output generation section.
---
src/shuf.c | 39 +++++++++++++++++++++++++++++----------
1 file changed, 29 insertions(+), 10 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index e357dce..15748a8 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -302,6 +302,10 @@ read_input (FILE *in, char eolbyte, char ***pline)
return n_lines;
}
+/* output 'n_lines' to stdout from 'line' array,
+ chosen by the indices in 'permutation'.
+ 'permutation' and 'line' must have at least 'n_lines' elements.
+ strings in 'line' must include the line-terminator character. */
static int
write_permuted_lines (size_t n_lines, char *const *line,
size_t const *permutation)
@@ -319,6 +323,8 @@ write_permuted_lines (size_t n_lines, char *const *line,
return 0;
}
+/* output 'n_lines' of numbers to stdout, from 'permutation' array.
+ 'permutation' must have at least 'n_lines' elements. */
static int
write_permuted_numbers (size_t n_lines, size_t lo_input,
size_t const *permutation, char eolbyte)
@@ -335,6 +341,8 @@ write_permuted_numbers (size_t n_lines, size_t lo_input,
return 0;
}
+/* output 'count' numbers to stdout, chosen randomly from range
+ lo_input to hi_input. */
static int
write_random_numbers (struct randint_source *s, size_t count,
size_t lo_input, size_t hi_input, char eolbyte)
@@ -352,6 +360,9 @@ write_random_numbers (struct randint_source *s, size_t count,
return 0;
}
+/* output 'count' lines to stdout from 'lines' array.
+ 'lines' must have at least 'n_lines' element in it.
+ strings in 'line' must include the line-terminator character. */
static int
write_random_lines (struct randint_source *s, size_t count,
char *const *lines, size_t n_lines)
@@ -574,17 +585,25 @@ main (int argc, char **argv)
if (outfile && ! freopen (outfile, "w", stdout))
error (EXIT_FAILURE, errno, "%s", quotearg_colon (outfile));
- if (use_reservoir_sampling)
- i = write_permuted_output_reservoir (n_lines, reservoir, permutation);
- else if (repetition && input_range)
- i = write_random_numbers (randint_source, head_lines,
- lo_input, hi_input, eolbyte);
- else if (repetition && !input_range)
- i = write_random_lines (randint_source, head_lines, line, n_lines);
- else if (input_range)
- i = write_permuted_numbers (head_lines, lo_input, permutation, eolbyte);
+ /* Generate output according to requested method */
+ if (repetition)
+ {
+ if (input_range)
+ i = write_random_numbers (randint_source, head_lines,
+ lo_input, hi_input, eolbyte);
+ else
+ i = write_random_lines (randint_source, head_lines, line, n_lines);
+ }
else
- i = write_permuted_lines (head_lines, line, permutation);
+ {
+ if (use_reservoir_sampling)
+ i = write_permuted_output_reservoir (n_lines, reservoir, permutation);
+ else if (input_range)
+ i = write_permuted_numbers (head_lines, lo_input,
+ permutation, eolbyte);
+ else
+ i = write_permuted_lines (head_lines, line, permutation);
+ }
if (i != 0)
error (EXIT_FAILURE, errno, _("write error"));
--
1.8.3.2
>From 0f09af3214eeb2551420c4307c2e6f3a704e61b8 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 14:55:17 -0600
Subject: [PATCH 13/14] shuf: code clean-up
* src/shuf.c: main() re-organize argument processing.
---
src/shuf.c | 35 ++++++++++++++++-------------------
1 file changed, 16 insertions(+), 19 deletions(-)
diff --git a/src/shuf.c b/src/shuf.c
index 15748a8..bfcb06d 100644
--- a/src/shuf.c
+++ b/src/shuf.c
@@ -497,41 +497,38 @@ main (int argc, char **argv)
n_operands = argc - optind;
operand = argv + optind;
+ /* Check invalid usage */
+ if (echo && input_range)
+ {
+ error (0, 0, _("cannot combine -e and -i options"));
+ usage (EXIT_FAILURE);
+ }
+ if ( (n_operands>0 && input_range)
+ || (!echo && !input_range && n_operands>=2))
+ {
+ error (0, 0, _("extra operand %s"), quote (operand[1]));
+ usage (EXIT_FAILURE);
+ }
+
+ /* Prepare input */
if (echo)
{
- if (input_range)
- error (EXIT_FAILURE, 0, _("cannot combine -e and -i options"));
input_from_argv (operand, n_operands, eolbyte);
n_lines = n_operands;
line = operand;
}
else if (input_range)
{
- if (n_operands)
- {
- error (0, 0, _("extra operand %s"), quote (operand[0]));
- usage (EXIT_FAILURE);
- }
n_lines = hi_input - lo_input + 1;
line = NULL;
}
else
{
- switch (n_operands)
- {
- case 0:
- break;
-
- case 1:
+ /* Input file specified, re-open it as STDIN */
+ if (n_operands==1)
if (! (STREQ (operand[0], "-") || ! head_lines
|| freopen (operand[0], "r", stdin)))
error (EXIT_FAILURE, errno, "%s", operand[0]);
- break;
-
- default:
- error (0, 0, _("extra operand %s"), quote (operand[1]));
- usage (EXIT_FAILURE);
- }
fadvise (stdin, FADVISE_SEQUENTIAL);
--
1.8.3.2
>From 5ba2828e72f6d276fc349f69824cd6cb626053a4 Mon Sep 17 00:00:00 2001
From: Assaf Gordon
Date: Fri, 5 Jul 2013 14:59:44 -0600
Subject: [PATCH 14/14] shuf: add tests for more erroneous usage
* test/misc/shuf.sh: test multiple '-o' and '--random-source'.
---
tests/misc/shuf.sh | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/tests/misc/shuf.sh b/tests/misc/shuf.sh
index 5c70a73..ecdfba7 100755
--- a/tests/misc/shuf.sh
+++ b/tests/misc/shuf.sh
@@ -87,6 +87,12 @@ do
shuf -i$ARG &&
{ fail=1; echo "shuf did not detect erroneous -i$ARG usage.">&2 ; }
done
+# multiple -o are forbidden
+shuf -i0-9 -o A -o B &&
+ { fail=1; echo "shuf did not detect erroneous multiple -o usage.">&2 ; }
+# multiple random-sources are forbidden
+shuf -i0-9 --random-source A --random-source B &&
+ { fail=1; echo "shuf did not detect multiple --random-source usage.">&2 ; }
# Test --repetition option
--
1.8.3.2