parallel: added --rrs with test. Passes testsuite.

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.67 for parallel 20110122.
+# Generated by GNU Autoconf 2.67 for parallel 20110126.
 #
 # Report bugs to <bug-parallel@gnu.org>.
 #
@@ -551,8 +551,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='parallel'
 PACKAGE_TARNAME='parallel'
-PACKAGE_VERSION='20110122'
+PACKAGE_VERSION='20110126'
-PACKAGE_STRING='parallel 20110122'
+PACKAGE_STRING='parallel 20110126'
 PACKAGE_BUGREPORT='bug-parallel@gnu.org'
 PACKAGE_URL=''
 
@@ -1168,7 +1168,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures parallel 20110122 to adapt to many kinds of systems.
+\`configure' configures parallel 20110126 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1234,7 +1234,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of parallel 20110122:";;
+     short | recursive ) echo "Configuration of parallel 20110126:";;
    esac
   cat <<\_ACEOF
 
@@ -1301,7 +1301,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-parallel configure 20110122
+parallel configure 20110126
 generated by GNU Autoconf 2.67
 
 Copyright (C) 2010 Free Software Foundation, Inc.
@@ -1318,7 +1318,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by parallel $as_me 20110122, which was
+It was created by parallel $as_me 20110126, which was
 generated by GNU Autoconf 2.67.  Invocation command line was
 
   $ $0 $@
@@ -2133,7 +2133,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='parallel'
- VERSION='20110122'
+ VERSION='20110126'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2684,7 +2684,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by parallel $as_me 20110122, which was
+This file was extended by parallel $as_me 20110126, which was
 generated by GNU Autoconf 2.67.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2746,7 +2746,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-parallel config.status 20110122
+parallel config.status 20110126
 configured by $0, generated by GNU Autoconf 2.67,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20110122], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20110126], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/FUTURE_IDEAS

@@ -1,3 +1,17 @@
+--remove-rec-sep
+test alternation with --recend 'a|b'
+echo 12a34a45a6 | src/parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo'
+echo 12a34b45a6 | src/parallel -k --pipe --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo'
+echo 12a34b45a6 | src/parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo'
+
+echo 12a34a45a6 | src/parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo'
+echo 12a34b45a6 | src/parallel -k --pipe --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo'
+echo 12a34b45a6 | src/parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo'
+
+Example:
+Chop mbox into emails
+Parallel sort
+
 codecoverage
 
 Testsuite: sem without ~/.parallel

src/niceload

@@ -236,7 +236,7 @@ B<parallel>(1), B<nice>(1)
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20110122;
+$Global::version = 20110126;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($::opt_version) {

src/parallel

@@ -106,6 +106,9 @@ sub spreadstdin {
 	$recend = $::opt_recend;
 	$recerror = "Warning: --recend unmatched. Is --blocksize too small?";
     }
+    # If $recstart/$recend contains '|' this should only apply to the regexp
+    $recstart = "(?:".$recstart.")";
+    $recend = "(?:".$recend.")";
 
     while(read(STDIN,$buf,$::opt_blocksize)) {
 	$record = $partial.$buf;
@@ -114,6 +117,12 @@ sub spreadstdin {
 	    while($record =~ s/(($recstart.*?$recend){$Global::max_number_of_args})($recstart.*)$/$1/os) {
 		$partial = $3;
 		::debug("Read record: ".length($record)."\n");
+		if($::opt_remove_rec_sep) {
+		    # Remove record separator
+		    $record =~ s/^$recstart//os;
+		    $record =~ s/$recend$//os;
+		    $record =~ s/$recstart$recend//gos;
+		}
 		write_record_to_pipe($record);
 		$record = $partial;
 	    }
@@ -127,6 +136,12 @@ sub spreadstdin {
 	    } else {
 		print $Global::original_stderr $recerror,"\n";
 	    }
+	    if($::opt_remove_rec_sep) {
+		# Remove record separator
+		$record =~ s/^$recstart//os;
+		$record =~ s/$recend$//os;
+		$record =~ s/$recstart$recend//gos;
+	    }
 	    ::debug("Read record: ".length($record)."\n");
 	    write_record_to_pipe($record);
 	    if(eof STDIN) {
@@ -289,6 +304,7 @@ sub get_options_from_array {
 	 "pipe|spreadstdin" => \$::opt_pipe,
 	 "recstart=s" => \$::opt_recstart,
 	 "recend=s" => \$::opt_recend,
+	 "remove-rec-sep|removerecsep|rrs" => \$::opt_remove_rec_sep,
 	 "files|output-as-files|outputasfiles" => \$::opt_files,
 	 "block|block-size|blocksize=s" => \$::opt_blocksize,
 	 # xargs-compatibility - implemented, man, testsuite
@@ -332,7 +348,7 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20110122;
+    $Global::version = 20110126;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;

src/parallel.pod

@@ -724,6 +724,17 @@ defaults to '\n'. To have no record separator use B<--recend "">.
 B<--recstart> and B<--recend> are used with B<--pipe>.
 
 
+=item B<--remove-rec-sep> (alpha testing)
+
+=item B<--removerecsep> (alpha testing)
+
+=item B<--rrs> (alpha testing)
+
+Remove the text matched by B<--recstart> and B<--recend> before piping
+it to the command.
+
+Only used with B<--pipe>.
+
 =item B<--retries> I<n> (beta testing)
 
 If a job fails, retry it on another computer. Do this I<n> times. If

src/sql

@@ -531,7 +531,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20110122;
+    $Global::version = 20110126;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/test15.sh

@@ -32,32 +32,41 @@ expect "opt--interactive 3"
 _EOF
 
 echo '### Test -L -l and --max-lines'
-(echo a_b;echo c) | parallel -km -L2  echo
+(echo a_b;echo c) | parallel -km -L2 echo
-(echo a_b;echo c) | parallel -k -L2  echo
+(echo a_b;echo c) | parallel -k -L2 echo
-(echo a_b;echo c) | xargs -L2  echo
+(echo a_b;echo c) | xargs -L2 echo
-(echo a_b;echo c) | parallel -km -L1  echo
+echo '### xargs -L1 echo'
-(echo a_b;echo c) | parallel -k -L1  echo
+(echo a_b;echo c) | parallel -km -L1 echo
-(echo a_b;echo c) | xargs -L1  echo
+(echo a_b;echo c) | parallel -k -L1 echo
-(echo a_b' ';echo c;echo d) | parallel -km -L1  echo
+(echo a_b;echo c) | xargs -L1 echo
-(echo a_b' ';echo c;echo d) | parallel -k -L1  echo
+echo 'Lines ending in space should continue on next line'
-(echo a_b' ';echo c;echo d) | xargs -L1  echo
+echo '### xargs -L1 echo'
+(echo a_b' ';echo c;echo d) | parallel -km -L1 echo
+(echo a_b' ';echo c;echo d) | parallel -k -L1 echo
+(echo a_b' ';echo c;echo d) | xargs -L1 echo
+echo '### xargs -L2 echo'
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo
-(echo a_b' ';echo c;echo d;echo e) | parallel -km -l echo
+echo '### xargs -l echo'
-(echo a_b' ';echo c;echo d;echo e) | parallel -k -l echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -km -l echo # This behaves wrong
+(echo a_b' ';echo c;echo d;echo e) | parallel -k -l echo # This behaves wrong
 (echo a_b' ';echo c;echo d;echo e) | xargs -l echo
+echo '### xargs -l2 echo'
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -k -l2 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo
+echo '### xargs -l1 echo'
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -k -l1 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -l1 echo
+echo '### xargs --max-lines=2 echo'
 (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo
-(echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines echo
+echo '### xargs --max-lines echo'
-(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines echo # This behaves wrong
+(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines echo # This behaves wrong
 (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo
 
 echo '### test too long args'

testsuite/tests-to-run/test27.sh

@@ -242,7 +242,8 @@ stdout parallel -k -l2 echo < files.xi
 echo '###  -s30 -t echo < stairs.xi - xargs'
 stdout xargs -s30 -t echo < stairs.xi
 echo '###  -s30 -t echo < stairs.xi - parallel'
-stdout parallel -k -X -s30 -t echo < stairs.xi
+echo 'Because of -t these lines can be flipped around therefore sort'
+stdout parallel -k -X -s30 -t echo < stairs.xi | sort 
 echo '###  -t echo this plus that < space.xi'
 stdout xargs -t echo this plus that < space.xi
 stdout parallel -k -t echo this plus that < space.xi

testsuite/tests-to-run/test51.sh

@@ -5,6 +5,26 @@ echo '### Test --pipe'
 seq 1 1000000 >/tmp/parallel-seq
 shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest
 
+echo '### Test --rrs -N1 --recend single'
+echo 12a34a45a6 |
+  parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
+echo '### Test --rrs -N1 --recend alternate'
+echo 12a34b45a6 |
+  parallel -k --pipe --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
+echo '### Test --rrs -N1 --recend single'
+echo 12a34b45a6 |
+  parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
+
+echo '### Test --rrs --recend single'
+echo 12a34a45a6 |
+  parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
+echo '### Test --rrs -N1 --recend alternate'
+echo 12a34b45a6 |
+  parallel -k --pipe --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
+echo '### Test --rrs -N1 --recend single'
+echo 12a34b45a6 |
+  parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
+
 echo '### Test -N even'
 seq 1 10 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 0.1"
 

testsuite/wanted-results/test15

@@ -15,24 +15,29 @@ a_b
 c
 a_b c
 a_b c
+### xargs -L1 echo
 a_b
 c
 a_b
 c
 a_b
 c
+Lines ending in space should continue on next line
+### xargs -L1 echo
 a_b c
 d
 a_b c
 d
 a_b c
 d
+### xargs -L2 echo
 a_b c d
 e
 a_b c d
 e
 a_b c d
 e
+### xargs -l echo
 a_b c
 d
 e
@@ -42,12 +47,14 @@ e
 a_b c
 d
 e
+### xargs -l2 echo
 a_b c d
 e
 a_b c d
 e
 a_b c d
 e
+### xargs -l1 echo
 a_b c
 d
 e
@@ -57,12 +64,14 @@ e
 a_b c
 d
 e
+### xargs --max-lines=2 echo
 a_b c d
 e
 a_b c d
 e
 a_b c d
 e
+### xargs --max-lines echo
 a_b c
 d
 e

testsuite/wanted-results/test27

@@ -1067,18 +1067,19 @@ echo 55555 666666 7777777
 echo 88888888 999999999 
 88888888 999999999
 ###  -s30 -t echo < stairs.xi - parallel
-echo 1 22 333 4444 55555
+Because of -t these lines can be flipped around therefore sort
-echo 666666 7777777 88888888
 1 22 333 4444 55555
-echo 999999999 1 22 333 4444
-666666 7777777 88888888
-echo 55555 666666 7777777
-999999999 1 22 333 4444
-echo 88888888
 55555 666666 7777777
-echo 999999999
+666666 7777777 88888888
 88888888
 999999999
+999999999 1 22 333 4444
+echo 1 22 333 4444 55555
+echo 55555 666666 7777777
+echo 666666 7777777 88888888
+echo 88888888
+echo 999999999
+echo 999999999 1 22 333 4444
 ###  -t echo this plus that < space.xi
 echo this plus that 
 this plus that

testsuite/wanted-results/test51

@@ -1,4 +1,32 @@
 ### Test --pipe
+### Test --rrs -N1 --recend single
+1>12
+2>34
+3>45
+4>6
+
+### Test --rrs -N1 --recend alternate
+1>12
+2>34
+3>45
+4>6
+
+### Test --rrs -N1 --recend single
+1>12a34
+2>45a6
+
+### Test --rrs --recend single
+1>123445
+
+2>6
+### Test --rrs -N1 --recend alternate
+1>123445
+
+2>6
+### Test --rrs -N1 --recend single
+1>12a34
+
+2>45a6
 ### Test -N even
 1
 2