parallel: Implemented --gnu and --tollef. Passes at least some tests.

This commit is contained in:
Ole Tange 2011-02-18 15:23:00 +01:00
parent 9cc892faf2
commit b7d707b346
8 changed files with 70 additions and 27 deletions

View file

@ -10,8 +10,6 @@ use Getopt::Long;
use strict; use strict;
use Carp; use Carp;
@ARGV and exit;
$::oodebug=0; $::oodebug=0;
$Global::original_sigterm = $SIG{TERM} || sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X $Global::original_sigterm = $SIG{TERM} || sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X
$SIG{TERM} = sub {}; # Dummy until jobs really start $SIG{TERM} = sub {}; # Dummy until jobs really start
@ -339,6 +337,8 @@ sub get_options_from_array {
"remove-rec-sep|removerecsep|rrs" => \$::opt_remove_rec_sep, "remove-rec-sep|removerecsep|rrs" => \$::opt_remove_rec_sep,
"files|output-as-files|outputasfiles" => \$::opt_files, "files|output-as-files|outputasfiles" => \$::opt_files,
"block|block-size|blocksize=s" => \$::opt_blocksize, "block|block-size|blocksize=s" => \$::opt_blocksize,
"tollef" => \$::opt_tollef,
"gnu" => \$::opt_gnu,
# xargs-compatibility - implemented, man, testsuite # xargs-compatibility - implemented, man, testsuite
"max-procs|P=s" => \$::opt_P, "max-procs|P=s" => \$::opt_P,
"delimiter|d=s" => \$::opt_d, "delimiter|d=s" => \$::opt_d,
@ -353,7 +353,7 @@ sub get_options_from_array {
"colsep|col-sep|C=s" => \$::opt_colsep, "colsep|col-sep|C=s" => \$::opt_colsep,
"help|h" => \$::opt_help, "help|h" => \$::opt_help,
"L=i" => \$::opt_L, "L=i" => \$::opt_L,
"max-lines|l:i" => \$::opt_l, "max-lines|l:s" => \$::opt_l,
"interactive|p" => \$::opt_p, "interactive|p" => \$::opt_p,
"verbose|t" => \$::opt_verbose, "verbose|t" => \$::opt_verbose,
"version|V" => \$::opt_version, "version|V" => \$::opt_version,
@ -380,7 +380,7 @@ sub get_options_from_array {
sub parse_options { sub parse_options {
# Returns: N/A # Returns: N/A
# Defaults: # Defaults:
$Global::version = 20110205; $Global::version = 20110218;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -473,6 +473,16 @@ sub parse_options {
$::opt_transfer = 1; $::opt_transfer = 1;
$::opt_cleanup = 1; $::opt_cleanup = 1;
} }
if($::opt_tollef and not $::opt_gnu) {
# Behave like tollef parallel (from moreutils)
if(defined $::opt_l) {
$::opt_load = $::opt_l;
$::opt_l = undef;
}
if(not defined $::opt_arg_sep) {
$Global::arg_sep = "--";
}
}
if(defined $::opt_L and $::opt_L or defined $::opt_l) { if(defined $::opt_L and $::opt_L or defined $::opt_l) {
$Global::max_lines = $::opt_l || $::opt_L || 1; $Global::max_lines = $::opt_l || $::opt_L || 1;
$Global::max_number_of_args ||= $Global::max_lines; $Global::max_number_of_args ||= $Global::max_lines;
@ -1490,7 +1500,7 @@ sub version {
# Returns: N/A # Returns: N/A
print join("\n", print join("\n",
"GNU $Global::progname $Global::version", "GNU $Global::progname $Global::version",
"Copyright (C) 2007,2008,2009,2010 Ole Tange and Free Software Foundation, Inc.", "Copyright (C) 2007,2008,2009,2010,2011 Ole Tange and Free Software Foundation, Inc.",
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>", "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
"This is free software: you are free to change and redistribute it.", "This is free software: you are free to change and redistribute it.",
"GNU $Global::progname comes with no warranty.", "GNU $Global::progname comes with no warranty.",

View file

@ -353,6 +353,12 @@ See also: B<--bg>
Implies B<--semaphore>. Implies B<--semaphore>.
=item B<--gnu>
Behave like GNU B<parallel>. If B<--tollef> and B<--gnu> are both set,
B<--gnu> takes precedence.
=item B<--group> =item B<--group>
=item B<-g> =item B<-g>
@ -361,6 +367,7 @@ Group output. Output from each jobs is grouped together and is only
printed when the command is finished. STDERR first followed by STDOUT. printed when the command is finished. STDERR first followed by STDOUT.
B<-g> is the default. Can be reversed with B<-u>. B<-g> is the default. Can be reversed with B<-u>.
=item B<--help> =item B<--help>
=item B<-h> =item B<-h>
@ -934,6 +941,12 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
setting $TMPDIR. setting $TMPDIR.
=item B<--tollef>
Make GNU B<parallel> behave like Tollef's parallel command. To
override use B<--gnu>.
=item B<--verbose> =item B<--verbose>
=item B<-t> =item B<-t>

View file

@ -49,8 +49,8 @@ echo '### xargs -L2 echo'
(echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo
(echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo (echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo
echo '### xargs -l echo' echo '### xargs -l echo'
(echo a_b' ';echo c;echo d;echo e) | parallel -km -l echo # This behaves wrong (echo a_b' ';echo c;echo d;echo e) | parallel -l -km echo # This behaves wrong
(echo a_b' ';echo c;echo d;echo e) | parallel -k -l echo # This behaves wrong (echo a_b' ';echo c;echo d;echo e) | parallel -l -k echo # This behaves wrong
(echo a_b' ';echo c;echo d;echo e) | xargs -l echo (echo a_b' ';echo c;echo d;echo e) | xargs -l echo
echo '### xargs -l2 echo' echo '### xargs -l2 echo'
(echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo
@ -65,8 +65,8 @@ echo '### xargs --max-lines=2 echo'
(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo
(echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo
echo '### xargs --max-lines echo' echo '### xargs --max-lines echo'
(echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines echo # This behaves wrong (echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -km echo # This behaves wrong
(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines echo # This behaves wrong (echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -k echo # This behaves wrong
(echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo
echo '### test too long args' echo '### test too long args'

View file

@ -229,10 +229,10 @@ stdout xargs -i__ echo FIRST __ IS OK < quotes.xi
stdout parallel -k -i__ echo FIRST __ IS OK < quotes.xi stdout parallel -k -i__ echo FIRST __ IS OK < quotes.xi
echo '### -l echo < ldatab.xi' echo '### -l echo < ldatab.xi'
stdout xargs -l echo < ldatab.xi stdout xargs -l echo < ldatab.xi
stdout parallel -k -l echo < ldatab.xi stdout parallel -l -k echo < ldatab.xi
echo '### -l echo < ldata.xi' echo '### -l echo < ldata.xi'
stdout xargs -l echo < ldata.xi stdout xargs -l echo < ldata.xi
stdout parallel -k -l echo < ldata.xi stdout parallel -l -k echo < ldata.xi
echo '### -l1 -n4 echo < files.xi' echo '### -l1 -n4 echo < files.xi'
stdout xargs -l1 -n4 echo < files.xi stdout xargs -l1 -n4 echo < files.xi
stdout parallel -k -l1 -n4 echo < files.xi stdout parallel -k -l1 -n4 echo < files.xi

View file

@ -6,16 +6,16 @@ seq 1 1000000 >/tmp/parallel-seq
shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest
echo '### Test -N with multiple jobslots and multiple args' echo '### Test -N with multiple jobslots and multiple args'
seq 1 1 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo a;sleep 0.1' seq 1 1 | parallel -j2 -k -N 3 --pipe 'cat;echo a;sleep 0.1'
seq 1 2 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo bb;sleep 0.1' seq 1 2 | parallel -j2 -k -N 3 --pipe 'cat;echo bb;sleep 0.1'
seq 1 3 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo ccc;sleep 0.1' seq 1 3 | parallel -j2 -k -N 3 --pipe 'cat;echo ccc;sleep 0.1'
seq 1 4 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo dddd;sleep 0.1' seq 1 4 | parallel -j2 -k -N 3 --pipe 'cat;echo dddd;sleep 0.1'
seq 1 5 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo eeeee;sleep 0.1' seq 1 5 | parallel -j2 -k -N 3 --pipe 'cat;echo eeeee;sleep 0.1'
seq 1 6 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo ffffff;sleep 0.1' seq 1 6 | parallel -j2 -k -N 3 --pipe 'cat;echo ffffff;sleep 0.1'
seq 1 7 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo ggggggg;sleep 0.1' seq 1 7 | parallel -j2 -k -N 3 --pipe 'cat;echo ggggggg;sleep 0.1'
seq 1 8 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo hhhhhhhh;sleep 0.1' seq 1 8 | parallel -j2 -k -N 3 --pipe 'cat;echo hhhhhhhh;sleep 0.1'
seq 1 9 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo iiiiiiiii;sleep 0.1' seq 1 9 | parallel -j2 -k -N 3 --pipe 'cat;echo iiiiiiiii;sleep 0.1'
seq 1 10 | ../src/parallel -j2 -k -N 3 --pipe 'cat;echo jjjjjjjjjj;sleep 0.1' seq 1 10 | parallel -j2 -k -N 3 --pipe 'cat;echo jjjjjjjjjj;sleep 0.1'
echo '### Test output is the same for different block size' echo '### Test output is the same for different block size'
echo -n 01a02a0a0a12a34a45a6a | echo -n 01a02a0a0a12a34a45a6a |
@ -38,8 +38,8 @@ echo '### Test 100M records with too big block'
echo start echo start
seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true
echo end echo end
) | stdout parallel -k --block 10M -j2 --pipe --recstart 'start\n' wc -c ) | stdout parallel -k --block 10M -j2 --pipe --recstart 'start\n' wc -c |
egrep -v '^0$'
echo '### Test 300M records with too small block' echo '### Test 300M records with too small block'
( (
@ -52,9 +52,8 @@ echo '### Test 300M records with too small block'
echo start echo start
seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true
echo end echo end
) | stdout parallel -k --block 200M -j2 --pipe --recend 'end\n' wc -c ) | stdout parallel -k --block 200M -j2 --pipe --recend 'end\n' wc -c |
egrep -v '^0$'
echo '### Test --rrs -N1 --recend single' echo '### Test --rrs -N1 --recend single'
echo 12a34a45a6 | echo 12a34a45a6 |

View file

@ -0,0 +1,10 @@
#!/bin/bash
echo '### Test --tollef'
parallel --tollef echo -- 1 2 3
echo '### Test --tollef --gnu'
parallel --tollef --gnu echo ::: 1 2 3
echo '### Test --gnu'
parallel --gnu echo ::: 1 2 3

View file

@ -94,7 +94,6 @@ jjjjjjjjjj
6888906 6888906
6888906 6888906
6888906 6888906
0
### Test 300M records with too small block ### Test 300M records with too small block
303111434 303111434
303111434 303111434

View file

@ -0,0 +1,12 @@
### Test --tollef
1
2
3
### Test --tollef --gnu
1
2
3
### Test --gnu
1
2
3