parallel: --plain + man + test implemented. Passes testsuite.

This commit is contained in:
Ole Tange 2012-08-08 21:25:18 +02:00
parent 783d2d42f3
commit ec76d20b25
7 changed files with 127 additions and 78 deletions

15
README
View file

@ -24,9 +24,9 @@ you would get had you run the commands sequentially. This makes it
possible to use output from GNU Parallel as input for other programs.
= Installation =
= Full installation =
Installation of GNU Parallel is as simple as:
Full installation of GNU Parallel is as simple as:
./configure && make && make install
@ -38,10 +38,19 @@ If you are not root you can add ~/bin to your path and install in
Or if your system lacks 'make' you can simply copy src/parallel
src/sem src/niceload src/sql to a dir in your path.
If you just need parallel in a pinch:
= Minimal installation =
If you just need parallel and do not have 'make' installed (maybe the
system is old or Microsoft Windows):
wget http://git.savannah.gnu.org/cgit/parallel.git/plain/src/parallel
chmod 755 parallel
cp parallel sem
mv parallel sem dir-in-your-$PATH/bin/
= Test the installation =
After this you should be able to do:

View file

@ -509,6 +509,7 @@ sub options_hash {
"arg-sep|argsep=s" => \$::opt_arg_sep,
"arg-file-sep|argfilesep=s" => \$::opt_arg_file_sep,
"trim=s" => \$::opt_trim,
"plain" => \$::opt_plain,
"profile|J=s" => \@::opt_profile,
"pipe|spreadstdin" => \$::opt_pipe,
"recstart=s" => \$::opt_recstart,
@ -689,7 +690,7 @@ sub parse_options {
wait_and_exit(0);
}
}
if($::opt_tollef and not $::opt_gnu) {
if($::opt_tollef and not $::opt_gnu and not $::opt_plain) {
# Behave like tollef parallel (from moreutils)
$::opt_u = 1;
$Global::grouped = 0;
@ -875,44 +876,49 @@ sub read_options {
Getopt::Long::Configure("bundling","pass_through");
# Check if there is a --profile to set @::opt_profile
GetOptions("profile|J=s" => \@::opt_profile) || die_usage();
# Add options from .parallel/config and other profiles
GetOptions("profile|J=s" => \@::opt_profile,
"plain" => \$::opt_plain) || die_usage();
my @ARGV_profile = ();
my @ARGV_env = ();
my @config_profiles = (
"/etc/parallel/config",
$ENV{'HOME'}."/.parallel/config",
$ENV{'HOME'}."/.parallelrc");
my @profiles = @config_profiles;
if(@::opt_profile) {
# --profile overrides default profiles
@profiles = ();
for my $profile (@::opt_profile) {
push @profiles, $ENV{'HOME'}."/.parallel/".$profile;
if(not $::opt_plain) {
# Add options from .parallel/config and other profiles
my @config_profiles = (
"/etc/parallel/config",
$ENV{'HOME'}."/.parallel/config",
$ENV{'HOME'}."/.parallelrc");
my @profiles = @config_profiles;
if(@::opt_profile) {
# --profile overrides default profiles
@profiles = ();
for my $profile (@::opt_profile) {
push @profiles, $ENV{'HOME'}."/.parallel/".$profile;
}
}
for my $profile (@profiles) {
if(-r $profile) {
open (IN, "<", $profile) || ::die_bug("read-profile: $profile");
while(<IN>) {
/^\s*\#/ and next;
chomp;
push @ARGV_profile, shell_unquote(split/(?<![\\])\s/, $_);
}
close IN;
} else {
if(grep /^$profile$/, @config_profiles) {
# config file is not required to exist
} else {
::error("$profile not readable.\n");
wait_and_exit(255);
}
}
}
# Add options from shell variable $PARALLEL
if($ENV{'PARALLEL'}) {
@ARGV_env = shell_unquote(split/(?<![\\])\s/, $ENV{'PARALLEL'});
}
}
for my $profile (@profiles) {
if(-r $profile) {
open (IN, "<", $profile) || ::die_bug("read-profile: $profile");
while(<IN>) {
/^\s*\#/ and next;
chomp;
push @ARGV_profile, shell_unquote(split/(?<![\\])\s/, $_);
}
close IN;
} else {
if(grep /^$profile$/, @config_profiles) {
# config file is not required to exist
} else {
::error("$profile not readable.\n");
wait_and_exit(255);
}
}
}
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV_profile) || die_usage();
# Add options from shell variable $PARALLEL
$ENV{'PARALLEL'} and @ARGV_env = shell_unquote(split/(?<![\\])\s/, $ENV{'PARALLEL'});
get_options_from_array(\@ARGV_env) || die_usage();
get_options_from_array(\@ARGV) || die_usage();

View file

@ -462,7 +462,7 @@ See also: B<--bg>, B<man sem>
Implies B<--semaphore>.
=item B<--filter-hosts> (beta testing)
=item B<--filter-hosts>
Remove down hosts. For each remote host: check that login through ssh
works. If not: do not use this host.
@ -549,7 +549,7 @@ specified, and for B<-I>{} otherwise. This option is deprecated;
use B<-I> instead.
=item B<--joblog> I<logfile> (beta testing)
=item B<--joblog> I<logfile>
Logfile for executed jobs. Save a list of the executed jobs to
I<logfile> in the following TAB separated format: sequence number,
@ -645,7 +645,7 @@ to see the difference:
parallel -j4 sleep {}\; echo {} ::: 2 1 4 3
parallel -j4 -k sleep {}\; echo {} ::: 2 1 4 3
=item B<-L> I<max-lines> (beta testing)
=item B<-L> I<max-lines>
When used with B<--pipe>: Read records of I<max-lines>.
@ -675,7 +675,7 @@ B<-l 0> is an alias for B<-l 1>.
Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
=item B<--load> I<max-load> (beta testing)
=item B<--load> I<max-load>
Do not start new jobs on a given computer unless the load is less than
I<max-load>. I<max-load> uses the same syntax as B<--jobs>, so I<100%>
@ -770,9 +770,9 @@ Instead of printing the output to stdout (standard output) the output
of each job is saved in a file and the filename is then printed.
=item B<--pipe> (beta testing)
=item B<--pipe>
=item B<--spreadstdin> (beta testing)
=item B<--spreadstdin>
Spread input to jobs on stdin (standard input). Read a block of data
from stdin (standard input) and give one block of data as input to one
@ -797,11 +797,12 @@ defaults to '\n'. To have no record separator use B<--recend "">.
B<--files> is often used with B<--pipe>.
=item B<--plain> (not implemented)
=item B<--plain> (alpha testing)
Ignore any B<--profile>, $PARALLEL, ~/.parallel/config, and
B<--tollef> to get full control on the command line (used by GNU
B<parallel> internally when called with B<--sshlogin>).
Ignore any B<--profile>, $PARALLEL, ~/.parallel/config, B<--tollef> to
get full control on the command line (used by GNU B<parallel> itself
when called with B<--sshlogin>).
=item B<--progress>
@ -1238,14 +1239,14 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
setting $TMPDIR.
=item B<--timeout> I<sec> (beta testing)
=item B<--timeout> I<sec>
Time out for command. If the command runs for longer than I<sec>
seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms
later, followed by SIGKILL 200 ms later.
=item B<--tollef> (beta testing)
=item B<--tollef>
Make GNU B<parallel> behave more like Tollef's parallel command. It
activates B<-u>, B<-q>, and B<--arg-sep -->. It also causes B<-l> to
@ -1451,9 +1452,9 @@ Compare these two:
See also B<--header>.
=item B<--shebang> (beta testing)
=item B<--shebang>
=item B<--hashbang> (beta testing)
=item B<--hashbang>
GNU B<Parallel> can be called as a shebang (#!) command as the first line of a script. Like this:

View file

@ -489,8 +489,8 @@ See also: @strong{--bg}, @strong{man sem}
Implies @strong{--semaphore}.
@item @strong{--filter-hosts} (beta testing)
@anchor{@strong{--filter-hosts} (beta testing)}
@item @strong{--filter-hosts}
@anchor{@strong{--filter-hosts}}
Remove down hosts. For each remote host: check that login through ssh
works. If not: do not use this host.
@ -581,8 +581,8 @@ This option is a synonym for @strong{-I}@emph{replace-str} if @emph{replace-str}
specified, and for @strong{-I}@{@} otherwise. This option is deprecated;
use @strong{-I} instead.
@item @strong{--joblog} @emph{logfile} (beta testing)
@anchor{@strong{--joblog} @emph{logfile} (beta testing)}
@item @strong{--joblog} @emph{logfile}
@anchor{@strong{--joblog} @emph{logfile}}
Logfile for executed jobs. Save a list of the executed jobs to
@emph{logfile} in the following TAB separated format: sequence number,
@ -696,8 +696,8 @@ to see the difference:
parallel -j4 -k sleep {}\; echo {} ::: 2 1 4 3
@end verbatim
@item @strong{-L} @emph{max-lines} (beta testing)
@anchor{@strong{-L} @emph{max-lines} (beta testing)}
@item @strong{-L} @emph{max-lines}
@anchor{@strong{-L} @emph{max-lines}}
When used with @strong{--pipe}: Read records of @emph{max-lines}.
@ -727,8 +727,8 @@ standard specifies @strong{-L} instead.
Implies @strong{-X} unless @strong{-m}, @strong{--xargs}, or @strong{--pipe} is set.
@item @strong{--load} @emph{max-load} (beta testing)
@anchor{@strong{--load} @emph{max-load} (beta testing)}
@item @strong{--load} @emph{max-load}
@anchor{@strong{--load} @emph{max-load}}
Do not start new jobs on a given computer unless the load is less than
@emph{max-load}. @emph{max-load} uses the same syntax as @strong{--jobs}, so @emph{100%}
@ -825,11 +825,11 @@ all the output from one server will be grouped together.
Instead of printing the output to stdout (standard output) the output
of each job is saved in a file and the filename is then printed.
@item @strong{--pipe} (beta testing)
@anchor{@strong{--pipe} (beta testing)}
@item @strong{--pipe}
@anchor{@strong{--pipe}}
@item @strong{--spreadstdin} (beta testing)
@anchor{@strong{--spreadstdin} (beta testing)}
@item @strong{--spreadstdin}
@anchor{@strong{--spreadstdin}}
Spread input to jobs on stdin (standard input). Read a block of data
from stdin (standard input) and give one block of data as input to one
@ -853,12 +853,12 @@ defaults to '\n'. To have no record separator use @strong{--recend ""}.
@strong{--files} is often used with @strong{--pipe}.
@item @strong{--plain} (not implemented)
@anchor{@strong{--plain} (not implemented)}
@item @strong{--plain} (alpha testing)
@anchor{@strong{--plain} (alpha testing)}
Ignore any @strong{--profile}, $PARALLEL, ~/.parallel/config, @strong{--tollef} to
get full control on the command line (used by GNU @strong{parallel} itself
when called with @strong{--sshlogin}).
Ignore any @strong{--profile}, $PARALLEL, ~/.parallel/config, and
@strong{--tollef} to get full control on the command line (used by GNU
@strong{parallel} internally when called with @strong{--sshlogin}).
@item @strong{--progress}
@anchor{@strong{--progress}}
@ -1318,15 +1318,15 @@ into temporary files in /tmp. By setting @strong{--tmpdir} you can use a
different dir for the files. Setting @strong{--tmpdir} is equivalent to
setting $TMPDIR.
@item @strong{--timeout} @emph{sec} (beta testing)
@anchor{@strong{--timeout} @emph{sec} (beta testing)}
@item @strong{--timeout} @emph{sec}
@anchor{@strong{--timeout} @emph{sec}}
Time out for command. If the command runs for longer than @emph{sec}
seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms
later, followed by SIGKILL 200 ms later.
@item @strong{--tollef} (beta testing)
@anchor{@strong{--tollef} (beta testing)}
@item @strong{--tollef}
@anchor{@strong{--tollef}}
Make GNU @strong{parallel} behave more like Tollef's parallel command. It
activates @strong{-u}, @strong{-q}, and @strong{--arg-sep --}. It also causes @strong{-l} to
@ -1547,11 +1547,11 @@ Compare these two:
See also @strong{--header}.
@item @strong{--shebang} (beta testing)
@anchor{@strong{--shebang} (beta testing)}
@item @strong{--shebang}
@anchor{@strong{--shebang}}
@item @strong{--hashbang} (beta testing)
@anchor{@strong{--hashbang} (beta testing)}
@item @strong{--hashbang}
@anchor{@strong{--hashbang}}
GNU @strong{Parallel} can be called as a shebang (#!) command as the first line of a script. Like this:

View file

@ -1,12 +1,12 @@
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereq
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
time sh Start.sh
date
local: ../src/parallel tests-to-run/*local* wanted-results/*local* startdb prereq
local: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal
time sh Start.sh local
date
prereq: installparallel
prereqlocal: installparallel
echo | mop || (echo mop is required for testsuite; /bin/false)
seq 1 2 | mop || (echo seq is required for testsuite; /bin/false)
stdout echo || (echo stdout is required for testsuite; /bin/false)
@ -21,6 +21,8 @@ prereq: installparallel
niceload true || (echo niceload is required for testsuite; /bin/false)
which burnP6 || (echo burnP6 is required for testsuite; /bin/false)
which timeout || (echo timeout is required for testsuite; /bin/false)
prereqremote: installparallel
parallel ssh parallel@parallel-server{} parallel --minversion 20100101 ::: 1 2 3 || (echo parallel on remote required for testsuite; /bin/true)
installparallel: ../src/parallel

View file

@ -36,6 +36,9 @@ cat <<EOF >~/.parallel/test_profile
EOF
parallel -v -J test_profile ::: <(echo a) <(echo b)
echo '### Test ugly quoting from profile file --plain'
parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b)
PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 -v ::: foo
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo
@ -45,15 +48,24 @@ cat <<EOF >~/.parallel/test_profile
EOF
parallel -v -J test_profile -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 '<(echo {})' ::: foo
echo '### Test quoting of $ in command from profile file --plain'
parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 'cat <(echo {})' ::: foo
echo '### Test quoting of $ in command from $PARALLEL'
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo
echo '### Test quoting of $ in command from $PARALLEL --plain'
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from profile file'
cat <<EOF >~/.parallel/test_profile
-k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe '\$a=1; print \$a'
EOF
parallel -v -J test_profile '<(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from profile file --plain'
parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from $PARALLEL'
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe "\\$a=1; print \\$a" ' parallel -v '<(echo {})' ::: foo
@ -73,3 +85,6 @@ echo -S .. > ~/.parallel/test_S..
echo parallel@parallel-server1 > ~/.parallel/sshloginfile
echo parallel@parallel-server2 >> ~/.parallel/sshloginfile
parallel -Jtest_tag -Jtest_S.. --nonall echo a | sort
echo '### Test merging of profiles - sort needed because -k only works on the single machine --plain'
parallel --plain -Jtest_tag -Jtest_S.. --nonall echo a | sort

View file

@ -35,6 +35,11 @@ perl -pe '$a=1; print $a' /dev/fd/63
1a
perl -pe '$a=1; print $a' /dev/fd/62
1b
### Test ugly quoting from profile file --plain
echo /dev/fd/63
/dev/fd/63
echo /dev/fd/62
/dev/fd/62
echo foo
foo
ssh parallel@parallel-server3 ssh parallel-server2 'eval `echo $SHELL | grep -E "/(t)?csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' perl\ -pe\ \"\\\$a=1\;\ print\ \\\$a\"\ \<\(echo\ foo\);
@ -42,12 +47,21 @@ ssh parallel@parallel-server3 ssh parallel-server2 'eval `echo $SHELL | grep -E
### Test quoting of $ in command from profile file
perl -pe '$a=1; print $a' <(echo foo)
1foo
### Test quoting of $ in command from profile file --plain
cat <(echo foo)
foo
### Test quoting of $ in command from $PARALLEL
perl -pe "\$a=1; print \$a" <(echo foo)
1foo
### Test quoting of $ in command from $PARALLEL --plain
cat <(echo foo)
foo
### Test quoting of space in arguments (-S) from profile file
perl -pe '$a=1; print $a' <(echo foo)
1foo
### Test quoting of space in arguments (-S) from profile file --plain
cat <(echo foo)
foo
### Test quoting of space in arguments (-S) from $PARALLEL
perl -pe "\$a=1; print \$a" <(echo foo)
1foo
@ -60,3 +74,5 @@ perl -pe "\$a=1; print \$a" <(echo foo)
### Test merging of profiles - sort needed because -k only works on the single machine
parallel@parallel-server1 a
parallel@parallel-server2 a
### Test merging of profiles - sort needed because -k only works on the single machine --plain
a