From f3d117e7bb28c2ead17dc80072b4897aa184166b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 22 Jun 2017 23:14:32 +0200 Subject: [PATCH] parallel: Fixed bug #51293: parallel does not preserve symlinked directory structure on remote. --- doc/release_new_version | 14 +- src/niceload | 2 +- src/parallel | 23 +- src/parallel.pod | 52 ++--- src/parset.pod | 3 +- src/sql | 2 +- testsuite/tests-to-run/parallel-local-ssh6.sh | 173 ++++++++------ testsuite/wanted-results/parallel-local-ssh6 | 215 ++++++++---------- 8 files changed, 246 insertions(+), 238 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 461f755d..625e01a5 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -4,7 +4,7 @@ Check that documentation is updated (compare to web): -Fixet for 20170322 +Fixet for 20170622 git diff last-release-commit Unmodified beta since last version => production Unmodified alpha since last version => beta @@ -214,20 +214,8 @@ New in this release: http://meta.askubuntu.com/a/16750/22307 http://meta.serverfault.com/a/9040/45704 -* Graph for overhead for 1000 jobs has been updated. It remains steadily at 3.5 ms per job on a 1.7 GHz machine https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf - -* GNU Parallel was cited in: The topology of the directed clique complex as a network invariant https://springerplus.springeropen.com/articles/10.1186/s40064-016-2022-y - * GNU Parallel was cited in: -* GNU Parallel was cited in: Parallel Execution of Structural Mechanic Tasks with Use of Small Supercomputers http://www.sciencedirect.com/science/article/pii/S1877705817324815 - -https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel - -https://pzelasko.github.io/posts/speed-up-ml-model-development-with-gnu-parallel - -http://tdhopper.com/blog/2017/Jun/07/parallelizing-a-python-function-for-the-extremely-lazy/ - <> <> diff --git a/src/niceload b/src/niceload index 2e88d032..3f76ead8 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20170622; +$Global::version = 20170623; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 3e616ba6..0db2d9fa 100755 --- a/src/parallel +++ b/src/parallel @@ -1,5 +1,10 @@ #!/usr/bin/env perl +# XDG_RUNTIME_DIR/parallel = TMPDIR +# XDG_CONFIG_HOME/parallel = ~/.parallel/config +# XDG_CONFIG_DIRS +# XDG_CACHE_HOME = ~/parallel/tmp + semaphores + # Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 # Ole Tange and Free Software Foundation, Inc. # @@ -1376,7 +1381,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170622; + $Global::version = 20170623; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3982,8 +3987,8 @@ sub usage { "See 'man $Global::progname' for details", "", "Academic tradition requires you to cite works you base your article on.", - "When using programs that use GNU Parallel to process data for publication", - "please cite:", + "If you use programs that use GNU Parallel to process data for an article in a", + "scientific publication, please cite:", "", " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", " ;login: The USENIX Magazine, February 2011:42-47.", @@ -4010,8 +4015,8 @@ sub citation_notice { } else { ::status ("Academic tradition requires you to cite works you base your article on.", - "When using programs that use GNU Parallel to process data for publication", - "please cite:", + "If you use programs that use GNU Parallel to process data for an article in a", + "scientific publication, please cite:", "", " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", " ;login: The USENIX Magazine, February 2011:42-47.", @@ -4085,8 +4090,8 @@ sub citation { # Returns: N/A ::status( "Academic tradition requires you to cite works you base your article on.", - "When using programs that use GNU Parallel to process data for publication", - "please cite:", + "If you use programs that use GNU Parallel to process data for an article in a", + "scientific publication, please cite:", "", "\@article{Tange2011a,", " title = {GNU Parallel - The Command-Line Power Tool},", @@ -4103,13 +4108,11 @@ sub citation { "}", "", "(Feel free to use \\nocite{Tange2011a})", - "", "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "", "If you send a copy of your published article to tange\@gnu.org, it will be", "mentioned in the release notes of next version of GNU Parallel.", - "", "" ); while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") { @@ -6405,7 +6408,7 @@ sub rsync_transfer_cmd { } $file = ::shell_quote_file($file); my $sshcmd = $self->sshcommand(); - my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd); + my $rsync_opt = "-KrlDzR -e" . ::shell_quote_scalar($sshcmd); my $serverlogin = $self->serverlogin(); # Make dir if it does not exist return "$sshcmd $serverlogin -- mkdir -p $rsync_destdir && " . diff --git a/src/parallel.pod b/src/parallel.pod index 657ee31b..6b0e89a0 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -95,7 +95,7 @@ B: Use B. The command cannot contain the character \257 (macron: ¯). -=item B<{}> (alpha testing) +=item B<{}> (beta testing) Input line. This replacement string will be replaced by a full line read from the input source. The input source is normally stdin @@ -112,7 +112,7 @@ parsed by the shell. The exception is if the command starts with a replacement string; then the string is not quoted. -=item B<{.}> (alpha testing) +=item B<{.}> (beta testing) Input line without extension. This replacement string will be replaced by the input with the extension removed. If the input line contains @@ -128,7 +128,7 @@ The replacement string B<{.}> can be changed with B<--er>. To understand replacement strings see B<{}>. -=item B<{/}> (alpha testing) +=item B<{/}> (beta testing) Basename of input line. This replacement string will be replaced by the input with the directory part removed. @@ -139,7 +139,7 @@ B<--basenamereplace>. To understand replacement strings see B<{}>. -=item B<{//}> (alpha testing) +=item B<{//}> (beta testing) Dirname of input line. This replacement string will be replaced by the dir of the input line. See B(1). @@ -150,7 +150,7 @@ B<--dirnamereplace>. To understand replacement strings see B<{}>. -=item B<{/.}> (alpha testing) +=item B<{/.}> (beta testing) Basename of input line without extension. This replacement string will be replaced by the input with the directory and extension part @@ -162,7 +162,7 @@ B<--basenameextensionreplace>. To understand replacement strings see B<{}>. -=item B<{#}> (alpha testing) +=item B<{#}> (beta testing) Sequence number of the job to run. This replacement string will be replaced by the sequence number of the job being run. It contains the @@ -173,7 +173,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>. To understand replacement strings see B<{}>. -=item B<{%}> (alpha testing) +=item B<{%}> (beta testing) Job slot number. This replacement string will be replaced by the job's slot number between 1 and number of jobs to run in parallel. There @@ -185,7 +185,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>. To understand replacement strings see B<{}>. -=item B<{>IB<}> (alpha testing) +=item B<{>IB<}> (beta testing) Argument from input source I or the I'th argument. This positional replacement string will be replaced by the input from input @@ -196,7 +196,7 @@ I'th last argument. To understand replacement strings see B<{}>. -=item B<{>I.B<}> (alpha testing) +=item B<{>I.B<}> (beta testing) Argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}> and B<{.}>. @@ -209,7 +209,7 @@ extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/B<}> (alpha testing) +=item B<{>I/B<}> (beta testing) Basename of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{/}>. @@ -222,7 +222,7 @@ directory (if any) removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I//B<}> (alpha testing) +=item B<{>I//B<}> (beta testing) Dirname of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{//}>. @@ -234,7 +234,7 @@ the I'th argument (when used with B<-N>). See B(1). To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/.B<}> (alpha testing) +=item B<{>I/.B<}> (beta testing) Basename of argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}>, B<{/}>, and @@ -248,7 +248,7 @@ directory (if any) and extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{=>IB<=}> (alpha testing) +=item B<{=>IB<=}> (beta testing) Replace with calculated I. B<$_> will contain the same as B<{}>. After evaluating I B<$_> will be used @@ -294,7 +294,7 @@ Example: See also: B<--rpl> B<--parens> -=item B<{=>I IB<=}> (alpha testing) +=item B<{=>I IB<=}> (beta testing) Positional equivalent to B<{=perl expression=}>. To understand positional replacement strings see B<{>IB<}>. @@ -341,7 +341,7 @@ B<:::> and B<::::> can be mixed. So these are equivalent: ::: 1 2 3 -=item B<:::+> I (beta testing) +=item B<:::+> I Like B<:::> but linked like B<--link> to the previous input source. @@ -362,7 +362,7 @@ B<:::> and B<::::> can be mixed. See B<-a>, B<:::> and B<--link>. -=item B<::::+> I (beta testing) +=item B<::::+> I Like B<::::> but linked like B<--link> to the previous input source. @@ -418,7 +418,7 @@ string that is not in the command line. See also: B<:::>. -=item B<--bar> (beta testing) +=item B<--bar> Show progress as a progress bar. In the bar is shown: % of jobs completed, estimated seconds left, and number of jobs started. @@ -656,7 +656,7 @@ variables, arrays, and functions) use B. See also: B<--record-env>. -=item B<--eta> (beta testing) +=item B<--eta> Show the estimated number of seconds before finishing. This forces GNU B to read all jobs before starting to find the number of @@ -744,9 +744,9 @@ See also: B<--line-buffer> B<--ungroup> Print a summary of the options to GNU B and exit. -=item B<--halt-on-error> I (beta testing) +=item B<--halt-on-error> I -=item B<--halt> I (beta testing) +=item B<--halt> I When should GNU B terminate? In some situations it makes no sense to run all jobs. GNU B should simply give up as soon @@ -891,7 +891,7 @@ specified, and for B<-I {}> otherwise. This option is deprecated; use B<-I> instead. -=item B<--joblog> I (beta testing) +=item B<--joblog> I Logfile for executed jobs. Save a list of the executed jobs to I in the following TAB separated format: sequence number, @@ -1076,7 +1076,7 @@ Arguments will be recycled if one input source has more arguments than the other See also B<--header>, B<:::+>, B<::::+>. -=item B<--load> I (alpha testing) +=item B<--load> I (beta testing) Do not start new jobs on a given computer unless the number of running processes on the computer is less than I. I uses @@ -1254,7 +1254,7 @@ control on the command line (used by GNU B internally when called with B<--sshlogin>). -=item B<--plus> (alpha testing) +=item B<--plus> (beta testing) Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} {...} {/..} {/...} {##}. The idea being that '{+foo}' matches the opposite of @@ -1767,7 +1767,7 @@ impossible to track which input block corresponds to which output. B<--round-robin> implies B<--pipe>, except if B<--pipepart> is given. -=item B<--rpl> 'I I' (alpha testing) +=item B<--rpl> 'I I' (beta testing) Use I as a replacement string for I. This makes it possible to define your own replacement strings. GNU B's @@ -2277,13 +2277,13 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--tmux> (beta testing) +=item B<--tmux> Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. -=item B<--tmuxpane> (beta testing) +=item B<--tmuxpane> Use B for output but put output into panes in the first window. Useful if you want to monitor the progress of less than 100 concurrent diff --git a/src/parset.pod b/src/parset.pod index aea353c4..a051e831 100644 --- a/src/parset.pod +++ b/src/parset.pod @@ -25,7 +25,8 @@ the destination variable and made to an array. If I contains multiple names separated by ',' or space, the names will be the destination variables. -B is not production ready and is beta quality. +B is beta quality and not production ready, but please use it +for everyday use and report bugs. =head1 OPTIONS diff --git a/src/sql b/src/sql index b5d66d7e..820cd1c9 100755 --- a/src/sql +++ b/src/sql @@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20170622; + $Global::version = 20170623; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-ssh6.sh b/testsuite/tests-to-run/parallel-local-ssh6.sh index 3420971f..54465a38 100644 --- a/testsuite/tests-to-run/parallel-local-ssh6.sh +++ b/testsuite/tests-to-run/parallel-local-ssh6.sh @@ -1,87 +1,126 @@ #!/bin/bash -SSHLOGIN1=parallel@lo -SSHLOGIN2=csh@lo +# At most 2 parallel sshs per function + +export SSHLOGIN1=parallel@lo +export SSHLOGIN2=csh@lo mkdir -p tmp -# -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj5 -k --joblog /tmp/jl-`basename $0` -L1 -echo '### Test --onall'; - parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 +par_test_onall() { + echo '### Test --onall' + parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 +} -echo '### Test | --onall'; - seq 3 | parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - +par_test_pipe_onall() { + echo '### Test | --onall' + seq 3 | + parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - +} -echo '### Test --onall -u'; - parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort +par_test_onall_u() { + echo '### Test --onall -u' + parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | + sort +} -echo '### Test --nonall'; - parallel --nonall -k -S $SSHLOGIN1,$SSHLOGIN2 pwd | sort +par_test_nonall() { + echo '### Test --nonall' + parallel --nonall -k -S $SSHLOGIN1,$SSHLOGIN2 pwd | + sort +} -echo '### Test --nonall -u - should be interleaved x y x y'; - parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;' +par_test_nonall_u() { + echo '### Test --nonall -u - should be interleaved x y x y' + parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;' +} -echo '### Test read sshloginfile from STDIN'; - echo $SSHLOGIN1,$SSHLOGIN2 | parallel -S - -k --nonall pwd; - echo $SSHLOGIN1,$SSHLOGIN2 | parallel --sshloginfile - -k --onall pwd\; echo ::: foo +par_read_sshloginfile_from_stdin() { + echo '### Test read sshloginfile from STDIN' + echo $SSHLOGIN1,$SSHLOGIN2 | parallel -S - -k --nonall pwd + echo $SSHLOGIN1,$SSHLOGIN2 | parallel --sshloginfile - -k --onall pwd\; echo ::: foo +} -echo '**' +par_nonall_basefile() { + echo '### Test --nonall --basefile' + touch tmp/nonall--basefile + stdout parallel --nonall --basefile tmp/nonall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile + stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile + stdout rm tmp/nonall--basefile +} -echo '### Test --nonall --basefile'; - touch tmp/nonall--basefile; - stdout parallel --nonall --basefile tmp/nonall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile; - stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile; - stdout rm tmp/nonall--basefile +par_onall_basefile() { + echo '### Test --onall --basefile' + touch tmp/onall--basefile + stdout parallel --onall --basefile tmp/onall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile + stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile + stdout rm tmp/onall--basefile +} -echo '**' +par_nonall_basefile_cleanup() { + echo '### Test --nonall --basefile --cleanup (rm should fail)' + touch tmp/nonall--basefile--clean + stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile--clean + stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile--clean + stdout rm tmp/nonall--basefile--clean +} -echo '### Test --onall --basefile'; - touch tmp/onall--basefile; - stdout parallel --onall --basefile tmp/onall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile; - stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile; - stdout rm tmp/onall--basefile +par_onall_basefile_cleanup() { + echo '### Test --onall --basefile --cleanup (rm should fail)' + touch tmp/onall--basefile--clean + stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile--clean + stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile--clean + stdout rm tmp/onall--basefile--clean +} -echo '**' +par_workdir_dot() { + echo '### Test --workdir .' + ssh $SSHLOGIN1 mkdir -p mydir + mkdir -p $HOME/mydir + cd $HOME/mydir + parallel --workdir . -S $SSHLOGIN1 ::: pwd +} -echo '### Test --nonall --basefile --cleanup (rm should fail)'; - touch tmp/nonall--basefile--clean; - stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile--clean; - stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile--clean; - stdout rm tmp/nonall--basefile--clean +par_wd_dot() { + echo '### Test --wd .' + ssh $SSHLOGIN2 mkdir -p mydir + mkdir -p $HOME/mydir + cd $HOME/mydir + parallel --workdir . -S $SSHLOGIN2 ::: pwd +} -echo '**' +par_wd_braces() { + echo '### Test --wd {}' + ssh $SSHLOGIN2 rm -rf wd1 wd2 + mkdir -p $HOME/mydir + cd $HOME/mydir + parallel --workdir {} -S $SSHLOGIN2 touch ::: wd1 wd2 + ssh $SSHLOGIN2 ls -d wd1 wd2 +} -echo '### Test --onall --basefile --cleanup (rm should fail)'; - touch tmp/onall--basefile--clean; - stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile--clean; - stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile--clean; - stdout rm tmp/onall--basefile--clean +par_wd_perlexpr() { + echo '### Test --wd {= =}' + ssh $SSHLOGIN2 rm -rf WD1 WD2 + mkdir -p $HOME/mydir + cd $HOME/mydir + parallel --workdir '{= $_=uc($_) =}' -S $SSHLOGIN2 touch ::: wd1 wd2 + ssh $SSHLOGIN2 ls -d WD1 WD2 +} -echo '**' +par_nonall_wd() { + echo '### Test --nonall --wd' + parallel --workdir /tmp -S $SSHLOGIN2 --nonall pwd +} -echo '### Test --workdir .'; - ssh $SSHLOGIN1 mkdir -p mydir; - mkdir -p $HOME/mydir; cd $HOME/mydir; - parallel --workdir . -S $SSHLOGIN1 ::: pwd +par_remote_symlink_dir() { + echo 'bug #51293: parallel does not preserve symlinked directory structure on remote' + ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd' + mkdir -p wd + touch wd/testfile + parallel --nonall -S parallel@lo --basefile wd/testfile + ssh parallel@lo rm wd && echo OK: wd is still a symlink +} -echo '### Test --wd .'; - ssh $SSHLOGIN2 mkdir -p mydir; - mkdir -p $HOME/mydir; cd $HOME/mydir; - parallel --workdir . -S $SSHLOGIN2 ::: pwd - -echo '### Test --wd {}'; - ssh $SSHLOGIN2 rm -rf wd1 wd2; - mkdir -p $HOME/mydir; cd $HOME/mydir; - parallel --workdir {} -S $SSHLOGIN2 touch ::: wd1 wd2; - ssh $SSHLOGIN2 ls -d wd1 wd2 - -echo '### Test --wd {= =}'; - ssh $SSHLOGIN2 rm -rf WD1 WD2; - mkdir -p $HOME/mydir; cd $HOME/mydir; - parallel --workdir '{= $_=uc($_) =}' -S $SSHLOGIN2 touch ::: wd1 wd2; - ssh $SSHLOGIN2 ls -d WD1 WD2 - -echo '### Test --nonall --wd'; - parallel --workdir /tmp -S $SSHLOGIN2 --nonall pwd - -EOF +export -f $(compgen -A function | grep par_) +#compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' +compgen -A function | grep par_ | sort | + parallel --joblog /tmp/jl-`basename $0` -j5 --tag -k '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-ssh6 b/testsuite/wanted-results/parallel-local-ssh6 index 0dafbd51..79e44b1a 100644 --- a/testsuite/wanted-results/parallel-local-ssh6 +++ b/testsuite/wanted-results/parallel-local-ssh6 @@ -1,119 +1,96 @@ -echo '### Test --onall'; parallel --onall --tag -k -S parallel@lo,csh@lo '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 -### Test --onall -csh@lo 1 -csh@lo 2 -csh@lo 1 -csh@lo 2 -csh@lo 1 -csh@lo 2 -parallel@lo 1 -parallel@lo 2 -parallel@lo 1 -parallel@lo 2 -parallel@lo 1 -parallel@lo 2 -echo '### Test | --onall'; seq 3 | parallel --onall --tag -k -S parallel@lo,csh@lo '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - -### Test | --onall -csh@lo 1 -csh@lo 2 -csh@lo 3 -csh@lo 1 -csh@lo 2 -csh@lo 3 -csh@lo 1 -csh@lo 2 -csh@lo 3 -parallel@lo 1 -parallel@lo 2 -parallel@lo 3 -parallel@lo 1 -parallel@lo 2 -parallel@lo 3 -parallel@lo 1 -parallel@lo 2 -parallel@lo 3 -echo '### Test --onall -u'; parallel --onall -S parallel@lo,csh@lo -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort -### Test --onall -u -1 -1 -1 -1 -1 -1 -2 -2 -2 -2 -2 -2 -3 -3 -3 -3 -3 -3 -echo '### Test --nonall'; parallel --nonall -k -S parallel@lo,csh@lo pwd | sort -### Test --nonall -/home/csh -/home/parallel -echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall -S parallel@lo,csh@lo -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;' -### Test --nonall -u - should be interleaved x y x y -/home/parallel -/home/csh -/home/parallel -/home/csh -echo '### Test read sshloginfile from STDIN'; echo parallel@lo,csh@lo | parallel -S - -k --nonall pwd; echo parallel@lo,csh@lo | parallel --sshloginfile - -k --onall pwd\; echo ::: foo -### Test read sshloginfile from STDIN -/home/csh -/home/parallel -/home/csh -foo -/home/parallel -foo -echo '**' -** -echo '### Test --nonall --basefile'; touch tmp/nonall--basefile; stdout parallel --nonall --basefile tmp/nonall--basefile -S parallel@lo,csh@lo ls tmp/nonall--basefile; stdout parallel --nonall -S parallel@lo,csh@lo rm tmp/nonall--basefile; stdout rm tmp/nonall--basefile -### Test --nonall --basefile -tmp/nonall--basefile -tmp/nonall--basefile -echo '**' -** -echo '### Test --onall --basefile'; touch tmp/onall--basefile; stdout parallel --onall --basefile tmp/onall--basefile -S parallel@lo,csh@lo ls {} ::: tmp/onall--basefile; stdout parallel --onall -S parallel@lo,csh@lo rm {} ::: tmp/onall--basefile; stdout rm tmp/onall--basefile -### Test --onall --basefile -tmp/onall--basefile -tmp/onall--basefile -echo '**' -** -echo '### Test --nonall --basefile --cleanup (rm should fail)'; touch tmp/nonall--basefile--clean; stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S parallel@lo,csh@lo ls tmp/nonall--basefile--clean; stdout parallel --nonall -S parallel@lo,csh@lo rm tmp/nonall--basefile--clean; stdout rm tmp/nonall--basefile--clean -### Test --nonall --basefile --cleanup (rm should fail) -tmp/nonall--basefile--clean -tmp/nonall--basefile--clean -rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory -rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory -echo '**' -** -echo '### Test --onall --basefile --cleanup (rm should fail)'; touch tmp/onall--basefile--clean; stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S parallel@lo,csh@lo ls {} ::: tmp/onall--basefile--clean; stdout parallel --onall -S parallel@lo,csh@lo rm {} ::: tmp/onall--basefile--clean; stdout rm tmp/onall--basefile--clean -### Test --onall --basefile --cleanup (rm should fail) -tmp/onall--basefile--clean -tmp/onall--basefile--clean -rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory -rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory -echo '**' -** -echo '### Test --workdir .'; ssh parallel@lo mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S parallel@lo ::: pwd -### Test --workdir . -/home/parallel/mydir -echo '### Test --wd .'; ssh csh@lo mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S csh@lo ::: pwd -### Test --wd . -/home/csh/mydir -echo '### Test --wd {}'; ssh csh@lo rm -rf wd1 wd2; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir {} -S csh@lo touch ::: wd1 wd2; ssh csh@lo ls -d wd1 wd2 -### Test --wd {} -wd1 -wd2 -echo '### Test --wd {= =}'; ssh csh@lo rm -rf WD1 WD2; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir '{= $_=uc($_) =}' -S csh@lo touch ::: wd1 wd2; ssh csh@lo ls -d WD1 WD2 -### Test --wd {= =} -WD1 -WD2 -echo '### Test --nonall --wd'; parallel --workdir /tmp -S csh@lo --nonall pwd -### Test --nonall --wd -/tmp +par_nonall_basefile ### Test --nonall --basefile +par_nonall_basefile tmp/nonall--basefile +par_nonall_basefile tmp/nonall--basefile +par_nonall_basefile_cleanup ### Test --nonall --basefile --cleanup (rm should fail) +par_nonall_basefile_cleanup tmp/nonall--basefile--clean +par_nonall_basefile_cleanup tmp/nonall--basefile--clean +par_nonall_basefile_cleanup rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory +par_nonall_basefile_cleanup rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory +par_nonall_wd ### Test --nonall --wd +par_nonall_wd /tmp +par_onall_basefile ### Test --onall --basefile +par_onall_basefile tmp/onall--basefile +par_onall_basefile tmp/onall--basefile +par_onall_basefile_cleanup ### Test --onall --basefile --cleanup (rm should fail) +par_onall_basefile_cleanup tmp/onall--basefile--clean +par_onall_basefile_cleanup tmp/onall--basefile--clean +par_onall_basefile_cleanup rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory +par_onall_basefile_cleanup rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory +par_read_sshloginfile_from_stdin ### Test read sshloginfile from STDIN +par_read_sshloginfile_from_stdin /home/csh +par_read_sshloginfile_from_stdin /home/parallel +par_read_sshloginfile_from_stdin /home/csh +par_read_sshloginfile_from_stdin foo +par_read_sshloginfile_from_stdin /home/parallel +par_read_sshloginfile_from_stdin foo +par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote +par_remote_symlink_dir OK: wd is still a symlink +par_test_nonall ### Test --nonall +par_test_nonall /home/csh +par_test_nonall /home/parallel +par_test_nonall_u ### Test --nonall -u - should be interleaved x y x y +par_test_nonall_u /home/parallel +par_test_nonall_u /home/csh +par_test_nonall_u /home/parallel +par_test_nonall_u /home/csh +par_test_onall ### Test --onall +par_test_onall csh@lo 1 +par_test_onall csh@lo 2 +par_test_onall csh@lo 1 +par_test_onall csh@lo 2 +par_test_onall csh@lo 1 +par_test_onall csh@lo 2 +par_test_onall parallel@lo 1 +par_test_onall parallel@lo 2 +par_test_onall parallel@lo 1 +par_test_onall parallel@lo 2 +par_test_onall parallel@lo 1 +par_test_onall parallel@lo 2 +par_test_onall_u ### Test --onall -u +par_test_onall_u 1 +par_test_onall_u 1 +par_test_onall_u 1 +par_test_onall_u 1 +par_test_onall_u 1 +par_test_onall_u 1 +par_test_onall_u 2 +par_test_onall_u 2 +par_test_onall_u 2 +par_test_onall_u 2 +par_test_onall_u 2 +par_test_onall_u 2 +par_test_onall_u 3 +par_test_onall_u 3 +par_test_onall_u 3 +par_test_onall_u 3 +par_test_onall_u 3 +par_test_onall_u 3 +par_test_pipe_onall ### Test | --onall +par_test_pipe_onall csh@lo 1 +par_test_pipe_onall csh@lo 2 +par_test_pipe_onall csh@lo 3 +par_test_pipe_onall csh@lo 1 +par_test_pipe_onall csh@lo 2 +par_test_pipe_onall csh@lo 3 +par_test_pipe_onall csh@lo 1 +par_test_pipe_onall csh@lo 2 +par_test_pipe_onall csh@lo 3 +par_test_pipe_onall parallel@lo 1 +par_test_pipe_onall parallel@lo 2 +par_test_pipe_onall parallel@lo 3 +par_test_pipe_onall parallel@lo 1 +par_test_pipe_onall parallel@lo 2 +par_test_pipe_onall parallel@lo 3 +par_test_pipe_onall parallel@lo 1 +par_test_pipe_onall parallel@lo 2 +par_test_pipe_onall parallel@lo 3 +par_wd_braces ### Test --wd {} +par_wd_braces wd1 +par_wd_braces wd2 +par_wd_dot ### Test --wd . +par_wd_dot /home/csh/mydir +par_wd_perlexpr ### Test --wd {= =} +par_wd_perlexpr WD1 +par_wd_perlexpr WD2 +par_workdir_dot ### Test --workdir . +par_workdir_dot /home/parallel/mydir