tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #51293: parallel does not preserve symlinked directory structure on remote.

Browse source
This commit is contained in:
Ole Tange 2017-06-22 23:14:32 +02:00
parent 526f2bfd4c
commit f3d117e7bb
8 changed files with 246 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
doc/release_new_version
View file

@ -4,7 +4,7 @@
Check that documentation is updated (compare to web): Check that documentation is updated (compare to web):
Fixet for 20170322 Fixet for 20170622
git diff last-release-commit git diff last-release-commit
Unmodified beta since last version => production Unmodified beta since last version => production
Unmodified alpha since last version => beta Unmodified alpha since last version => beta
@ -214,20 +214,8 @@ New in this release:
http://meta.askubuntu.com/a/16750/22307 http://meta.askubuntu.com/a/16750/22307
http://meta.serverfault.com/a/9040/45704 http://meta.serverfault.com/a/9040/45704
* Graph for overhead for 1000 jobs has been updated. It remains steadily at 3.5 ms per job on a 1.7 GHz machine https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf
* GNU Parallel was cited in: The topology of the directed clique complex as a network invariant https://springerplus.springeropen.com/articles/10.1186/s40064-016-2022-y
* GNU Parallel was cited in: * GNU Parallel was cited in:
* GNU Parallel was cited in: Parallel Execution of Structural Mechanic Tasks with Use of Small Supercomputers http://www.sciencedirect.com/science/article/pii/S1877705817324815
https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel
https://pzelasko.github.io/posts/speed-up-ml-model-development-with-gnu-parallel
http://tdhopper.com/blog/2017/Jun/07/parallelizing-a-python-function-for-the-extremely-lazy/
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>> <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
<<Wrong citation https://iris.sissa.it/retrieve/handle/20.500.11767/36149/10823/And%C3%B2_tesi.pdf>> <<Wrong citation https://iris.sissa.it/retrieve/handle/20.500.11767/36149/10823/And%C3%B2_tesi.pdf>>

2
src/niceload
View file

@ -24,7 +24,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20170622; $Global::version = 20170623;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

23
src/parallel
View file

@ -1,5 +1,10 @@
#!/usr/bin/env perl #!/usr/bin/env perl
# XDG_RUNTIME_DIR/parallel = TMPDIR
# XDG_CONFIG_HOME/parallel = ~/.parallel/config
# XDG_CONFIG_DIRS
# XDG_CACHE_HOME = ~/parallel/tmp + semaphores
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 # Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017
# Ole Tange and Free Software Foundation, Inc. # Ole Tange and Free Software Foundation, Inc.
# #
@ -1376,7 +1381,7 @@ sub check_invalid_option_combinations {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20170622; $Global::version = 20170623;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -3982,8 +3987,8 @@ sub usage {
"See 'man $Global::progname' for details", "See 'man $Global::progname' for details",
"", "",
"Academic tradition requires you to cite works you base your article on.", "Academic tradition requires you to cite works you base your article on.",
"When using programs that use GNU Parallel to process data for publication", "If you use programs that use GNU Parallel to process data for an article in a",
"please cite:", "scientific publication, please cite:",
"", "",
" O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
" ;login: The USENIX Magazine, February 2011:42-47.", " ;login: The USENIX Magazine, February 2011:42-47.",
@ -4010,8 +4015,8 @@ sub citation_notice {
} else { } else {
::status ::status
("Academic tradition requires you to cite works you base your article on.", ("Academic tradition requires you to cite works you base your article on.",
"When using programs that use GNU Parallel to process data for publication", "If you use programs that use GNU Parallel to process data for an article in a",
"please cite:", "scientific publication, please cite:",
"", "",
" O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
" ;login: The USENIX Magazine, February 2011:42-47.", " ;login: The USENIX Magazine, February 2011:42-47.",
@ -4085,8 +4090,8 @@ sub citation {
# Returns: N/A # Returns: N/A
::status( ::status(
"Academic tradition requires you to cite works you base your article on.", "Academic tradition requires you to cite works you base your article on.",
"When using programs that use GNU Parallel to process data for publication", "If you use programs that use GNU Parallel to process data for an article in a",
"please cite:", "scientific publication, please cite:",
"", "",
"\@article{Tange2011a,", "\@article{Tange2011a,",
" title = {GNU Parallel - The Command-Line Power Tool},", " title = {GNU Parallel - The Command-Line Power Tool},",
@ -4103,13 +4108,11 @@ sub citation {
"}", "}",
"", "",
"(Feel free to use \\nocite{Tange2011a})", "(Feel free to use \\nocite{Tange2011a})",
"",
"This helps funding further development; AND IT WON'T COST YOU A CENT.", "This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
"", "",
"If you send a copy of your published article to tange\@gnu.org, it will be", "If you send a copy of your published article to tange\@gnu.org, it will be",
"mentioned in the release notes of next version of GNU Parallel.", "mentioned in the release notes of next version of GNU Parallel.",
"",
"" ""
); );
while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") { while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") {
@ -6405,7 +6408,7 @@ sub rsync_transfer_cmd {
} }
$file = ::shell_quote_file($file); $file = ::shell_quote_file($file);
my $sshcmd = $self->sshcommand(); my $sshcmd = $self->sshcommand();
my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd); my $rsync_opt = "-KrlDzR -e" . ::shell_quote_scalar($sshcmd);
my $serverlogin = $self->serverlogin(); my $serverlogin = $self->serverlogin();
# Make dir if it does not exist # Make dir if it does not exist
return "$sshcmd $serverlogin -- mkdir -p $rsync_destdir && " . return "$sshcmd $serverlogin -- mkdir -p $rsync_destdir && " .

52
src/parallel.pod
View file

@ -95,7 +95,7 @@ B<Zsh, Fish, Ksh, and Pdksh functions and aliases>: Use B<env_parallel>.
The command cannot contain the character \257 (macron: ¯). The command cannot contain the character \257 (macron: ¯).
=item B<{}> (alpha testing) =item B<{}> (beta testing)
Input line. This replacement string will be replaced by a full line Input line. This replacement string will be replaced by a full line
read from the input source. The input source is normally stdin read from the input source. The input source is normally stdin
@ -112,7 +112,7 @@ parsed by the shell. The exception is if the command starts with a
replacement string; then the string is not quoted. replacement string; then the string is not quoted.
=item B<{.}> (alpha testing) =item B<{.}> (beta testing)
Input line without extension. This replacement string will be replaced Input line without extension. This replacement string will be replaced
by the input with the extension removed. If the input line contains by the input with the extension removed. If the input line contains
@ -128,7 +128,7 @@ The replacement string B<{.}> can be changed with B<--er>.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{/}> (alpha testing) =item B<{/}> (beta testing)
Basename of input line. This replacement string will be replaced by Basename of input line. This replacement string will be replaced by
the input with the directory part removed. the input with the directory part removed.
@ -139,7 +139,7 @@ B<--basenamereplace>.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{//}> (alpha testing) =item B<{//}> (beta testing)
Dirname of input line. This replacement string will be replaced by the Dirname of input line. This replacement string will be replaced by the
dir of the input line. See B<dirname>(1). dir of the input line. See B<dirname>(1).
@ -150,7 +150,7 @@ B<--dirnamereplace>.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{/.}> (alpha testing) =item B<{/.}> (beta testing)
Basename of input line without extension. This replacement string will Basename of input line without extension. This replacement string will
be replaced by the input with the directory and extension part be replaced by the input with the directory and extension part
@ -162,7 +162,7 @@ B<--basenameextensionreplace>.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{#}> (alpha testing) =item B<{#}> (beta testing)
Sequence number of the job to run. This replacement string will be Sequence number of the job to run. This replacement string will be
replaced by the sequence number of the job being run. It contains the replaced by the sequence number of the job being run. It contains the
@ -173,7 +173,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{%}> (alpha testing) =item B<{%}> (beta testing)
Job slot number. This replacement string will be replaced by the job's Job slot number. This replacement string will be replaced by the job's
slot number between 1 and number of jobs to run in parallel. There slot number between 1 and number of jobs to run in parallel. There
@ -185,7 +185,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{>I<n>B<}> (alpha testing) =item B<{>I<n>B<}> (beta testing)
Argument from input source I<n> or the I<n>'th argument. This Argument from input source I<n> or the I<n>'th argument. This
positional replacement string will be replaced by the input from input positional replacement string will be replaced by the input from input
@ -196,7 +196,7 @@ I<n>'th last argument.
To understand replacement strings see B<{}>. To understand replacement strings see B<{}>.
=item B<{>I<n>.B<}> (alpha testing) =item B<{>I<n>.B<}> (beta testing)
Argument from input source I<n> or the I<n>'th argument without Argument from input source I<n> or the I<n>'th argument without
extension. It is a combination of B<{>I<n>B<}> and B<{.}>. extension. It is a combination of B<{>I<n>B<}> and B<{.}>.
@ -209,7 +209,7 @@ extension removed.
To understand positional replacement strings see B<{>I<n>B<}>. To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{>I<n>/B<}> (alpha testing) =item B<{>I<n>/B<}> (beta testing)
Basename of argument from input source I<n> or the I<n>'th argument. Basename of argument from input source I<n> or the I<n>'th argument.
It is a combination of B<{>I<n>B<}> and B<{/}>. It is a combination of B<{>I<n>B<}> and B<{/}>.
@ -222,7 +222,7 @@ directory (if any) removed.
To understand positional replacement strings see B<{>I<n>B<}>. To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{>I<n>//B<}> (alpha testing) =item B<{>I<n>//B<}> (beta testing)
Dirname of argument from input source I<n> or the I<n>'th argument. Dirname of argument from input source I<n> or the I<n>'th argument.
It is a combination of B<{>I<n>B<}> and B<{//}>. It is a combination of B<{>I<n>B<}> and B<{//}>.
@ -234,7 +234,7 @@ the I<n>'th argument (when used with B<-N>). See B<dirname>(1).
To understand positional replacement strings see B<{>I<n>B<}>. To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{>I<n>/.B<}> (alpha testing) =item B<{>I<n>/.B<}> (beta testing)
Basename of argument from input source I<n> or the I<n>'th argument Basename of argument from input source I<n> or the I<n>'th argument
without extension. It is a combination of B<{>I<n>B<}>, B<{/}>, and without extension. It is a combination of B<{>I<n>B<}>, B<{/}>, and
@ -248,7 +248,7 @@ directory (if any) and extension removed.
To understand positional replacement strings see B<{>I<n>B<}>. To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{=>I<perl expression>B<=}> (alpha testing) =item B<{=>I<perl expression>B<=}> (beta testing)
Replace with calculated I<perl expression>. B<$_> will contain the Replace with calculated I<perl expression>. B<$_> will contain the
same as B<{}>. After evaluating I<perl expression> B<$_> will be used same as B<{}>. After evaluating I<perl expression> B<$_> will be used
@ -294,7 +294,7 @@ Example:
See also: B<--rpl> B<--parens> See also: B<--rpl> B<--parens>
=item B<{=>I<n> I<perl expression>B<=}> (alpha testing) =item B<{=>I<n> I<perl expression>B<=}> (beta testing)
Positional equivalent to B<{=perl expression=}>. To understand Positional equivalent to B<{=perl expression=}>. To understand
positional replacement strings see B<{>I<n>B<}>. positional replacement strings see B<{>I<n>B<}>.
@ -341,7 +341,7 @@ B<:::> and B<::::> can be mixed. So these are equivalent:
::: 1 2 3 ::: 1 2 3
=item B<:::+> I<arguments> (beta testing) =item B<:::+> I<arguments>
Like B<:::> but linked like B<--link> to the previous input source. Like B<:::> but linked like B<--link> to the previous input source.
@ -362,7 +362,7 @@ B<:::> and B<::::> can be mixed.
See B<-a>, B<:::> and B<--link>. See B<-a>, B<:::> and B<--link>.
=item B<::::+> I<argfiles> (beta testing) =item B<::::+> I<argfiles>
Like B<::::> but linked like B<--link> to the previous input source. Like B<::::> but linked like B<--link> to the previous input source.
@ -418,7 +418,7 @@ string that is not in the command line.
See also: B<:::>. See also: B<:::>.
=item B<--bar> (beta testing) =item B<--bar>
Show progress as a progress bar. In the bar is shown: % of jobs Show progress as a progress bar. In the bar is shown: % of jobs
completed, estimated seconds left, and number of jobs started. completed, estimated seconds left, and number of jobs started.
@ -656,7 +656,7 @@ variables, arrays, and functions) use B<env_parallel>.
See also: B<--record-env>. See also: B<--record-env>.
=item B<--eta> (beta testing) =item B<--eta>
Show the estimated number of seconds before finishing. This forces GNU Show the estimated number of seconds before finishing. This forces GNU
B<parallel> to read all jobs before starting to find the number of B<parallel> to read all jobs before starting to find the number of
@ -744,9 +744,9 @@ See also: B<--line-buffer> B<--ungroup>
Print a summary of the options to GNU B<parallel> and exit. Print a summary of the options to GNU B<parallel> and exit.
=item B<--halt-on-error> I<val> (beta testing) =item B<--halt-on-error> I<val>
=item B<--halt> I<val> (beta testing) =item B<--halt> I<val>
When should GNU B<parallel> terminate? In some situations it makes no When should GNU B<parallel> terminate? In some situations it makes no
sense to run all jobs. GNU B<parallel> should simply give up as soon sense to run all jobs. GNU B<parallel> should simply give up as soon
@ -891,7 +891,7 @@ specified, and for B<-I {}> otherwise. This option is deprecated;
use B<-I> instead. use B<-I> instead.
=item B<--joblog> I<logfile> (beta testing) =item B<--joblog> I<logfile>
Logfile for executed jobs. Save a list of the executed jobs to Logfile for executed jobs. Save a list of the executed jobs to
I<logfile> in the following TAB separated format: sequence number, I<logfile> in the following TAB separated format: sequence number,
@ -1076,7 +1076,7 @@ Arguments will be recycled if one input source has more arguments than the other
See also B<--header>, B<:::+>, B<::::+>. See also B<--header>, B<:::+>, B<::::+>.
=item B<--load> I<max-load> (alpha testing) =item B<--load> I<max-load> (beta testing)
Do not start new jobs on a given computer unless the number of running Do not start new jobs on a given computer unless the number of running
processes on the computer is less than I<max-load>. I<max-load> uses processes on the computer is less than I<max-load>. I<max-load> uses
@ -1254,7 +1254,7 @@ control on the command line (used by GNU B<parallel> internally when
called with B<--sshlogin>). called with B<--sshlogin>).
=item B<--plus> (alpha testing) =item B<--plus> (beta testing)
Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} Activate additional replacement strings: {+/} {+.} {+..} {+...} {..}
{...} {/..} {/...} {##}. The idea being that '{+foo}' matches the opposite of {...} {/..} {/...} {##}. The idea being that '{+foo}' matches the opposite of
@ -1767,7 +1767,7 @@ impossible to track which input block corresponds to which output.
B<--round-robin> implies B<--pipe>, except if B<--pipepart> is given. B<--round-robin> implies B<--pipe>, except if B<--pipepart> is given.
=item B<--rpl> 'I<tag> I<perl expression>' (alpha testing) =item B<--rpl> 'I<tag> I<perl expression>' (beta testing)
Use I<tag> as a replacement string for I<perl expression>. This makes Use I<tag> as a replacement string for I<perl expression>. This makes
it possible to define your own replacement strings. GNU B<parallel>'s it possible to define your own replacement strings. GNU B<parallel>'s
@ -2277,13 +2277,13 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
setting $TMPDIR. setting $TMPDIR.
=item B<--tmux> (beta testing) =item B<--tmux>
Use B<tmux> for output. Start a B<tmux> session and run each job in a Use B<tmux> for output. Start a B<tmux> session and run each job in a
window in that session. No other output will be produced. window in that session. No other output will be produced.
=item B<--tmuxpane> (beta testing) =item B<--tmuxpane>
Use B<tmux> for output but put output into panes in the first window. Use B<tmux> for output but put output into panes in the first window.
Useful if you want to monitor the progress of less than 100 concurrent Useful if you want to monitor the progress of less than 100 concurrent

3
src/parset.pod
View file

@ -25,7 +25,8 @@ the destination variable and made to an array.
If I<variablename> contains multiple names separated by ',' or space, If I<variablename> contains multiple names separated by ',' or space,
the names will be the destination variables. the names will be the destination variables.
B<parset> is not production ready and is beta quality. B<parset> is beta quality and not production ready, but please use it
for everyday use and report bugs.
=head1 OPTIONS =head1 OPTIONS

2
src/sql
View file

@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20170622; $Global::version = 20170623;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

173
testsuite/tests-to-run/parallel-local-ssh6.sh
View file

@ -1,87 +1,126 @@
#!/bin/bash #!/bin/bash
SSHLOGIN1=parallel@lo # At most 2 parallel sshs per function
SSHLOGIN2=csh@lo
export SSHLOGIN1=parallel@lo
export SSHLOGIN2=csh@lo
mkdir -p tmp mkdir -p tmp
# -L1 will join lines ending in ' ' par_test_onall() {
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj5 -k --joblog /tmp/jl-`basename $0` -L1 echo '### Test --onall'
echo '### Test --onall'; parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2
parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 }
echo '### Test | --onall'; par_test_pipe_onall() {
seq 3 | parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - echo '### Test | --onall'
seq 3 |
parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: -
}
echo '### Test --onall -u'; par_test_onall_u() {
parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort echo '### Test --onall -u'
parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 |
sort
}
echo '### Test --nonall'; par_test_nonall() {
parallel --nonall -k -S $SSHLOGIN1,$SSHLOGIN2 pwd | sort echo '### Test --nonall'
parallel --nonall -k -S $SSHLOGIN1,$SSHLOGIN2 pwd |
sort
}
echo '### Test --nonall -u - should be interleaved x y x y'; par_test_nonall_u() {
parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;' echo '### Test --nonall -u - should be interleaved x y x y'
parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;'
}
echo '### Test read sshloginfile from STDIN'; par_read_sshloginfile_from_stdin() {
echo $SSHLOGIN1,$SSHLOGIN2 | parallel -S - -k --nonall pwd; echo '### Test read sshloginfile from STDIN'
echo $SSHLOGIN1,$SSHLOGIN2 | parallel --sshloginfile - -k --onall pwd\; echo ::: foo echo $SSHLOGIN1,$SSHLOGIN2 | parallel -S - -k --nonall pwd
echo $SSHLOGIN1,$SSHLOGIN2 | parallel --sshloginfile - -k --onall pwd\; echo ::: foo
}
echo '**' par_nonall_basefile() {
echo '### Test --nonall --basefile'
touch tmp/nonall--basefile
stdout parallel --nonall --basefile tmp/nonall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile
stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile
stdout rm tmp/nonall--basefile
}
echo '### Test --nonall --basefile'; par_onall_basefile() {
touch tmp/nonall--basefile; echo '### Test --onall --basefile'
stdout parallel --nonall --basefile tmp/nonall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile; touch tmp/onall--basefile
stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile; stdout parallel --onall --basefile tmp/onall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile
stdout rm tmp/nonall--basefile stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile
stdout rm tmp/onall--basefile
}
echo '**' par_nonall_basefile_cleanup() {
echo '### Test --nonall --basefile --cleanup (rm should fail)'
touch tmp/nonall--basefile--clean
stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile--clean
stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile--clean
stdout rm tmp/nonall--basefile--clean
}
echo '### Test --onall --basefile'; par_onall_basefile_cleanup() {
touch tmp/onall--basefile; echo '### Test --onall --basefile --cleanup (rm should fail)'
stdout parallel --onall --basefile tmp/onall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile; touch tmp/onall--basefile--clean
stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile; stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile--clean
stdout rm tmp/onall--basefile stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile--clean
stdout rm tmp/onall--basefile--clean
}
echo '**' par_workdir_dot() {
echo '### Test --workdir .'
ssh $SSHLOGIN1 mkdir -p mydir
mkdir -p $HOME/mydir
cd $HOME/mydir
parallel --workdir . -S $SSHLOGIN1 ::: pwd
}
echo '### Test --nonall --basefile --cleanup (rm should fail)'; par_wd_dot() {
touch tmp/nonall--basefile--clean; echo '### Test --wd .'
stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile--clean; ssh $SSHLOGIN2 mkdir -p mydir
stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile--clean; mkdir -p $HOME/mydir
stdout rm tmp/nonall--basefile--clean cd $HOME/mydir
parallel --workdir . -S $SSHLOGIN2 ::: pwd
}
echo '**' par_wd_braces() {
echo '### Test --wd {}'
ssh $SSHLOGIN2 rm -rf wd1 wd2
mkdir -p $HOME/mydir
cd $HOME/mydir
parallel --workdir {} -S $SSHLOGIN2 touch ::: wd1 wd2
ssh $SSHLOGIN2 ls -d wd1 wd2
}
echo '### Test --onall --basefile --cleanup (rm should fail)'; par_wd_perlexpr() {
touch tmp/onall--basefile--clean; echo '### Test --wd {= =}'
stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile--clean; ssh $SSHLOGIN2 rm -rf WD1 WD2
stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile--clean; mkdir -p $HOME/mydir
stdout rm tmp/onall--basefile--clean cd $HOME/mydir
parallel --workdir '{= $_=uc($_) =}' -S $SSHLOGIN2 touch ::: wd1 wd2
ssh $SSHLOGIN2 ls -d WD1 WD2
}
echo '**' par_nonall_wd() {
echo '### Test --nonall --wd'
parallel --workdir /tmp -S $SSHLOGIN2 --nonall pwd
}
echo '### Test --workdir .'; par_remote_symlink_dir() {
ssh $SSHLOGIN1 mkdir -p mydir; echo 'bug #51293: parallel does not preserve symlinked directory structure on remote'
mkdir -p $HOME/mydir; cd $HOME/mydir; ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd'
parallel --workdir . -S $SSHLOGIN1 ::: pwd mkdir -p wd
touch wd/testfile
parallel --nonall -S parallel@lo --basefile wd/testfile
ssh parallel@lo rm wd && echo OK: wd is still a symlink
}
echo '### Test --wd .'; export -f $(compgen -A function | grep par_)
ssh $SSHLOGIN2 mkdir -p mydir; #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1'
mkdir -p $HOME/mydir; cd $HOME/mydir; compgen -A function | grep par_ | sort |
parallel --workdir . -S $SSHLOGIN2 ::: pwd parallel --joblog /tmp/jl-`basename $0` -j5 --tag -k '{} 2>&1'
echo '### Test --wd {}';
ssh $SSHLOGIN2 rm -rf wd1 wd2;
mkdir -p $HOME/mydir; cd $HOME/mydir;
parallel --workdir {} -S $SSHLOGIN2 touch ::: wd1 wd2;
ssh $SSHLOGIN2 ls -d wd1 wd2
echo '### Test --wd {= =}';
ssh $SSHLOGIN2 rm -rf WD1 WD2;
mkdir -p $HOME/mydir; cd $HOME/mydir;
parallel --workdir '{= $_=uc($_) =}' -S $SSHLOGIN2 touch ::: wd1 wd2;
ssh $SSHLOGIN2 ls -d WD1 WD2
echo '### Test --nonall --wd';
parallel --workdir /tmp -S $SSHLOGIN2 --nonall pwd
EOF

215
testsuite/wanted-results/parallel-local-ssh6
View file

@ -1,119 +1,96 @@
echo '### Test --onall'; parallel --onall --tag -k -S parallel@lo,csh@lo '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 par_nonall_basefile ### Test --nonall --basefile
### Test --onall par_nonall_basefile tmp/nonall--basefile
csh@lo 1 par_nonall_basefile tmp/nonall--basefile
csh@lo 2 par_nonall_basefile_cleanup ### Test --nonall --basefile --cleanup (rm should fail)
csh@lo 1 par_nonall_basefile_cleanup tmp/nonall--basefile--clean
csh@lo 2 par_nonall_basefile_cleanup tmp/nonall--basefile--clean
csh@lo 1 par_nonall_basefile_cleanup rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory
csh@lo 2 par_nonall_basefile_cleanup rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory
parallel@lo 1 par_nonall_wd ### Test --nonall --wd
parallel@lo 2 par_nonall_wd /tmp
parallel@lo 1 par_onall_basefile ### Test --onall --basefile
parallel@lo 2 par_onall_basefile tmp/onall--basefile
parallel@lo 1 par_onall_basefile tmp/onall--basefile
parallel@lo 2 par_onall_basefile_cleanup ### Test --onall --basefile --cleanup (rm should fail)
echo '### Test | --onall'; seq 3 | parallel --onall --tag -k -S parallel@lo,csh@lo '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - par_onall_basefile_cleanup tmp/onall--basefile--clean
### Test | --onall par_onall_basefile_cleanup tmp/onall--basefile--clean
csh@lo 1 par_onall_basefile_cleanup rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory
csh@lo 2 par_onall_basefile_cleanup rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory
csh@lo 3 par_read_sshloginfile_from_stdin ### Test read sshloginfile from STDIN
csh@lo 1 par_read_sshloginfile_from_stdin /home/csh
csh@lo 2 par_read_sshloginfile_from_stdin /home/parallel
csh@lo 3 par_read_sshloginfile_from_stdin /home/csh
csh@lo 1 par_read_sshloginfile_from_stdin foo
csh@lo 2 par_read_sshloginfile_from_stdin /home/parallel
csh@lo 3 par_read_sshloginfile_from_stdin foo
parallel@lo 1 par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote
parallel@lo 2 par_remote_symlink_dir OK: wd is still a symlink
parallel@lo 3 par_test_nonall ### Test --nonall
parallel@lo 1 par_test_nonall /home/csh
parallel@lo 2 par_test_nonall /home/parallel
parallel@lo 3 par_test_nonall_u ### Test --nonall -u - should be interleaved x y x y
parallel@lo 1 par_test_nonall_u /home/parallel
parallel@lo 2 par_test_nonall_u /home/csh
parallel@lo 3 par_test_nonall_u /home/parallel
echo '### Test --onall -u'; parallel --onall -S parallel@lo,csh@lo -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort par_test_nonall_u /home/csh
### Test --onall -u par_test_onall ### Test --onall
1 par_test_onall csh@lo 1
1 par_test_onall csh@lo 2
1 par_test_onall csh@lo 1
1 par_test_onall csh@lo 2
1 par_test_onall csh@lo 1
1 par_test_onall csh@lo 2
2 par_test_onall parallel@lo 1
2 par_test_onall parallel@lo 2
2 par_test_onall parallel@lo 1
2 par_test_onall parallel@lo 2
2 par_test_onall parallel@lo 1
2 par_test_onall parallel@lo 2
3 par_test_onall_u ### Test --onall -u
3 par_test_onall_u 1
3 par_test_onall_u 1
3 par_test_onall_u 1
3 par_test_onall_u 1
3 par_test_onall_u 1
echo '### Test --nonall'; parallel --nonall -k -S parallel@lo,csh@lo pwd | sort par_test_onall_u 1
### Test --nonall par_test_onall_u 2
/home/csh par_test_onall_u 2
/home/parallel par_test_onall_u 2
echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall -S parallel@lo,csh@lo -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;' par_test_onall_u 2
### Test --nonall -u - should be interleaved x y x y par_test_onall_u 2
/home/parallel par_test_onall_u 2
/home/csh par_test_onall_u 3
/home/parallel par_test_onall_u 3
/home/csh par_test_onall_u 3
echo '### Test read sshloginfile from STDIN'; echo parallel@lo,csh@lo | parallel -S - -k --nonall pwd; echo parallel@lo,csh@lo | parallel --sshloginfile - -k --onall pwd\; echo ::: foo par_test_onall_u 3
### Test read sshloginfile from STDIN par_test_onall_u 3
/home/csh par_test_onall_u 3
/home/parallel par_test_pipe_onall ### Test | --onall
/home/csh par_test_pipe_onall csh@lo 1
foo par_test_pipe_onall csh@lo 2
/home/parallel par_test_pipe_onall csh@lo 3
foo par_test_pipe_onall csh@lo 1
echo '**' par_test_pipe_onall csh@lo 2
** par_test_pipe_onall csh@lo 3
echo '### Test --nonall --basefile'; touch tmp/nonall--basefile; stdout parallel --nonall --basefile tmp/nonall--basefile -S parallel@lo,csh@lo ls tmp/nonall--basefile; stdout parallel --nonall -S parallel@lo,csh@lo rm tmp/nonall--basefile; stdout rm tmp/nonall--basefile par_test_pipe_onall csh@lo 1
### Test --nonall --basefile par_test_pipe_onall csh@lo 2
tmp/nonall--basefile par_test_pipe_onall csh@lo 3
tmp/nonall--basefile par_test_pipe_onall parallel@lo 1
echo '**' par_test_pipe_onall parallel@lo 2
** par_test_pipe_onall parallel@lo 3
echo '### Test --onall --basefile'; touch tmp/onall--basefile; stdout parallel --onall --basefile tmp/onall--basefile -S parallel@lo,csh@lo ls {} ::: tmp/onall--basefile; stdout parallel --onall -S parallel@lo,csh@lo rm {} ::: tmp/onall--basefile; stdout rm tmp/onall--basefile par_test_pipe_onall parallel@lo 1
### Test --onall --basefile par_test_pipe_onall parallel@lo 2
tmp/onall--basefile par_test_pipe_onall parallel@lo 3
tmp/onall--basefile par_test_pipe_onall parallel@lo 1
echo '**' par_test_pipe_onall parallel@lo 2
** par_test_pipe_onall parallel@lo 3
echo '### Test --nonall --basefile --cleanup (rm should fail)'; touch tmp/nonall--basefile--clean; stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S parallel@lo,csh@lo ls tmp/nonall--basefile--clean; stdout parallel --nonall -S parallel@lo,csh@lo rm tmp/nonall--basefile--clean; stdout rm tmp/nonall--basefile--clean par_wd_braces ### Test --wd {}
### Test --nonall --basefile --cleanup (rm should fail) par_wd_braces wd1
tmp/nonall--basefile--clean par_wd_braces wd2
tmp/nonall--basefile--clean par_wd_dot ### Test --wd .
rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory par_wd_dot /home/csh/mydir
rm: cannot remove 'tmp/nonall--basefile--clean': No such file or directory par_wd_perlexpr ### Test --wd {= =}
echo '**' par_wd_perlexpr WD1
** par_wd_perlexpr WD2
echo '### Test --onall --basefile --cleanup (rm should fail)'; touch tmp/onall--basefile--clean; stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S parallel@lo,csh@lo ls {} ::: tmp/onall--basefile--clean; stdout parallel --onall -S parallel@lo,csh@lo rm {} ::: tmp/onall--basefile--clean; stdout rm tmp/onall--basefile--clean par_workdir_dot ### Test --workdir .
### Test --onall --basefile --cleanup (rm should fail) par_workdir_dot /home/parallel/mydir
tmp/onall--basefile--clean
tmp/onall--basefile--clean
rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory
rm: cannot remove 'tmp/onall--basefile--clean': No such file or directory
echo '**'
**
echo '### Test --workdir .'; ssh parallel@lo mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S parallel@lo ::: pwd
### Test --workdir .
/home/parallel/mydir
echo '### Test --wd .'; ssh csh@lo mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S csh@lo ::: pwd
### Test --wd .
/home/csh/mydir
echo '### Test --wd {}'; ssh csh@lo rm -rf wd1 wd2; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir {} -S csh@lo touch ::: wd1 wd2; ssh csh@lo ls -d wd1 wd2
### Test --wd {}
wd1
wd2
echo '### Test --wd {= =}'; ssh csh@lo rm -rf WD1 WD2; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir '{= $_=uc($_) =}' -S csh@lo touch ::: wd1 wd2; ssh csh@lo ls -d WD1 WD2
### Test --wd {= =}
WD1
WD2
echo '### Test --nonall --wd'; parallel --workdir /tmp -S csh@lo --nonall pwd
### Test --nonall --wd
/tmp