mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-10-18 12:59:39 +00:00
Fixed bug #46050: --ssh lsh supported and partly --ssh autossh.
This commit is contained in:
Ole Tange
parent
51f5363211
commit
6b90d81add
|
|
@ -234,6 +234,8 @@ New in this release:
|
|||
* <<kontaktet 2015-06-22 Afventer svar fra journal>> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
|
||||
http://www.researchgate.net/profile/Christoph_Junghans/publication/276178326_TADSim_Discrete_Event-Based_Performance_Prediction_for_Temperature-Accelerated_Dynamics/links/55562b6708ae980ca60c8369.pdf
|
||||
|
||||
* GNU Parallel was cited in: There is no (75, 32, 10, 16) strongly regular graph http://arxiv.org/pdf/1509.05933.pdf
|
||||
|
||||
* GNU Parallel was cited in: Roary: rapid large-scale prokaryote pan genome analysis http://bioinformatics.oxfordjournals.org/content/early/2015/08/05/bioinformatics.btv421.full.pdf+html
|
||||
|
||||
* GNU Parallel is used in TraitAR: https://testpypi.python.org/pypi/traitar/0.1.4
|
||||
|
|
|
|||
40
src/parallel
40
src/parallel
|
|
@ -2861,8 +2861,9 @@ sub parse_host_filtering {
|
|||
my $host = $1;
|
||||
$host =~ tr/\\//d;
|
||||
$Global::host{$host} or next;
|
||||
if($col[6] eq "255" or $col[6] eq "-1") {
|
||||
if($col[6] eq "255" or $col[6] eq "-1" or $col[6] eq "1") {
|
||||
# exit == 255 or exit == timeout (-1): ssh failed/timedout
|
||||
# exit == 1: lsh failed
|
||||
# Remove sshlogin
|
||||
::debug("init", "--filtered $host\n");
|
||||
push(@down_hosts, $host);
|
||||
|
|
@ -2932,11 +2933,11 @@ sub parallelized_host_filtering {
|
|||
if($host eq ":") { next }
|
||||
# The 'true' is used to get the $host out later
|
||||
my $sshcmd = "true $host; exec " . $sshlogin->sshcommand()." ".$sshlogin->serverlogin();
|
||||
push(@cores, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cores\n\0");
|
||||
push(@cpus, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cpus\n\0");
|
||||
push(@maxline, $host."\t".$sshcmd." ".$envvar." parallel --max-line-length-allowed\n\0");
|
||||
push(@cores, $host."\t".$sshcmd." -- ".$envvar." parallel --number-of-cores\n\0");
|
||||
push(@cpus, $host."\t".$sshcmd." -- ".$envvar." parallel --number-of-cpus\n\0");
|
||||
push(@maxline, $host."\t".$sshcmd." -- ".$envvar." parallel --max-line-length-allowed\n\0");
|
||||
# 'echo' is used to get the best possible value for an ssh login time
|
||||
push(@echo, $host."\t".$sshcmd." echo\n\0");
|
||||
push(@echo, $host."\t".$sshcmd." -- echo\n\0");
|
||||
}
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh");
|
||||
print $fh @cores, @cpus, @maxline, @echo;
|
||||
|
|
@ -3765,7 +3766,7 @@ sub which {
|
|||
# posh rbash rc rush rzsh sash sh static-sh tcsh yash zsh
|
||||
my @shells = (qw(ash bash csh dash fdsh fish fizsh ksh
|
||||
ksh93 lksh mksh pdksh posh rbash rc rush rzsh sash sh
|
||||
static-sh tcsh yash zsh -sh -csh),
|
||||
static-sh tcsh yash zsh -sh -csh -bash),
|
||||
'-sh (sh)' # sh on FreeBSD
|
||||
);
|
||||
# Can be formatted as:
|
||||
|
|
@ -3780,6 +3781,8 @@ sub which {
|
|||
# csh and tcsh disguise themselves as -sh/-csh
|
||||
"-sh" => ["csh", "tcsh"],
|
||||
"-csh" => ["tcsh", "csh"],
|
||||
# bash disguises itself as -bash
|
||||
"-bash" => ["bash", "sh"],
|
||||
);
|
||||
}
|
||||
my ($children_of_ref, $parent_of_ref, $name_of_ref) = pid_table();
|
||||
|
|
@ -4971,7 +4974,7 @@ sub simultaneous_sshlogin {
|
|||
my $serverlogin = $self->serverlogin();
|
||||
my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : "";
|
||||
# TODO sh -c wrapper to work for csh
|
||||
my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
|
||||
my $cmd = "$sshdelay$sshcmd $serverlogin -- echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
|
||||
::debug("init", "Trying $wanted_processes logins at $serverlogin\n");
|
||||
open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or
|
||||
::die_bug("simultaneouslogin");
|
||||
|
|
@ -5049,10 +5052,10 @@ sub ncpus {
|
|||
my $ncpu;
|
||||
my $sqe = ::shell_quote_scalar($Global::envvar);
|
||||
if($opt::use_cpus_instead_of_cores) {
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin $sqe parallel --number-of-cpus);
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cpus);
|
||||
} else {
|
||||
::debug("init",qq(echo|$sshcmd $serverlogin $sqe parallel --number-of-cores\n));
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin $sqe parallel --number-of-cores);
|
||||
::debug("init",qq(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores\n));
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores);
|
||||
}
|
||||
chomp $ncpu;
|
||||
if($ncpu =~ /^\s*[0-9]+\s*$/s) {
|
||||
|
|
@ -5580,7 +5583,7 @@ sub sshcommand_of_sshlogin {
|
|||
my $sleep = ::shell_quote_scalar
|
||||
('$|=1;while(1){sleep 1;print "foo\n"}');
|
||||
my @master = ($opt::ssh, "-tt", "-MTS",
|
||||
$control_path, $serverlogin, "perl", "-e",
|
||||
$control_path, $serverlogin, "--", "perl", "-e",
|
||||
$sleep);
|
||||
exec(@master);
|
||||
}
|
||||
|
|
@ -5589,6 +5592,13 @@ sub sshcommand_of_sshlogin {
|
|||
$sshcmd = $opt::ssh; $serverlogin = $self->{'string'};
|
||||
}
|
||||
}
|
||||
|
||||
if($serverlogin =~ s/(\S+)\@(\S+)/$2/) {
|
||||
# convert user@server to '-l user server'
|
||||
# because lsh does not support user@server
|
||||
$sshcmd = $sshcmd." -l ".$1;
|
||||
}
|
||||
|
||||
$self->{'sshcommand'} = $sshcmd;
|
||||
$self->{'serverlogin'} = $serverlogin;
|
||||
}
|
||||
|
|
@ -5634,7 +5644,7 @@ sub rsync_transfer_cmd {
|
|||
my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd);
|
||||
my $serverlogin = $self->serverlogin();
|
||||
# Make dir if it does not exist
|
||||
return "( $sshcmd $serverlogin mkdir -p $rsync_destdir;" .
|
||||
return "( $sshcmd $serverlogin -- mkdir -p $rsync_destdir;" .
|
||||
rsync()." $rsync_opt $file $serverlogin:$rsync_destdir )";
|
||||
}
|
||||
|
||||
|
|
@ -5672,7 +5682,7 @@ sub cleanup_cmd {
|
|||
$f = ::shell_quote_file($f);
|
||||
my $sshcmd = $self->sshcommand();
|
||||
my $serverlogin = $self->serverlogin();
|
||||
return "$sshcmd $serverlogin ".::shell_quote_scalar("(rm -f $f; $rmdir)");
|
||||
return "$sshcmd $serverlogin -- ".::shell_quote_scalar("(rm -f $f; $rmdir)");
|
||||
}
|
||||
|
||||
{
|
||||
|
|
@ -6799,7 +6809,7 @@ sub sshlogin_wrap {
|
|||
}
|
||||
$self->{'sshlogin_wrap'} =
|
||||
($pre
|
||||
. "$sshcmd $serverlogin exec "
|
||||
. "$sshcmd $serverlogin -- exec "
|
||||
. $quoted_remote_command
|
||||
. ";"
|
||||
. $post);
|
||||
|
|
@ -6939,7 +6949,7 @@ sub sshcleanup {
|
|||
$cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";";
|
||||
}
|
||||
if(defined $opt::workdir and $opt::workdir eq "...") {
|
||||
$cleancmd .= "$sshcmd $serverlogin rm -rf " . ::shell_quote_scalar($workdir).';';
|
||||
$cleancmd .= "$sshcmd $serverlogin -- rm -rf " . ::shell_quote_scalar($workdir).';';
|
||||
}
|
||||
return $cleancmd;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ B<Ksh functions>: If it is a Ksh function you can encode the function in a varia
|
|||
export PARALLEL_ENV=`typeset -f foo`
|
||||
parallel foo ::: works
|
||||
|
||||
To export all functions and also make them available when running remote:
|
||||
To export all functions, aliases, and variables and also make them available when running remote:
|
||||
|
||||
env_parallel() {
|
||||
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";
|
||||
|
|
@ -122,7 +122,7 @@ B<Zsh functions>: If it is a Zsh function you can encode the function in a varia
|
|||
export PARALLEL_ENV
|
||||
parallel foo ::: works
|
||||
|
||||
To export all functions and also make them available when running remote:
|
||||
To export all functions, aliases, and variables and also make them available when running remote:
|
||||
|
||||
env_parallel() {
|
||||
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
|
||||
|
|
@ -144,10 +144,11 @@ B<Fish functions>: If it is a Fish function you can encode the function in a var
|
|||
setenv PARALLEL_ENV (functions foo|perl -pe 's/\n/\001/')
|
||||
parallel foo ::: works
|
||||
|
||||
To export all functions and also make them available when running remote:
|
||||
To export all functions and simple variables (not arrays) and also make them available
|
||||
when running remote:
|
||||
|
||||
function env_parallel
|
||||
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/')
|
||||
setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
|
||||
parallel $argv;
|
||||
set -e PARALLEL_ENV
|
||||
end
|
||||
|
|
@ -2352,23 +2353,7 @@ B<--env>:
|
|||
|
||||
If your environment (aliases, variables, and functions) is small you
|
||||
can copy the full environment without having to B<export -f>
|
||||
anything. Just run this first:
|
||||
|
||||
env_parallel() {
|
||||
export parallel_bash_environment='() {
|
||||
'"$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"'
|
||||
}'
|
||||
# Run as: env_parallel parallel_bash_environment "2>/dev/null;" ...
|
||||
`which parallel` "$@"
|
||||
unset parallel_bash_environment
|
||||
}
|
||||
|
||||
And then call as:
|
||||
|
||||
env_parallel doit ::: 1 2 3
|
||||
env_parallel doubleit ::: 1 2 3 ::: a b
|
||||
env_parallel -S server doit ::: 1 2 3
|
||||
env_parallel -S server doubleit ::: 1 2 3 ::: a b
|
||||
anything. See B<env_parallel> earlier in the man page.
|
||||
|
||||
|
||||
=head1 EXAMPLE: Function tester
|
||||
|
|
@ -4089,10 +4074,9 @@ B<prll> is also a tool for running jobs in parallel. It does not
|
|||
support running jobs on remote computers.
|
||||
|
||||
B<prll> encourages using BASH aliases and BASH functions instead of
|
||||
scripts. GNU B<parallel> will never support running aliases (see why
|
||||
http://www.perlmonks.org/index.pl?node_id=484296). However, scripts,
|
||||
composed commands, or functions exported with B<export -f> work just
|
||||
fine.
|
||||
scripts. GNU B<parallel> supports scripts directly, functions if they
|
||||
are exported using B<export -f>, and aliases if using B<env_parallel>
|
||||
described earlier.
|
||||
|
||||
B<prll> generates a lot of status information on stderr (standard
|
||||
error) which makes it harder to use the stderr (standard error) output
|
||||
|
|
@ -4467,9 +4451,8 @@ or:
|
|||
|
||||
it may be because I<command> is not known, but it could also be
|
||||
because I<command> is an alias or a function. If it is a function you
|
||||
need to B<export -f> the function first. An alias will, however, not
|
||||
work (see why http://www.perlmonks.org/index.pl?node_id=484296), so
|
||||
change your alias to a script.
|
||||
need to B<export -f> the function first. An alias will only work if you use
|
||||
B<env_parallel> described earlier.
|
||||
|
||||
|
||||
=head1 REPORTING BUGS
|
||||
|
|
|
|||
|
|
@ -213,6 +213,9 @@ EOS
|
|||
echo
|
||||
echo "### Fish environment"
|
||||
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
|
||||
# All variables cannot reliably be exported
|
||||
# perl -e 'print map { "setenv///$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:///: :g' |fish
|
||||
|
||||
set myvar "myvar works"
|
||||
setenv myenvvar "myenvvar works"
|
||||
|
||||
|
|
@ -220,6 +223,7 @@ set funky (perl -e 'print pack "c*", 1..255')
|
|||
setenv funkyenv (perl -e 'print pack "c*", 1..255')
|
||||
|
||||
set myarray '' array_val2 3 '' 5
|
||||
# Assoc arrays do not exist
|
||||
#typeset -A assocarr
|
||||
#assocarr[a]=assoc_val_a
|
||||
#assocarr[b]=assoc_val_b
|
||||
|
|
@ -229,27 +233,39 @@ function func_echo
|
|||
echo $argv;
|
||||
echo "$myvar"
|
||||
echo "$myenvvar"
|
||||
echo $myarray[2]
|
||||
# Arrays are not exported
|
||||
# echo $myarray[2]
|
||||
# Assoc arrays do not exist in fish
|
||||
# echo ${assocarr[a]}
|
||||
echo
|
||||
echo
|
||||
echo
|
||||
echo Funky-"$funky"-funky
|
||||
echo Funky-"$funkyenv"-funky
|
||||
echo
|
||||
echo
|
||||
echo
|
||||
end
|
||||
|
||||
function env_parallel
|
||||
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/')
|
||||
setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
|
||||
parallel $argv;
|
||||
set -e PARALLEL_ENV
|
||||
end
|
||||
|
||||
env_parallel alias_echo ::: alias_does_not_work
|
||||
env_parallel alias_echo ::: alias_works
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel -S fish@lo alias_echo ::: alias_does_not_work_over_ssh
|
||||
env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
|
||||
env_parallel -S fish@lo func_echo ::: function_works_over_ssh
|
||||
EOS
|
||||
|
||||
echo
|
||||
echo "### csh environment"
|
||||
# http://hyperpolyglot.org/unix-shells
|
||||
# makealias:
|
||||
# alias quote "/bin/sed -e 's/\\!/\\\\\!/g' -e 's/'\\\''/'\\\'\\\\\\\'\\\''/g' -e 's/^/'\''/' -e 's/"\$"/'\''/'"
|
||||
# alias makealias "quote | /bin/sed 's/^/alias \!:1 /' \!:2*"
|
||||
|
||||
stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
|
||||
set myvar = "myvar works"
|
||||
set funky = "`perl -e 'print pack q(c*), 1..255'`"
|
||||
|
|
@ -258,6 +274,7 @@ set myarray = ('' 'array_val2' '3' '' '5')
|
|||
#assocarr[a]=assoc_val_a
|
||||
#assocarr[b]=assoc_val_b
|
||||
alias alias_echo echo 3 arg;
|
||||
alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$funky"-funky'
|
||||
|
||||
#function func_echo
|
||||
# echo $argv;
|
||||
|
|
@ -267,31 +284,42 @@ alias alias_echo echo 3 arg;
|
|||
# echo Funky-"$funky"-funky
|
||||
#end
|
||||
|
||||
# s/'/'"'"'/g;
|
||||
# ' => \047 " => \042
|
||||
# s/\047/\047\042\047\042\047/g;
|
||||
# Quoted: s/\\047/\\047\\042\\047\\042\\047/g\;
|
||||
# ALIAS TO EXPORT ALIASES:
|
||||
|
||||
# s/^(\S+)(\s+)\((.*)\)/\1\2\3/
|
||||
# \047 => '
|
||||
# s/^(\S+)(\s+)\((.*)\)/\1\2\3/;
|
||||
# Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;
|
||||
# Quote ' by putting it inside "
|
||||
# s/'/'"'"'/g;
|
||||
# ' => \047 " => \042
|
||||
# s/\047/\047\042\047\042\047/g;
|
||||
# Quoted: s/\\047/\\047\\042\\047\\042\\047/g\;
|
||||
|
||||
# s/^(\S+)(\s+)(.*)/\1\2'\3'/
|
||||
# \047 => '
|
||||
# s/^(\S+)(\s+)(.*)/\1\2\047\3\047/;
|
||||
# Quoted: s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;
|
||||
# Remove () from second column
|
||||
# s/^(\S+)(\s+)\((.*)\)/\1\2\3/
|
||||
# \047 => '
|
||||
# s/^(\S+)(\s+)\((.*)\)/\1\2\3/;
|
||||
# Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;
|
||||
|
||||
# s/^/\001alias /;
|
||||
# Quoted: s/\^/\\001alias\ /\;
|
||||
# Add ' around second column
|
||||
# s/^(\S+)(\s+)(.*)/\1\2'\3'/
|
||||
# \047 => '
|
||||
# s/^(\S+)(\s+)(.*)/\1\2\047\3\047/;
|
||||
# Quoted: s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;
|
||||
|
||||
alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;`"'
|
||||
# Quote ! as \!
|
||||
# s/\!/\\\!/g;
|
||||
# Quoted: s/\\\!/\\\\\\\!/g;
|
||||
|
||||
# Prepend with "\nalias "
|
||||
# s/^/\001alias /;
|
||||
# Quoted: s/\^/\\001alias\ /\;
|
||||
|
||||
alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
|
||||
|
||||
alias pp 'parallel \!*'
|
||||
|
||||
env_parallel alias_echo ::: alias_works
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel alias_echo_var ::: alias_var_does_not_work
|
||||
env_parallel func_echo ::: function_does_not_work
|
||||
env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh
|
||||
env_parallel -S csh@lo func_echo ::: function_works_over_ssh
|
||||
env_parallel -S csh@lo alias_echo_var ::: alias_var_does_not_work
|
||||
env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh
|
||||
EOS
|
||||
|
||||
|
|
|
|||
24
testsuite/tests-to-run/parallel-local-ssh5.sh
Normal file
24
testsuite/tests-to-run/parallel-local-ssh5.sh
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
|
||||
# SSH only allowed to localhost/lo
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj2 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
|
||||
seq 10000 | xargs | parallel --pipe -S 10/localhost cat 2>/dev/null | wc
|
||||
|
||||
echo 'bug #36707: --controlmaster eats jobs'
|
||||
seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}
|
||||
|
||||
echo '### --ssh lsh'
|
||||
parallel --ssh lsh -S lo echo ::: OK
|
||||
echo OK | parallel --ssh lsh --pipe -S csh@lo cat
|
||||
# Todo rsync/trc csh@lo
|
||||
# Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler
|
||||
|
||||
echo '### --ssh autossh - add commands that fail here'
|
||||
export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0;
|
||||
parallel -S lo echo ::: OK;
|
||||
echo OK | parallel --pipe -S lo cat;
|
||||
parallel -S lo false ::: a || echo OK should fail;
|
||||
touch foo_autossh; stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh; rm foo_autossh*;
|
||||
|
||||
EOF
|
||||
|
|
@ -1,12 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
|
||||
seq 10000 | xargs | parallel --pipe -S 10/localhost cat 2>/dev/null | wc
|
||||
|
||||
echo 'bug #36707: --controlmaster eats jobs'
|
||||
seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### -L -n with pipe'
|
||||
seq 14 | parallel --pipe -k -L 3 -n 2 'cat;echo 6 Ln line record'
|
||||
|
||||
|
|
|
|||
|
|
@ -183,31 +183,29 @@ Funky-
|
|||
|
||||
0 updates are security updates.
|
||||
|
||||
3 arg alias_does_not_work
|
||||
3 arg alias_works
|
||||
function_works
|
||||
|
||||
myenvvar works
|
||||
Funky--funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€<7F>‚ƒ„…†‡ˆ‰Š‹Œ<E280B9>Ž<EFBFBD><C5BD>‘’“”•–—˜™š›œ<E280BA>žŸ ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky
|
||||
Array index out of bounds
|
||||
fish: echo $myarray[2]
|
||||
^
|
||||
in function “func_echoâ€<C3A2>
|
||||
called on standard input
|
||||
with parameter list “function_worksâ€<C3A2>
|
||||
|
||||
3 arg alias_does_not_work_over_ssh
|
||||
|
||||
|
||||
Funky--funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
|
||||
|
||||
|
||||
3 arg alias_works_over_ssh
|
||||
function_works_over_ssh
|
||||
|
||||
myenvvar works
|
||||
|
||||
|
||||
|
||||
Funky--funky
|
||||
Funky--funky
|
||||
Array index out of bounds
|
||||
fish: echo $myarray[2]
|
||||
^
|
||||
in function “func_echoâ€<C3A2>
|
||||
called on standard input
|
||||
with parameter list “function_works_over_sshâ€<C3A2>
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
|
||||
|
||||
|
||||
|
||||
### csh environment
|
||||
|
|
@ -218,4 +216,10 @@ in function “func_echo”
|
|||
|
||||
Warning: no access to tty (Bad file descriptor).
|
||||
Thus no job control in this shell.
|
||||
setenv: Too many arguments.
|
||||
3 arg alias_works
|
||||
|
||||
myvar: Undefined variable.
|
||||
func_echo: Command not found.
|
||||
3 arg alias_works_over_ssh
|
||||
myvar: Undefined variable.
|
||||
func_echo: Command not found.
|
||||
|
|
|
|||
86
testsuite/wanted-results/parallel-local-ssh5
Normal file
86
testsuite/wanted-results/parallel-local-ssh5
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
|
||||
### bug #36595: silent loss of input with --pipe and --sshlogin
|
||||
seq 10000 | xargs | parallel --pipe -S 10/localhost cat 2>/dev/null | wc
|
||||
1 10000 48894
|
||||
echo 'bug #36707: --controlmaster eats jobs'
|
||||
bug #36707: --controlmaster eats jobs
|
||||
seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}
|
||||
OK1
|
||||
OK2
|
||||
echo '### --ssh lsh'
|
||||
### --ssh lsh
|
||||
parallel --ssh lsh -S lo echo ::: OK
|
||||
OK
|
||||
echo OK | parallel --ssh lsh --pipe -S csh@lo cat
|
||||
OK
|
||||
# Todo rsync/trc csh@lo
|
||||
# Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler
|
||||
echo '### --ssh autossh - add commands that fail here'
|
||||
### --ssh autossh - add commands that fail here
|
||||
export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0; parallel -S lo echo ::: OK; echo OK | parallel --pipe -S lo cat; parallel -S lo false ::: a || echo OK should fail; touch foo_autossh; stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh; rm foo_autossh*;
|
||||
OK
|
||||
OK
|
||||
/usr/lib/autossh/autossh: invalid option -- '-'
|
||||
usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS]
|
||||
|
||||
-M specifies monitor port. Overrides the environment
|
||||
variable AUTOSSH_PORT. 0 turns monitoring loop off.
|
||||
Alternatively, a port for an echo service on the remote
|
||||
machine may be specified. (Normally port 7.)
|
||||
-f run in background (autossh handles this, and does not
|
||||
pass it to ssh.)
|
||||
-V print autossh version and exit.
|
||||
|
||||
Environment variables are:
|
||||
AUTOSSH_GATETIME - how long must an ssh session be established
|
||||
before we decide it really was established
|
||||
(in seconds). Default is 30 seconds; use of -f
|
||||
flag sets this to 0.
|
||||
AUTOSSH_LOGFILE - file to log to (default is to use the syslog
|
||||
facility)
|
||||
AUTOSSH_LOGLEVEL - level of log verbosity
|
||||
AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds)
|
||||
AUTOSSH_MAXSTART - max times to restart (default is no limit)
|
||||
AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes)
|
||||
AUTOSSH_PATH - path to ssh if not default
|
||||
AUTOSSH_PIDFILE - write pid to this file
|
||||
AUTOSSH_POLL - how often to check the connection (seconds)
|
||||
AUTOSSH_FIRST_POLL - time before first connection check (seconds)
|
||||
AUTOSSH_PORT - port to use for monitor connection
|
||||
AUTOSSH_DEBUG - turn logging to maximum verbosity and log to
|
||||
stderr
|
||||
|
||||
rsync: connection unexpectedly closed (0 bytes received so far) [sender]
|
||||
rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.1]
|
||||
/usr/lib/autossh/autossh: invalid option -- '-'
|
||||
usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS]
|
||||
|
||||
-M specifies monitor port. Overrides the environment
|
||||
variable AUTOSSH_PORT. 0 turns monitoring loop off.
|
||||
Alternatively, a port for an echo service on the remote
|
||||
machine may be specified. (Normally port 7.)
|
||||
-f run in background (autossh handles this, and does not
|
||||
pass it to ssh.)
|
||||
-V print autossh version and exit.
|
||||
|
||||
Environment variables are:
|
||||
AUTOSSH_GATETIME - how long must an ssh session be established
|
||||
before we decide it really was established
|
||||
(in seconds). Default is 30 seconds; use of -f
|
||||
flag sets this to 0.
|
||||
AUTOSSH_LOGFILE - file to log to (default is to use the syslog
|
||||
facility)
|
||||
AUTOSSH_LOGLEVEL - level of log verbosity
|
||||
AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds)
|
||||
AUTOSSH_MAXSTART - max times to restart (default is no limit)
|
||||
AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes)
|
||||
AUTOSSH_PATH - path to ssh if not default
|
||||
AUTOSSH_PIDFILE - write pid to this file
|
||||
AUTOSSH_POLL - how often to check the connection (seconds)
|
||||
AUTOSSH_FIRST_POLL - time before first connection check (seconds)
|
||||
AUTOSSH_PORT - port to use for monitor connection
|
||||
AUTOSSH_DEBUG - turn logging to maximum verbosity and log to
|
||||
stderr
|
||||
|
||||
rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
|
||||
rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.1]
|
||||
|
|
@ -1,12 +1,3 @@
|
|||
echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
|
||||
### bug #36595: silent loss of input with --pipe and --sshlogin
|
||||
seq 10000 | xargs | parallel --pipe -S 10/localhost cat 2>/dev/null | wc
|
||||
1 10000 48894
|
||||
echo 'bug #36707: --controlmaster eats jobs'
|
||||
bug #36707: --controlmaster eats jobs
|
||||
seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}
|
||||
OK1
|
||||
OK2
|
||||
echo '### -L -n with pipe'
|
||||
### -L -n with pipe
|
||||
seq 14 | parallel --pipe -k -L 3 -n 2 'cat;echo 6 Ln line record'
|
||||
|
|
|
|||
|
|
@ -735,14 +735,14 @@ _
|
|||
|
||||
/bin/bash: my_func2: command not found
|
||||
parallel -vv -S $SERVER1 echo ::: bar
|
||||
ssh parallel@lo exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my($in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;eval$eval;'\' BASE64;
|
||||
ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my($in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;eval$eval;'\' BASE64;
|
||||
bar
|
||||
my_func3() {
|
||||
echo in my_func $1 > $1.out
|
||||
}
|
||||
export -f my_func3
|
||||
parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file
|
||||
( ssh parallel@lo mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh ./abc-file parallel@lo:./.TMPWORKDIR );ssh parallel@lo exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my($in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;eval$eval;'\' BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh parallel@lo:./abc-file.out ./.;ssh parallel@lo \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh parallel@lo \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh parallel@lo rm -rf .TMPWORKDIR; exit $_EXIT_status;
|
||||
( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my($in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;eval$eval;'\' BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
|
||||
cat num1000000 | parallel --pipe wc
|
||||
165668 165668 1048571
|
||||
149796 149796 1048572
|
||||
|
|
|
|||
Loading…
Reference in a new issue