tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 22:17:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Commands longer than 1000 chars are bzip2'ed + hex encoded due to csh limit.

Browse source
This commit is contained in:
Ole Tange 2015-01-02 13:07:44 +01:00
parent 3071e28c15
commit 3d2ecc7651
17 changed files with 276 additions and 246 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
doc/release_new_version
View file

@ -236,6 +236,10 @@ Haiku of the month:
New in this release: New in this release:
* Remote jobs now sends stderr (standard error) to stderr (standard error) instead of stdout (standard output)
* Remote execution is now packed using hex encoded bzip2. That means that big environments (app. 250 KB) can be transferred.
* A semibig refactoring of big functions. All non-trivial functions are now less than 100 lines. The refactoring makes this release beta quality. * A semibig refactoring of big functions. All non-trivial functions are now less than 100 lines. The refactoring makes this release beta quality.
* GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137 * GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137
@ -246,8 +250,12 @@ New in this release:
* GNU Parallel: Open Source For You (OSFY) magazine, October 2013 edition http://www.shakthimaan.com/posts/2014/11/27/gnu-parallel/news.html * GNU Parallel: Open Source For You (OSFY) magazine, October 2013 edition http://www.shakthimaan.com/posts/2014/11/27/gnu-parallel/news.html
* Multithreaded Encryption and Compression http://www.krazyworks.com/multithreaded-encryption-and-compression/
* コマンドを並列に実行するGNU parallelがとても便利 http://bicycle1885.hatenablog.com/entry/2014/08/10/143612 * コマンドを並列に実行するGNU parallelがとても便利 http://bicycle1885.hatenablog.com/entry/2014/08/10/143612
* GNU Parallel on OSX and Linux https://darknightelf.wordpress.com/2015/01/01/gnu-parallel-on-osx/
* Bug fixes and man page updates. * Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane. GNU Parallel - For people who live life in the parallel lane.

361
src/parallel
View file

@ -1124,90 +1124,7 @@ sub parse_env_var {
# %ENV # %ENV
# Returns: N/A # Returns: N/A
$Global::envvar = ""; $Global::envvar = "";
$Global::envwarn = "";
my @vars = ('parallel_bash_environment');
for my $varstring (@opt::env) {
# Split up --env VAR1,VAR2
push @vars, split /,/, $varstring;
}
if(grep { /^_$/ } @vars) {
# --env _
# Include all vars that are not in a clean environment
if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) {
my @ignore = <$vars_fh>;
chomp @ignore;
my %ignore;
@ignore{@ignore} = @ignore;
close $vars_fh;
push @vars, grep { not defined $ignore{$_} } keys %ENV;
@vars = grep { not /^_$/ } @vars;
} else {
::error("Run '$Global::progname --record-env' in a clean environment first.\n");
::wait_and_exit(255);
}
}
# Duplicate vars as BASH functions to include post-shellshock functions.
# So --env myfunc should also look for BASH_FUNC_myfunc()
@vars = map { $_, "BASH_FUNC_$_()", "BASH_FUNC_$_%%" } @vars;
# Keep only defined variables
@vars = grep { defined($ENV{$_}) } @vars;
# Pre-shellshock style bash function:
# myfunc=() { echo myfunc
# }
# Post-shellshock style bash function:
# BASH_FUNC_myfunc()=() { echo myfunc
# }
# BASH_FUNC_myfunc%%=() { echo myfunc
# }
my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars;
if(@bash_functions) {
# Functions are not supported for all shells
if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
::warning("Shell functions may not be supported in $Global::shell\n");
}
}
# Pre-shellschock names are without ()
my @bash_pre_shellshock = grep { not /\(\)|%%/ } @bash_functions;
# Post-shellschock names are with ()
my @bash_post_shellshock = grep { /\(\)|%%/ } @bash_functions;
my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) }
grep { not /^parallel_bash_environment$/ } @non_functions);
my @qbash = (map { my $a=$_; "export $a=" . env_quote($ENV{$a}) }
@non_functions, @bash_pre_shellshock);
push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_pre_shellshock;
push @qbash, map { /BASH_FUNC_(.*)(\(\)|%%)/; "$1 $ENV{$_}" } @bash_post_shellshock;
#ssh -tt -oLogLevel=quiet lo 'eval `echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ BASH_FUNC_myfunc\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ a\"'
#'\"\\\}\ \|\|\ myfunc\(\)\ \{\ \ echo\ a'
#'\}\ \;myfunc\ 1;
# Check if any variables contain \n
if(my @v = map { s/BASH_FUNC_(.*)(\(\)|%%)/$1/; $_ } grep { $ENV{$_}=~/\n/ } @vars) {
# \n is bad for csh and will cause it to fail.
$Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn;
}
if(not @qcsh) { push @qcsh, "true"; }
if(not @qbash) { push @qbash, "true"; }
# Create lines like:
# echo $SHELL | grep "/t\\{0,1\\}csh" >/dev/null && setenv V1 val1 &&
# setenv V2 val2 || export V1=val1 && export V2=val2 ; echo "$V1$V2"
if(@vars) {
$Global::envvar .=
join"",
(q{echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null && }
. join(" && ", @qcsh)
. q{ || }
. join(" && ", @qbash)
.q{;});
if($ENV{'parallel_bash_environment'}) {
$Global::envvar .= 'eval "$parallel_bash_environment";'."\n";
}
}
$Global::envvarlen = length $Global::envvar; $Global::envvarlen = length $Global::envvar;
} }
@ -1554,6 +1471,18 @@ sub shellwords {
return Text::ParseWords::shellwords(@_); return Text::ParseWords::shellwords(@_);
} }
sub perl_quote_scalar {
# Quote the string so perl's eval will not expand any special chars
# Inputs:
# $string = string to be quoted
# Returns:
# $shell_quoted = string quoted with \ as needed by perl's eval
my $a = $_[0];
if(defined $a) {
$a =~ s/[\\\"\$\@]/\\$&/go;
}
return $a;
}
sub __FILEHANDLES__ {} sub __FILEHANDLES__ {}
@ -3179,6 +3108,24 @@ sub undef_as_empty {
return $a ? $a : ""; return $a ? $a : "";
} }
sub spacefree {
# Remove comments and spaces
# Inputs:
# $spaces = keep 1 space?
# $s = string to remove spaces from
# Returns:
# $s = with spaces removed
my $spaces = shift;
my $s = shift;
$s =~ s/#.*//mg;
if($spaces) {
$s =~ s/\s+/ /mg;
} else {
$s =~ s/\s//mg;
}
return $s;
}
{ {
my $hostname; my $hostname;
sub hostname { sub hostname {
@ -5794,7 +5741,7 @@ sub wrapped {
# $self->{'wrapped'} = the command wrapped with the above # $self->{'wrapped'} = the command wrapped with the above
my $self = shift; my $self = shift;
if(not defined $self->{'wrapped'}) { if(not defined $self->{'wrapped'}) {
my $command = $Global::envvar.$self->replaced(); my $command = $self->replaced();
if($opt::shellquote) { if($opt::shellquote) {
# Prepend echo # Prepend echo
# and quote twice # and quote twice
@ -5871,8 +5818,73 @@ sub sshlogin {
return $self->{'sshlogin'}; return $self->{'sshlogin'};
} }
sub string_zip_hex {
# Pipe string through 'bzip2 -9' and hex encode it into 1000 byte
# blocks.
# 1000 bytes is the largest word size csh supports
# Input:
# @strings = to be encoded
# Returns:
# @hex = 1000 byte block
my($zipin_fh, $zipout_fh,@hex);
::open3($zipin_fh,$zipout_fh,">&STDERR","bzip2 -9");
if(fork) {
close $zipin_fh;
# Chop hex into 1000 byte blocks
@hex = unpack("(H1000)*",join"",<$zipout_fh>);
close $zipout_fh;
} else {
close $zipout_fh;
print $zipin_fh @_;
close $zipin_fh;
exit;
}
::debug("hex","Orig:@_\nAs hex:@hex\n");
return @hex;
}
sub hex_zip_eval {
# Script that:
# * reads hexstrings from @ARGV
# * unhexes them
# * pipes through 'bzip2 -dc'
# * evals the result
# Reverse of string_bz_hex + eval
# Will be wrapped in ' so single quote is forbidden
# Returns:
# $script = 1-liner for perl -e
my $script = ::spacefree(0,q{
@GNU_Parallel=("use","IPC::Open3;");
eval "@GNU_Parallel";
$SIG{CHLD}="IGNORE";
# Search for bzip2. Not found => use default path
my $zip = (grep { -x $_ } "/usr/local/bin/bzip2")[0] || "bzip2";
# $in = stdin on $zip, $out = stdout from $zip
my($in, $out,$eval);
open3($in,$out,">&STDERR",$zip,"-dc");
if(my $perlpid = fork) {
close $in;
$eval = join "", <$out>;
close $out;
} else {
close $out;
# Pipe decoded hex into 'bzip2 -dc'
print $in (pack("H*",join"",@ARGV));
close $in;
exit;
}
wait;
eval $eval;
});
::debug("hex",$script,"\n");
return $script;
}
sub sshlogin_wrap { sub sshlogin_wrap {
# Wrap the command with the commands needed to run remotely # Wrap the command with the commands needed to run remotely
# Input:
# $command = command to run
# Returns: # Returns:
# $self->{'sshlogin_wrap'} = command wrapped with ssh+transfer commands # $self->{'sshlogin_wrap'} = command wrapped with ssh+transfer commands
sub monitor_parent_sshd_script { sub monitor_parent_sshd_script {
@ -5882,20 +5894,17 @@ sub sshlogin_wrap {
# If its parent is ssh: all good # If its parent is ssh: all good
# If its parent is init(1): ssh died, so kill children # If its parent is init(1): ssh died, so kill children
my $monitor_parent_sshd_script; my $monitor_parent_sshd_script;
if(not $monitor_parent_sshd_script) { if(not $monitor_parent_sshd_script) {
$monitor_parent_sshd_script = $monitor_parent_sshd_script =
# This will be packed in ', so only use " # This will be packed in ', so only use "
q{ ::spacefree(0,q{
# If packed as hex, unpack
# if($ARGV[0] =~ s/^0x//) {
# $ARGV[0] =~ s/(..)/chr(hex $1)/ge;
#}
$SIG{CHLD} = sub { $done = 1; }; $SIG{CHLD} = sub { $done = 1; };
$pid = fork; $pid = fork;
unless($pid) { unless($pid) {
# Make own process group to be able to kill HUP it later # Make own process group to be able to kill HUP it later
setpgrp; setpgrp;
exec $ENV{SHELL}, "-c", @ARGV; exec $ENV{SHELL}, "-c", ($bashfunc."@ARGV");
die "exec: $!\n"; die "exec: $!\n";
} }
do { do {
@ -5908,15 +5917,82 @@ sub sshlogin_wrap {
kill(SIGHUP, -${pid}) unless $done; kill(SIGHUP, -${pid}) unless $done;
wait; wait;
exit ($?&127 ? 128+($?&127) : 1+$?>>8) exit ($?&127 ? 128+($?&127) : 1+$?>>8)
}; });
$monitor_parent_sshd_script =~ s/#.*//mg;
$monitor_parent_sshd_script =~ s/\s//mg;
} }
return $monitor_parent_sshd_script; return $monitor_parent_sshd_script;
} }
#sub hex_pack {
#return unpack("H*","@_"); sub vars_to_export {
#} # Uses:
# @opt::env
my @vars = ("parallel_bash_environment");
for my $varstring (@opt::env) {
# Split up --env VAR1,VAR2
push @vars, split /,/, $varstring;
}
if(grep { /^_$/ } @vars) {
# --env _
# Include all vars that are not in a clean environment
if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) {
my @ignore = <$vars_fh>;
chomp @ignore;
my %ignore;
@ignore{@ignore} = @ignore;
close $vars_fh;
push @vars, grep { not defined $ignore{$_} } keys %ENV;
@vars = grep { not /^_$/ } @vars;
} else {
::error("Run '$Global::progname --record-env' in a clean environment first.\n");
::wait_and_exit(255);
}
}
# Duplicate vars as BASH functions to include post-shellshock functions (v1+v2)
# So --env myfunc should also look for BASH_FUNC_myfunc()
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
# Keep only defined variables
return grep { defined($ENV{$_}) } @vars;
}
sub env_as_eval {
# Returns:
# $eval = '$ENV{"..."}=...; ...'
my @vars = vars_to_export();
my $csh_friendly = not grep { /\n/ } @ENV{@vars};
my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars;
# eval of @envset will set %ENV
my $envset = join"", map {
'$ENV{"'.::perl_quote_scalar($_).'"}="'. ::perl_quote_scalar($ENV{$_}).'";'; } @non_functions;
# running @bashfunc on the command line, will set the functions
my @bashfunc = map {
my $v=$_; s/BASH_FUNC_(.*)(\(\)|%%)/$1/; "$_$ENV{$v};" } @bash_functions;
# eval $bashfuncset will set $bashfunc
my $bashfuncset;
if(@bashfunc) {
# Functions are not supported for all shells
if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
::warning("Shell functions may not be supported in $Global::shell\n");
}
$bashfuncset =
'@bash_functions=qw('."@bash_functions".");".
::spacefree(1,q{
if($ENV{"SHELL"}=~/csh/) {
print STDERR "CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset @bash_functions\n";
exec "false";
}
}).
"\n".'$bashfunc = "'.::perl_quote_scalar("@bashfunc").'";';
} else {
$bashfuncset = '$bashfunc = "";'
}
if($ENV{"parallel_bash_environment"}) {
$bashfuncset .= '$bashfunc .= "eval\ \"\$parallel_bash_environment\"\;";';
}
::debug("hex",$envset,$bashfuncset,"\n");
return $csh_friendly,$envset,$bashfuncset;
}
my $self = shift; my $self = shift;
my $command = shift; my $command = shift;
@ -5925,9 +6001,8 @@ sub sshlogin_wrap {
my $sshcmd = $sshlogin->sshcommand(); my $sshcmd = $sshlogin->sshcommand();
my $serverlogin = $sshlogin->serverlogin(); my $serverlogin = $sshlogin->serverlogin();
my ($pre,$post,$cleanup)=("","",""); my ($pre,$post,$cleanup)=("","","");
if($serverlogin eq ":") { if($serverlogin eq ":") {
# No transfer neeeded # No transfer/environment neeeded
$self->{'sshlogin_wrap'} = $command; $self->{'sshlogin_wrap'} = $command;
} else { } else {
# --transfer # --transfer
@ -5944,74 +6019,34 @@ sub sshlogin_wrap {
# otherwise use 'export' # otherwise use 'export'
# We cannot use parse_env_var(), as PARALLEL_SEQ changes # We cannot use parse_env_var(), as PARALLEL_SEQ changes
# for each command # for each command
my $parallel_env = $ENV{'PARALLEL_SEQ'} = $self->seq();
($Global::envwarn $ENV{'PARALLEL_PID'} = $$;
. q{ 'eval `echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null } my $pwd = "";
. q{ && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; }
. q{ setenv PARALLEL_PID '$PARALLEL_PID' }
. q{ || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; }
. q{ PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' });
my $remote_pre = "";
my $ssh_options = "";
if(($opt::pipe or $opt::pipepart) and $opt::ctrlc
or
not ($opt::pipe or $opt::pipepart) and not $opt::noctrlc) {
# Propagating CTRL-C to kill remote jobs requires
# remote jobs to be run with a terminal.
# $ssh_options = "-tt -oLogLevel=quiet";
$ssh_options = "";
# tty - check if we have a tty.
# stty:
# -onlcr - make output 8-bit clean
# isig - pass CTRL-C as signal
# -echo - do not echo input
# $remote_pre .= ::shell_quote_scalar('tty >/dev/null && stty isig -onlcr -echo;');
}
if($opt::workdir) { if($opt::workdir) {
my $wd = ::shell_quote_file($self->workdir()); # Create remote workdir if needed. Then cd to it.
$remote_pre .= ::shell_quote_scalar("mkdir -p ") . $wd . my $wd = $self->workdir();
::shell_quote_scalar("; cd ") . $wd . $pwd = qq{system("mkdir","-p","--","$wd") || chdir "$wd" ||}.
# exit 255 (instead of exec false) would be the correct thing, qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;};
# but that fails on tcsh $command = "cd $wd;$command";
::shell_quote_scalar(qq{ || exec false;}); }
my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
my $remote_command = $pwd.$envset.$bashfuncset.
'@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script();
my $quoted_remote_command = "exec perl -e ".::shell_quote_scalar(::shell_quote_scalar($remote_command));
if(length $quoted_remote_command > 1000
or
not $csh_friendly
or
$command =~ "\n") {
# csh does not deal well with > 1000 chars in one word
# csh does not deal well with $ENV with \n
$quoted_remote_command = "exec perl -e '".::shell_quote_scalar(hex_zip_eval())."' ".
join" ",string_zip_hex($remote_command);
} }
my $signal_script = "exec perl -e '".monitor_parent_sshd_script()."' ";
# TODO clean this up
# TODO Maybe env vars should be set in the perl script
# It works on Linux but not Solaris
# Finishes on Solaris, but wrong exit code:
# $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)};
# Hangs on Solaris, but correct exit code on Linux:
# $SIG{CHLD} = sub { $done = 1 };
# $p->poll;
# $SIG{CHLD} = sub { $done = 1; ($^O eq "solaris") && exit 0 };
# $SIG{CHLD} = sub { $done = 1; (1) && exit ($?&127 ? 128+($?&127) : 1+$?>>8); };
# $SIG{CHLD} = sub { $done = 1; exit ($?&127 ? 128+($?&127) : 1+$?>>8); };
# -> Linux: "" +script
# -> Solaris: -tt -script
# --ctrl-c => -tt
# q{
# ($^O eq "solaris") && exec $ENV{SHELL}, "-c", @ARGV;
# $|=1;
# use IO::Poll;
# $SIG{CHLD} = sub {$done = 1; };
# $pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"}
# $p = IO::Poll->new;
# $p->mask(STDOUT, POLLHUP);
# $p->poll;
# kill SIGHUP, -${pid} unless $done;
# wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8)
# } . "' ";
# $signal_script =~ s/\s+/ /g;
$self->{'sshlogin_wrap'} = $self->{'sshlogin_wrap'} =
($pre ($pre
. "$sshcmd $ssh_options $serverlogin $parallel_env " . "$sshcmd $serverlogin "
. $remote_pre . $quoted_remote_command
. ::shell_quote_scalar($signal_script . ::shell_quote_scalar($command))
# . ::shell_quote_scalar($signal_script . hex_pack($command))
# . ::shell_quote_scalar($command)
. ";" . ";"
. $post); . $post);
} }
@ -8527,5 +8562,5 @@ sub mkdir_or_die {
} }
# Keep perl -w happy # Keep perl -w happy
$opt::x = $Semaphore::timeout = $Semaphore::wait = $opt::ctrlc = $opt::x = $Semaphore::timeout = $Semaphore::wait =
$Job::file_descriptor_warning_printed = 0; $Job::file_descriptor_warning_printed = 0;

2
testsuite/tests-to-run/parallel-freebsd.sh
View file

@ -54,8 +54,6 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
export -f funky; export -f funky;
parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
EOF EOF
VBoxManage controlvm FreeBSD71 savestate VBoxManage controlvm FreeBSD71 savestate

3
testsuite/tests-to-run/parallel-local-ssh1.sh
View file

@ -9,6 +9,9 @@ cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout paral
echo '### Stop if all hosts are filtered and there are no hosts left to run on' echo '### Stop if all hosts are filtered and there are no hosts left to run on'
stdout parallel --filter-hosts -S no-such.host echo ::: 1 stdout parallel --filter-hosts -S no-such.host echo ::: 1
echo '### Can csh propagate a variable containing \n';
export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
echo '** test_zsh' echo '** test_zsh'
FOO=test_zsh parallel --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME' FOO=test_zsh parallel --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME'

2
testsuite/tests-to-run/parallel-remote1.sh
View file

@ -11,7 +11,7 @@ echo 'TODO test ssh with > 9 simultaneous'
echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1 echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1
echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2
chmod 755 /tmp/myssh1 /tmp/myssh2 chmod 755 /tmp/myssh1 /tmp/myssh2
seq 1 100 | parallel --sshdelay 0.05 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo seq 1 100 | parallel --sshdelay 0.01 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj1 -k -L1 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj1 -k -L1
echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host'

24
testsuite/tests-to-run/sem01.sh
View file

@ -1,9 +1,9 @@
#!/bin/bash #!/bin/bash
echo '### Test mutex. This should not mix output' echo '### Test mutex. This should not mix output';
parallel -u --semaphore seq 1 10 '|' pv -qL 20 parallel -u --semaphore seq 1 10 '|' pv -qL 20;
parallel -u --semaphore seq 11 20 '|' pv -qL 100 parallel -u --semaphore seq 11 20 '|' pv -qL 100;
parallel --semaphore --wait parallel --semaphore --wait;
echo done echo done
echo '### Test semaphore 2 jobs running simultaneously' echo '### Test semaphore 2 jobs running simultaneously'
@ -27,17 +27,17 @@ echo done
echo '### Test similar example as from man page - run 2 jobs simultaneously' echo '### Test similar example as from man page - run 2 jobs simultaneously'
echo 'Expect done: 1 2 5 3 4' echo 'Expect done: 1 2 5 3 4'
for i in 5 1 2 3 4 ; do for i in 5 1 2 3 4 ; do
sleep 0.2 sleep 0.2;
echo Scheduling $i echo Scheduling $i;
sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i;
done done;
sem --wait sem --wait
echo '### Test --fg followed by --bg' echo '### Test --fg followed by --bg'
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30 parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30;
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30 parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30;
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30 parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30;
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30 parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30;
sem --wait sem --wait
echo '### Test bug #33621: --bg -p should give an error message' echo '### Test bug #33621: --bg -p should give an error message'

4
testsuite/tests-to-run/test15.sh
View file

@ -241,8 +241,8 @@ echo '### Test --nice locally'
parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b
echo '### Test --nice remote' echo '### Test --nice remote'
stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b |
| perl -pe 's/\S*parallel-server\S*/one-server/' perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
echo '### Test distribute arguments at EOF to 2 jobslots' echo '### Test distribute arguments at EOF to 2 jobslots'
seq 1 92 | parallel -j+0 -kX -s 100 echo seq 1 92 | parallel -j+0 -kX -s 100 echo

3
testsuite/tests-to-run/test19.sh
View file

@ -122,6 +122,7 @@ stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
# Should give: No such file or directory # Should give: No such file or directory
stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
echo 'Input for ssh' echo 'Input for ssh'
cat /tmp/myssh1-run /tmp/myssh2-run | perl -pe 's/(PID.)\d+/${1}00000/g;s/(SEQ[ =]|line)\d/$1X/g;' cat /tmp/myssh1-run /tmp/myssh2-run | perl -pe 's/(PID.)\d+/${1}00000/g;s/(SEQ[ =]|line)\d/$1X/g;' |
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
rm /tmp/myssh1-run /tmp/myssh2-run rm /tmp/myssh1-run /tmp/myssh2-run

20
testsuite/tests-to-run/test37.sh
View file

@ -39,27 +39,27 @@ parallel -v -J test_profile ::: <(echo a) <(echo b)
echo '### Test ugly quoting from profile file --plain' echo '### Test ugly quoting from profile file --plain'
parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b) parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b)
PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 -v ::: foo PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1 -v ::: foo
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo
echo '### Test quoting of $ in command from profile file' echo '### Test quoting of $ in command from profile file'
cat <<EOF >~/.parallel/test_profile cat <<EOF >~/.parallel/test_profile
-k --jobs 1 perl -pe \'\\\$a=1; print \\\$a\' -k --jobs 1 perl -pe \'\\\$a=1; print \\\$a\'
EOF EOF
parallel -v -J test_profile -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 '<(echo {})' ::: foo parallel -v -J test_profile -S ssh\ parallel\@$SERVER1 '<(echo {})' ::: foo
echo '### Test quoting of $ in command from profile file --plain' echo '### Test quoting of $ in command from profile file --plain'
parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 'cat <(echo {})' ::: foo parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1 'cat <(echo {})' ::: foo
echo '### Test quoting of $ in command from $PARALLEL' echo '### Test quoting of $ in command from $PARALLEL'
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo
echo '### Test quoting of $ in command from $PARALLEL --plain' echo '### Test quoting of $ in command from $PARALLEL --plain'
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1 -v 'cat <(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from profile file' echo '### Test quoting of space in arguments (-S) from profile file'
cat <<EOF >~/.parallel/test_profile cat <<EOF >~/.parallel/test_profile
-k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\' -k --jobs 1 -S ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\'
EOF EOF
parallel -v -J test_profile '<(echo {})' ::: foo parallel -v -J test_profile '<(echo {})' ::: foo
@ -67,17 +67,17 @@ echo '### Test quoting of space in arguments (-S) from profile file --plain'
parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from $PARALLEL' echo '### Test quoting of space in arguments (-S) from $PARALLEL'
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
echo '### Test quoting of space in long arguments (--sshlogin) from profile file' echo '### Test quoting of space in long arguments (--sshlogin) from profile file'
cat <<EOF >~/.parallel/test_profile cat <<EOF >~/.parallel/test_profile
# testprofile # testprofile
-k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\' -k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\'
EOF EOF
parallel -v -J test_profile '<(echo {})' ::: foo parallel -v -J test_profile '<(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from $PARALLEL' echo '### Test quoting of space in arguments (-S) from $PARALLEL'
PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
echo '### Test merging of profiles - sort needed because -k only works on the single machine' echo '### Test merging of profiles - sort needed because -k only works on the single machine'
echo --tag > ~/.parallel/test_tag echo --tag > ~/.parallel/test_tag

8
testsuite/tests-to-run/test61.sh
View file

@ -3,12 +3,14 @@
SERVER1=parallel-server3 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1
echo '### Test --return of weirdly named file' echo '### Test --return of weirdly named file'
stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b' stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b' |
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'; rm 'aa<${#}" b'
echo '### Test if remote login shell is csh' echo '### Test if remote login shell is csh'
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c |
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
echo '### Test {} multiple times in different commands' echo '### Test {} multiple times in different commands'
seq 10 | parallel -v -Xj1 echo {} \; echo {} seq 10 | parallel -v -Xj1 echo {} \; echo {}

5
testsuite/wanted-results/parallel-local-ssh1
View file

@ -3,6 +3,11 @@ echo '### Stop if all hosts are filtered and there are no hosts left to run on'
stdout parallel --filter-hosts -S no-such.host echo ::: 1 stdout parallel --filter-hosts -S no-such.host echo ::: 1
parallel: Warning: Removed no-such.host parallel: Warning: Removed no-such.host
parallel: Error: There are no hosts left to run on. parallel: Error: There are no hosts left to run on.
echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy
### Can csh propagate a variable containing \n
1
2
3
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
### bug #41805: Idea: propagate --env for parallel --number-of-cores ### bug #41805: Idea: propagate --env for parallel --number-of-cores
echo '** test_zsh' echo '** test_zsh'

14
testsuite/wanted-results/parallel-local-ssh2
View file

@ -12,30 +12,26 @@ echo '### --wd no-such-dir - csh'
### --wd no-such-dir - csh ### --wd no-such-dir - csh
stdout parallel --wd /no-such-dir -S csh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? stdout parallel --wd /no-such-dir -S csh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $?
mkdir: cannot create directory ‘/no-such-dir’: Permission denied mkdir: cannot create directory ‘/no-such-dir’: Permission denied
/no-such-dir: No such file or directory. parallel: Cannot chdir to /no-such-dir
Exit code 1 Exit code 1
echo '### --wd no-such-dir - tcsh' echo '### --wd no-such-dir - tcsh'
### --wd no-such-dir - tcsh ### --wd no-such-dir - tcsh
stdout parallel --wd /no-such-dir -S tcsh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? stdout parallel --wd /no-such-dir -S tcsh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $?
mkdir: cannot create directory ‘/no-such-dir’: Permission denied mkdir: cannot create directory ‘/no-such-dir’: Permission denied
/no-such-dir: No such file or directory. parallel: Cannot chdir to /no-such-dir
Exit code 1 Exit code 1
echo '### --wd no-such-dir - bash' echo '### --wd no-such-dir - bash'
### --wd no-such-dir - bash ### --wd no-such-dir - bash
stdout parallel --wd /no-such-dir -S parallel@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? stdout parallel --wd /no-such-dir -S parallel@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $?
mkdir: cannot create directory ‘/no-such-dir’: Permission denied mkdir: cannot create directory ‘/no-such-dir’: Permission denied
bash: line 0: cd: /no-such-dir: No such file or directory parallel: Cannot chdir to /no-such-dir
Exit code 1 Exit code 1
echo '### bug #42725: csh with \n in variables' echo '### bug #42725: csh with \n in variables'
### bug #42725: csh with \n in variables ### bug #42725: csh with \n in variables
not_csh() { echo This is not csh/tcsh; }; export -f not_csh; parallel --env not_csh -S csh@lo not_csh ::: 1; parallel --env not_csh -S tcsh@lo not_csh ::: 1; parallel --env not_csh -S parallel@lo not_csh ::: 1 not_csh() { echo This is not csh/tcsh; }; export -f not_csh; parallel --env not_csh -S csh@lo not_csh ::: 1; parallel --env not_csh -S tcsh@lo not_csh ::: 1; parallel --env not_csh -S parallel@lo not_csh ::: 1
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
This is not csh/tcsh This is not csh/tcsh
Badly placed ()'s. CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
}: Command not found. CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
Badly placed ()'s.
}: Command not found.
echo '### bug #43358: shellshock breaks exporting functions using --env' echo '### bug #43358: shellshock breaks exporting functions using --env'
### bug #43358: shellshock breaks exporting functions using --env ### bug #43358: shellshock breaks exporting functions using --env
echo shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@localhost funky ::: shellshock-hardened echo shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@localhost funky ::: shellshock-hardened

10
testsuite/wanted-results/parallel-local22
View file

@ -130,12 +130,10 @@ shell? /bin/bash -c cp `which yash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit e
which bash => shell path /bin/bash which bash => shell path /bin/bash
shell? /bin/bash -c cp `which zsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which; shell? /bin/bash -c cp `which zsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash which bash => shell path /bin/bash
MANPATH: Undefined variable.
/tmp/SHELL: -c: bad option(s) /tmp/SHELL: -c: bad option(s)
Local configuration error occurred. Local configuration error occurred.
Contact the systems administrator for further assistance. Contact the systems administrator for further assistance.
SHELL: applet not found SHELL: applet not found
MANPATH: Undefined variable.
echo '## Known shells -c' echo '## Known shells -c'
## Known shells -c ## Known shells -c
parallel -k "\`which {}\` -c 'parallel -Dinit echo ::: 1' | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh parallel -k "\`which {}\` -c 'parallel -Dinit echo ::: 1' | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
@ -168,12 +166,8 @@ shell? /bin/bash -c `which yash` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash which bash => shell path /bin/bash
shell? /bin/bash -c `which zsh` -c 'parallel -Dinit echo ::: 1' | grep which; shell? /bin/bash -c `which zsh` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash which bash => shell path /bin/bash
MANPATH: Undefined variable.
MANPATH: Undefined variable.
Local configuration error occurred. Local configuration error occurred.
Contact the systems administrator for further assistance. Contact the systems administrator for further assistance.
MANPATH: Undefined variable.
MANPATH: Undefined variable.
echo '## Known shells |' echo '## Known shells |'
## Known shells | ## Known shells |
parallel -k "echo 'parallel -Dinit echo ::: 1' | \`which {}\` | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh parallel -k "echo 'parallel -Dinit echo ::: 1' | \`which {}\` | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
@ -197,12 +191,8 @@ which static-sh => shell path /bin/static-sh
which tcsh => shell path /usr/bin/tcsh which tcsh => shell path /usr/bin/tcsh
which yash => shell path /usr/bin/yash which yash => shell path /usr/bin/yash
which zsh => shell path /usr/bin/zsh which zsh => shell path /usr/bin/zsh
MANPATH: Undefined variable.
MANPATH: Undefined variable.
Local configuration error occurred. Local configuration error occurred.
Contact the systems administrator for further assistance. Contact the systems administrator for further assistance.
MANPATH: Undefined variable.
MANPATH: Undefined variable.
echo '## Started directly from perl' echo '## Started directly from perl'
## Started directly from perl ## Started directly from perl
perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which

2
testsuite/wanted-results/test15
View file

@ -754,7 +754,7 @@ args on cmdline
\nice -n1 /bin/bash -c PAR=a\ bash\ -c\ \"echo\ \ \\\$PAR\ b\" \nice -n1 /bin/bash -c PAR=a\ bash\ -c\ \"echo\ \ \\\$PAR\ b\"
a b a b
### Test --nice remote ### Test --nice remote
ssh one-server 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ \\\\nice\\\ -n1\\\ /bin/bash\\\ -c\\\ PAR=a\\\\\\\ bash\\\\\\\ -c\\\\\\\ \\\\\\\"echo\\\\\\\ \\\\\\\ \\\\\\\\\\\\\\\$PAR\\\\\\\ b\\\\\\\"; ssh one-server exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
a b a b
### Test distribute arguments at EOF to 2 jobslots ### Test distribute arguments at EOF to 2 jobslots
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34

40
testsuite/wanted-results/test19
View file

@ -75,37 +75,29 @@ OK
ls: tmp/parallel.file*: No such file or directory ls: tmp/parallel.file*: No such file or directory
OK OK
Input for ssh Input for ssh
parallel@parallel-server1 mkdir -p ./. one-server mkdir -p ./.
-l parallel parallel-server1 rsync --server -lDrRze.iLsfx . ./. -l parallel one-server rsync --server -lDrRze.iLsfx . ./.
parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ X\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=X\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; exec perl -e '$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){setpgrp;exec$ENV{SHELL},"-c",@ARGV;die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)' cat\ tmp/parallel.file.\'' one-server exec perl -e '@u=("use","IPC::Open3;");eval"@u";$SIG{CHLD}='IGNORE';my($gzin_fh,$gzout_fh,$eval);my$gzpid=open3($gzin_fh,$gzout_fh,">&STDERR","gzip","-dc");if(my$perlpid=fork){close$gzin_fh;$eval=join"",<$gzout_fh>;close$gzout_fh;}else{close$gzout_fh;print$gzin_fh(pack("H*",join"",@ARGV));close$gzin_fh;exit;}wait;eval$eval;' hex
'\'newlineX\ \>\ tmp/parallel.file.\'' -l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
'\'newlineX.out\;cat\ tmp/parallel.file.\''
'\'newlineX\ \>\ tmp/parallel.file.\''
'\'newlineX.out2
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
'newlineX.out 'newlineX.out
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' -l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
'newlineX.out2 'newlineX.out2
parallel@parallel-server1 (rm -f ./tmp/parallel.file.' one-server (rm -f ./tmp/parallel.file.'
'newlineX; rmdir ./tmp/ ./ 2>/dev/null;) 'newlineX; rmdir ./tmp/ ./ 2>/dev/null;)
parallel@parallel-server1 (rm -f ./tmp/parallel.file.' one-server (rm -f ./tmp/parallel.file.'
'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;) 'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;)
parallel@parallel-server1 (rm -f ./tmp/parallel.file.' one-server (rm -f ./tmp/parallel.file.'
'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;) 'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;)
parallel@parallel-server2 mkdir -p ./. one-server mkdir -p ./.
-l parallel parallel-server2 rsync --server -lDrRze.iLsfx . ./. -l parallel one-server rsync --server -lDrRze.iLsfx . ./.
parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ X\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=X\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; exec perl -e '$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){setpgrp;exec$ENV{SHELL},"-c",@ARGV;die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)' cat\ tmp/parallel.file.\'' one-server exec perl -e '@u=("use","IPC::Open3;");eval"@u";$SIG{CHLD}='IGNORE';my($gzin_fh,$gzout_fh,$eval);my$gzpid=open3($gzin_fh,$gzout_fh,">&STDERR","gzip","-dc");if(my$perlpid=fork){close$gzin_fh;$eval=join"",<$gzout_fh>;close$gzout_fh;}else{close$gzout_fh;print$gzin_fh(pack("H*",join"",@ARGV));close$gzin_fh;exit;}wait;eval$eval;' hex
'\'newlineX\ \>\ tmp/parallel.file.\'' -l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
'\'newlineX.out\;cat\ tmp/parallel.file.\''
'\'newlineX\ \>\ tmp/parallel.file.\''
'\'newlineX.out2
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
'newlineX.out 'newlineX.out
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' -l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
'newlineX.out2 'newlineX.out2
parallel@parallel-server2 (rm -f ./tmp/parallel.file.' one-server (rm -f ./tmp/parallel.file.'
'newlineX; rmdir ./tmp/ ./ 2>/dev/null;) 'newlineX; rmdir ./tmp/ ./ 2>/dev/null;)
parallel@parallel-server2 (rm -f ./tmp/parallel.file.' one-server (rm -f ./tmp/parallel.file.'
'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;) 'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;)
parallel@parallel-server2 (rm -f ./tmp/parallel.file.' one-server (rm -f ./tmp/parallel.file.'
'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;) 'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;)

2
testsuite/wanted-results/test37
View file

@ -42,7 +42,7 @@ echo /dev/fd/62
/dev/fd/62 /dev/fd/62
echo foo echo foo
foo foo
ssh parallel@redhat9.tange.dk ssh centos3.tange.dk 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ perl\\\ -pe\\\ \\\"\\\\\\\$a=1\\\;\\\ print\\\ \\\\\\\$a\\\"\\\ \\\<\\\(echo\\\ foo\\\); perl -pe "\$a=1; print \$a" <(echo foo)
1foo 1foo
### Test quoting of $ in command from profile file ### Test quoting of $ in command from profile file
perl -pe '$a=1; print $a' <(echo foo) perl -pe '$a=1; print $a' <(echo foo)

12
testsuite/wanted-results/test61
View file

@ -1,15 +1,15 @@
echo '### Test --return of weirdly named file' echo '### Test --return of weirdly named file'
### Test --return of weirdly named file ### Test --return of weirdly named file
stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b' stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b' | perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'; rm 'aa<${#}" b'
ssh parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\>aa\\\\\\\<\\\\\\\$\\\\\\\{\\\\\\\#\\\\\\\}\\\\\\\"\\\\\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status; ssh one-server exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
echo '### Test if remote login shell is csh' echo '### Test if remote login shell is csh'
### Test if remote login shell is csh ### Test if remote login shell is csh
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c | perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ a\\\|\\\ wc\\\ -w; ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
3 3
ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ b\\\|\\\ wc\\\ -w; ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
3 3
ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ c\\\|\\\ wc\\\ -w; ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
3 3
echo '### Test {} multiple times in different commands' echo '### Test {} multiple times in different commands'
### Test {} multiple times in different commands ### Test {} multiple times in different commands