From 3d2ecc7651121dcf8acd5dfad48df107ef407ef8 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 2 Jan 2015 13:07:44 +0100 Subject: [PATCH] parallel: Commands longer than 1000 chars are bzip2'ed + hex encoded due to csh limit. --- doc/release_new_version | 8 + src/parallel | 361 ++++++++++-------- testsuite/tests-to-run/parallel-freebsd.sh | 2 - testsuite/tests-to-run/parallel-local-ssh1.sh | 3 + testsuite/tests-to-run/parallel-remote1.sh | 2 +- testsuite/tests-to-run/sem01.sh | 26 +- testsuite/tests-to-run/test15.sh | 4 +- testsuite/tests-to-run/test19.sh | 3 +- testsuite/tests-to-run/test37.sh | 20 +- testsuite/tests-to-run/test61.sh | 8 +- testsuite/wanted-results/parallel-local-ssh1 | 5 + testsuite/wanted-results/parallel-local-ssh2 | 14 +- testsuite/wanted-results/parallel-local22 | 10 - testsuite/wanted-results/test15 | 2 +- testsuite/wanted-results/test19 | 40 +- testsuite/wanted-results/test37 | 2 +- testsuite/wanted-results/test61 | 12 +- 17 files changed, 276 insertions(+), 246 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index a9e83c7a..c44e590a 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -236,6 +236,10 @@ Haiku of the month: New in this release: +* Remote jobs now sends stderr (standard error) to stderr (standard error) instead of stdout (standard output) + +* Remote execution is now packed using hex encoded bzip2. That means that big environments (app. 250 KB) can be transferred. + * A semibig refactoring of big functions. All non-trivial functions are now less than 100 lines. The refactoring makes this release beta quality. * GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137 @@ -246,8 +250,12 @@ New in this release: * GNU Parallel: Open Source For You (OSFY) magazine, October 2013 edition http://www.shakthimaan.com/posts/2014/11/27/gnu-parallel/news.html +* Multithreaded Encryption and Compression http://www.krazyworks.com/multithreaded-encryption-and-compression/ + * コマンドを並列に実行するGNU parallelがとても便利 http://bicycle1885.hatenablog.com/entry/2014/08/10/143612 +* GNU Parallel on OSX and Linux https://darknightelf.wordpress.com/2015/01/01/gnu-parallel-on-osx/ + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel b/src/parallel index 55ab043a..c9348557 100755 --- a/src/parallel +++ b/src/parallel @@ -1124,90 +1124,7 @@ sub parse_env_var { # %ENV # Returns: N/A $Global::envvar = ""; - $Global::envwarn = ""; - my @vars = ('parallel_bash_environment'); - for my $varstring (@opt::env) { - # Split up --env VAR1,VAR2 - push @vars, split /,/, $varstring; - } - if(grep { /^_$/ } @vars) { - # --env _ - # Include all vars that are not in a clean environment - if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) { - my @ignore = <$vars_fh>; - chomp @ignore; - my %ignore; - @ignore{@ignore} = @ignore; - close $vars_fh; - push @vars, grep { not defined $ignore{$_} } keys %ENV; - @vars = grep { not /^_$/ } @vars; - } else { - ::error("Run '$Global::progname --record-env' in a clean environment first.\n"); - ::wait_and_exit(255); - } - } - # Duplicate vars as BASH functions to include post-shellshock functions. - # So --env myfunc should also look for BASH_FUNC_myfunc() - @vars = map { $_, "BASH_FUNC_$_()", "BASH_FUNC_$_%%" } @vars; - # Keep only defined variables - @vars = grep { defined($ENV{$_}) } @vars; - # Pre-shellshock style bash function: - # myfunc=() { echo myfunc - # } - # Post-shellshock style bash function: - # BASH_FUNC_myfunc()=() { echo myfunc - # } - # BASH_FUNC_myfunc%%=() { echo myfunc - # } - my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars; - my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars; - if(@bash_functions) { - # Functions are not supported for all shells - if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) { - ::warning("Shell functions may not be supported in $Global::shell\n"); - } - } - # Pre-shellschock names are without () - my @bash_pre_shellshock = grep { not /\(\)|%%/ } @bash_functions; - # Post-shellschock names are with () - my @bash_post_shellshock = grep { /\(\)|%%/ } @bash_functions; - - my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) } - grep { not /^parallel_bash_environment$/ } @non_functions); - my @qbash = (map { my $a=$_; "export $a=" . env_quote($ENV{$a}) } - @non_functions, @bash_pre_shellshock); - - push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_pre_shellshock; - push @qbash, map { /BASH_FUNC_(.*)(\(\)|%%)/; "$1 $ENV{$_}" } @bash_post_shellshock; - - #ssh -tt -oLogLevel=quiet lo 'eval `echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ BASH_FUNC_myfunc\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ a\"' - #'\"\\\}\ \|\|\ myfunc\(\)\ \{\ \ echo\ a' - #'\}\ \;myfunc\ 1; - - # Check if any variables contain \n - if(my @v = map { s/BASH_FUNC_(.*)(\(\)|%%)/$1/; $_ } grep { $ENV{$_}=~/\n/ } @vars) { - # \n is bad for csh and will cause it to fail. - $Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn; - } - - if(not @qcsh) { push @qcsh, "true"; } - if(not @qbash) { push @qbash, "true"; } - # Create lines like: - # echo $SHELL | grep "/t\\{0,1\\}csh" >/dev/null && setenv V1 val1 && - # setenv V2 val2 || export V1=val1 && export V2=val2 ; echo "$V1$V2" - if(@vars) { - $Global::envvar .= - join"", - (q{echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null && } - . join(" && ", @qcsh) - . q{ || } - . join(" && ", @qbash) - .q{;}); - if($ENV{'parallel_bash_environment'}) { - $Global::envvar .= 'eval "$parallel_bash_environment";'."\n"; - } - } $Global::envvarlen = length $Global::envvar; } @@ -1554,6 +1471,18 @@ sub shellwords { return Text::ParseWords::shellwords(@_); } +sub perl_quote_scalar { + # Quote the string so perl's eval will not expand any special chars + # Inputs: + # $string = string to be quoted + # Returns: + # $shell_quoted = string quoted with \ as needed by perl's eval + my $a = $_[0]; + if(defined $a) { + $a =~ s/[\\\"\$\@]/\\$&/go; + } + return $a; +} sub __FILEHANDLES__ {} @@ -3179,6 +3108,24 @@ sub undef_as_empty { return $a ? $a : ""; } +sub spacefree { + # Remove comments and spaces + # Inputs: + # $spaces = keep 1 space? + # $s = string to remove spaces from + # Returns: + # $s = with spaces removed + my $spaces = shift; + my $s = shift; + $s =~ s/#.*//mg; + if($spaces) { + $s =~ s/\s+/ /mg; + } else { + $s =~ s/\s//mg; + } + return $s; +} + { my $hostname; sub hostname { @@ -5794,7 +5741,7 @@ sub wrapped { # $self->{'wrapped'} = the command wrapped with the above my $self = shift; if(not defined $self->{'wrapped'}) { - my $command = $Global::envvar.$self->replaced(); + my $command = $self->replaced(); if($opt::shellquote) { # Prepend echo # and quote twice @@ -5871,8 +5818,73 @@ sub sshlogin { return $self->{'sshlogin'}; } +sub string_zip_hex { + # Pipe string through 'bzip2 -9' and hex encode it into 1000 byte + # blocks. + # 1000 bytes is the largest word size csh supports + # Input: + # @strings = to be encoded + # Returns: + # @hex = 1000 byte block + my($zipin_fh, $zipout_fh,@hex); + ::open3($zipin_fh,$zipout_fh,">&STDERR","bzip2 -9"); + if(fork) { + close $zipin_fh; + # Chop hex into 1000 byte blocks + @hex = unpack("(H1000)*",join"",<$zipout_fh>); + close $zipout_fh; + } else { + close $zipout_fh; + print $zipin_fh @_; + close $zipin_fh; + exit; + } + ::debug("hex","Orig:@_\nAs hex:@hex\n"); + return @hex; +} + +sub hex_zip_eval { + # Script that: + # * reads hexstrings from @ARGV + # * unhexes them + # * pipes through 'bzip2 -dc' + # * evals the result + # Reverse of string_bz_hex + eval + # Will be wrapped in ' so single quote is forbidden + # Returns: + # $script = 1-liner for perl -e + my $script = ::spacefree(0,q{ + @GNU_Parallel=("use","IPC::Open3;"); + eval "@GNU_Parallel"; + + $SIG{CHLD}="IGNORE"; + # Search for bzip2. Not found => use default path + my $zip = (grep { -x $_ } "/usr/local/bin/bzip2")[0] || "bzip2"; + # $in = stdin on $zip, $out = stdout from $zip + my($in, $out,$eval); + open3($in,$out,">&STDERR",$zip,"-dc"); + if(my $perlpid = fork) { + close $in; + $eval = join "", <$out>; + close $out; + } else { + close $out; + # Pipe decoded hex into 'bzip2 -dc' + print $in (pack("H*",join"",@ARGV)); + close $in; + exit; + } + wait; + eval $eval; + }); + ::debug("hex",$script,"\n"); + return $script; +} + sub sshlogin_wrap { # Wrap the command with the commands needed to run remotely + # Input: + # $command = command to run # Returns: # $self->{'sshlogin_wrap'} = command wrapped with ssh+transfer commands sub monitor_parent_sshd_script { @@ -5882,20 +5894,17 @@ sub sshlogin_wrap { # If its parent is ssh: all good # If its parent is init(1): ssh died, so kill children my $monitor_parent_sshd_script; + if(not $monitor_parent_sshd_script) { $monitor_parent_sshd_script = # This will be packed in ', so only use " - q{ - # If packed as hex, unpack - # if($ARGV[0] =~ s/^0x//) { - # $ARGV[0] =~ s/(..)/chr(hex $1)/ge; - #} + ::spacefree(0,q{ $SIG{CHLD} = sub { $done = 1; }; $pid = fork; unless($pid) { # Make own process group to be able to kill HUP it later setpgrp; - exec $ENV{SHELL}, "-c", @ARGV; + exec $ENV{SHELL}, "-c", ($bashfunc."@ARGV"); die "exec: $!\n"; } do { @@ -5908,15 +5917,82 @@ sub sshlogin_wrap { kill(SIGHUP, -${pid}) unless $done; wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8) - }; - $monitor_parent_sshd_script =~ s/#.*//mg; - $monitor_parent_sshd_script =~ s/\s//mg; + }); } return $monitor_parent_sshd_script; } - #sub hex_pack { - #return unpack("H*","@_"); - #} + + sub vars_to_export { + # Uses: + # @opt::env + my @vars = ("parallel_bash_environment"); + for my $varstring (@opt::env) { + # Split up --env VAR1,VAR2 + push @vars, split /,/, $varstring; + } + if(grep { /^_$/ } @vars) { + # --env _ + # Include all vars that are not in a clean environment + if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) { + my @ignore = <$vars_fh>; + chomp @ignore; + my %ignore; + @ignore{@ignore} = @ignore; + close $vars_fh; + push @vars, grep { not defined $ignore{$_} } keys %ENV; + @vars = grep { not /^_$/ } @vars; + } else { + ::error("Run '$Global::progname --record-env' in a clean environment first.\n"); + ::wait_and_exit(255); + } + } + # Duplicate vars as BASH functions to include post-shellshock functions (v1+v2) + # So --env myfunc should also look for BASH_FUNC_myfunc() + push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", + map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars); + # Keep only defined variables + return grep { defined($ENV{$_}) } @vars; + } + + sub env_as_eval { + # Returns: + # $eval = '$ENV{"..."}=...; ...' + my @vars = vars_to_export(); + my $csh_friendly = not grep { /\n/ } @ENV{@vars}; + my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars; + my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars; + # eval of @envset will set %ENV + my $envset = join"", map { + '$ENV{"'.::perl_quote_scalar($_).'"}="'. ::perl_quote_scalar($ENV{$_}).'";'; } @non_functions; + + # running @bashfunc on the command line, will set the functions + my @bashfunc = map { + my $v=$_; s/BASH_FUNC_(.*)(\(\)|%%)/$1/; "$_$ENV{$v};" } @bash_functions; + # eval $bashfuncset will set $bashfunc + my $bashfuncset; + if(@bashfunc) { + # Functions are not supported for all shells + if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) { + ::warning("Shell functions may not be supported in $Global::shell\n"); + } + $bashfuncset = + '@bash_functions=qw('."@bash_functions".");". + ::spacefree(1,q{ + if($ENV{"SHELL"}=~/csh/) { + print STDERR "CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset @bash_functions\n"; + exec "false"; + } + }). + "\n".'$bashfunc = "'.::perl_quote_scalar("@bashfunc").'";'; + } else { + $bashfuncset = '$bashfunc = "";' + } + if($ENV{"parallel_bash_environment"}) { + $bashfuncset .= '$bashfunc .= "eval\ \"\$parallel_bash_environment\"\;";'; + } + ::debug("hex",$envset,$bashfuncset,"\n"); + return $csh_friendly,$envset,$bashfuncset; + } my $self = shift; my $command = shift; @@ -5925,9 +6001,8 @@ sub sshlogin_wrap { my $sshcmd = $sshlogin->sshcommand(); my $serverlogin = $sshlogin->serverlogin(); my ($pre,$post,$cleanup)=("","",""); - if($serverlogin eq ":") { - # No transfer neeeded + # No transfer/environment neeeded $self->{'sshlogin_wrap'} = $command; } else { # --transfer @@ -5944,74 +6019,34 @@ sub sshlogin_wrap { # otherwise use 'export' # We cannot use parse_env_var(), as PARALLEL_SEQ changes # for each command - my $parallel_env = - ($Global::envwarn - . q{ 'eval `echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null } - . q{ && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; } - . q{ setenv PARALLEL_PID '$PARALLEL_PID' } - . q{ || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; } - . q{ PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' }); - my $remote_pre = ""; - my $ssh_options = ""; - if(($opt::pipe or $opt::pipepart) and $opt::ctrlc - or - not ($opt::pipe or $opt::pipepart) and not $opt::noctrlc) { - # Propagating CTRL-C to kill remote jobs requires - # remote jobs to be run with a terminal. -# $ssh_options = "-tt -oLogLevel=quiet"; - $ssh_options = ""; - # tty - check if we have a tty. - # stty: - # -onlcr - make output 8-bit clean - # isig - pass CTRL-C as signal - # -echo - do not echo input -# $remote_pre .= ::shell_quote_scalar('tty >/dev/null && stty isig -onlcr -echo;'); - } + $ENV{'PARALLEL_SEQ'} = $self->seq(); + $ENV{'PARALLEL_PID'} = $$; + my $pwd = ""; if($opt::workdir) { - my $wd = ::shell_quote_file($self->workdir()); - $remote_pre .= ::shell_quote_scalar("mkdir -p ") . $wd . - ::shell_quote_scalar("; cd ") . $wd . - # exit 255 (instead of exec false) would be the correct thing, - # but that fails on tcsh - ::shell_quote_scalar(qq{ || exec false;}); + # Create remote workdir if needed. Then cd to it. + my $wd = $self->workdir(); + $pwd = qq{system("mkdir","-p","--","$wd") || chdir "$wd" ||}. + qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;}; + $command = "cd $wd;$command"; + } + my ($csh_friendly,$envset,$bashfuncset) = env_as_eval(); + my $remote_command = $pwd.$envset.$bashfuncset. + '@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script(); + my $quoted_remote_command = "exec perl -e ".::shell_quote_scalar(::shell_quote_scalar($remote_command)); + if(length $quoted_remote_command > 1000 + or + not $csh_friendly + or + $command =~ "\n") { + # csh does not deal well with > 1000 chars in one word + # csh does not deal well with $ENV with \n + $quoted_remote_command = "exec perl -e '".::shell_quote_scalar(hex_zip_eval())."' ". + join" ",string_zip_hex($remote_command); } - my $signal_script = "exec perl -e '".monitor_parent_sshd_script()."' "; - # TODO clean this up - # TODO Maybe env vars should be set in the perl script - # It works on Linux but not Solaris - # Finishes on Solaris, but wrong exit code: - # $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)}; - # Hangs on Solaris, but correct exit code on Linux: - # $SIG{CHLD} = sub { $done = 1 }; - # $p->poll; - - # $SIG{CHLD} = sub { $done = 1; ($^O eq "solaris") && exit 0 }; - # $SIG{CHLD} = sub { $done = 1; (1) && exit ($?&127 ? 128+($?&127) : 1+$?>>8); }; - # $SIG{CHLD} = sub { $done = 1; exit ($?&127 ? 128+($?&127) : 1+$?>>8); }; - # -> Linux: "" +script - # -> Solaris: -tt -script - # --ctrl-c => -tt -# q{ -# ($^O eq "solaris") && exec $ENV{SHELL}, "-c", @ARGV; -# $|=1; -# use IO::Poll; -# $SIG{CHLD} = sub {$done = 1; }; -# $pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"} -# $p = IO::Poll->new; -# $p->mask(STDOUT, POLLHUP); -# $p->poll; -# kill SIGHUP, -${pid} unless $done; -# wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8) -# } . "' "; -# $signal_script =~ s/\s+/ /g; - $self->{'sshlogin_wrap'} = ($pre - . "$sshcmd $ssh_options $serverlogin $parallel_env " - . $remote_pre - . ::shell_quote_scalar($signal_script . ::shell_quote_scalar($command)) -# . ::shell_quote_scalar($signal_script . hex_pack($command)) -# . ::shell_quote_scalar($command) + . "$sshcmd $serverlogin " + . $quoted_remote_command . ";" . $post); } @@ -8527,5 +8562,5 @@ sub mkdir_or_die { } # Keep perl -w happy -$opt::x = $Semaphore::timeout = $Semaphore::wait = +$opt::ctrlc = $opt::x = $Semaphore::timeout = $Semaphore::wait = $Job::file_descriptor_warning_printed = 0; diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index de24f5e6..931dcef4 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -54,8 +54,6 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; export -f funky; parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' - - EOF VBoxManage controlvm FreeBSD71 savestate diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 6e3bd99d..c9f6161a 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -9,6 +9,9 @@ cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout paral echo '### Stop if all hosts are filtered and there are no hosts left to run on' stdout parallel --filter-hosts -S no-such.host echo ::: 1 +echo '### Can csh propagate a variable containing \n'; + export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy + echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' echo '** test_zsh' FOO=test_zsh parallel --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME' diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index 1d1f0a73..27fd1d30 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -11,7 +11,7 @@ echo 'TODO test ssh with > 9 simultaneous' echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1 echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 chmod 755 /tmp/myssh1 /tmp/myssh2 -seq 1 100 | parallel --sshdelay 0.05 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo +seq 1 100 | parallel --sshdelay 0.01 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj1 -k -L1 echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' diff --git a/testsuite/tests-to-run/sem01.sh b/testsuite/tests-to-run/sem01.sh index 31f64af7..4819b365 100755 --- a/testsuite/tests-to-run/sem01.sh +++ b/testsuite/tests-to-run/sem01.sh @@ -1,9 +1,9 @@ #!/bin/bash -echo '### Test mutex. This should not mix output' -parallel -u --semaphore seq 1 10 '|' pv -qL 20 -parallel -u --semaphore seq 11 20 '|' pv -qL 100 -parallel --semaphore --wait +echo '### Test mutex. This should not mix output'; +parallel -u --semaphore seq 1 10 '|' pv -qL 20; +parallel -u --semaphore seq 11 20 '|' pv -qL 100; +parallel --semaphore --wait; echo done echo '### Test semaphore 2 jobs running simultaneously' @@ -26,18 +26,18 @@ echo done echo '### Test similar example as from man page - run 2 jobs simultaneously' echo 'Expect done: 1 2 5 3 4' -for i in 5 1 2 3 4 ; do - sleep 0.2 - echo Scheduling $i - sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i -done +for i in 5 1 2 3 4 ; do + sleep 0.2; + echo Scheduling $i; + sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i; +done; sem --wait echo '### Test --fg followed by --bg' -parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30 -parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30 -parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30 -parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30 +parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30; +parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30; +parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30; +parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30; sem --wait echo '### Test bug #33621: --bg -p should give an error message' diff --git a/testsuite/tests-to-run/test15.sh b/testsuite/tests-to-run/test15.sh index 95d153c5..881fea92 100755 --- a/testsuite/tests-to-run/test15.sh +++ b/testsuite/tests-to-run/test15.sh @@ -241,8 +241,8 @@ echo '### Test --nice locally' parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b echo '### Test --nice remote' -stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b - | perl -pe 's/\S*parallel-server\S*/one-server/' +stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | + perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;' echo '### Test distribute arguments at EOF to 2 jobslots' seq 1 92 | parallel -j+0 -kX -s 100 echo diff --git a/testsuite/tests-to-run/test19.sh b/testsuite/tests-to-run/test19.sh index 2658cbae..c7c7a908 100644 --- a/testsuite/tests-to-run/test19.sh +++ b/testsuite/tests-to-run/test19.sh @@ -122,6 +122,7 @@ stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK echo 'Input for ssh' -cat /tmp/myssh1-run /tmp/myssh2-run | perl -pe 's/(PID.)\d+/${1}00000/g;s/(SEQ[ =]|line)\d/$1X/g;' +cat /tmp/myssh1-run /tmp/myssh2-run | perl -pe 's/(PID.)\d+/${1}00000/g;s/(SEQ[ =]|line)\d/$1X/g;' | + perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;' rm /tmp/myssh1-run /tmp/myssh2-run diff --git a/testsuite/tests-to-run/test37.sh b/testsuite/tests-to-run/test37.sh index 1a9a1d99..f2dbc747 100644 --- a/testsuite/tests-to-run/test37.sh +++ b/testsuite/tests-to-run/test37.sh @@ -39,27 +39,27 @@ parallel -v -J test_profile ::: <(echo a) <(echo b) echo '### Test ugly quoting from profile file --plain' parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b) -PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 -v ::: foo -PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1 -v ::: foo +PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo echo '### Test quoting of $ in command from profile file' cat <~/.parallel/test_profile -k --jobs 1 perl -pe \'\\\$a=1; print \\\$a\' EOF -parallel -v -J test_profile -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 '<(echo {})' ::: foo +parallel -v -J test_profile -S ssh\ parallel\@$SERVER1 '<(echo {})' ::: foo echo '### Test quoting of $ in command from profile file --plain' -parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 'cat <(echo {})' ::: foo +parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1 'cat <(echo {})' ::: foo echo '### Test quoting of $ in command from $PARALLEL' -PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo echo '### Test quoting of $ in command from $PARALLEL --plain' -PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo +PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1 -v 'cat <(echo {})' ::: foo echo '### Test quoting of space in arguments (-S) from profile file' cat <~/.parallel/test_profile --k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\' +-k --jobs 1 -S ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\' EOF parallel -v -J test_profile '<(echo {})' ::: foo @@ -67,17 +67,17 @@ echo '### Test quoting of space in arguments (-S) from profile file --plain' parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo echo '### Test quoting of space in arguments (-S) from $PARALLEL' -PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo echo '### Test quoting of space in long arguments (--sshlogin) from profile file' cat <~/.parallel/test_profile # testprofile --k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\' +-k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\' EOF parallel -v -J test_profile '<(echo {})' ::: foo echo '### Test quoting of space in arguments (-S) from $PARALLEL' -PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo echo '### Test merging of profiles - sort needed because -k only works on the single machine' echo --tag > ~/.parallel/test_tag diff --git a/testsuite/tests-to-run/test61.sh b/testsuite/tests-to-run/test61.sh index f1e2dddd..25728847 100644 --- a/testsuite/tests-to-run/test61.sh +++ b/testsuite/tests-to-run/test61.sh @@ -3,12 +3,14 @@ SERVER1=parallel-server3 SERVER2=parallel-server2 -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 echo '### Test --return of weirdly named file' -stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b' +stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b' | + perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'; rm 'aa<${#}" b' echo '### Test if remote login shell is csh' -stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c +stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c | + perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;' echo '### Test {} multiple times in different commands' seq 10 | parallel -v -Xj1 echo {} \; echo {} diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 9a6a801c..a6e668fe 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -3,6 +3,11 @@ echo '### Stop if all hosts are filtered and there are no hosts left to run on' stdout parallel --filter-hosts -S no-such.host echo ::: 1 parallel: Warning: Removed no-such.host parallel: Error: There are no hosts left to run on. +echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy +### Can csh propagate a variable containing \n +1 +2 +3 echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' ### bug #41805: Idea: propagate --env for parallel --number-of-cores echo '** test_zsh' diff --git a/testsuite/wanted-results/parallel-local-ssh2 b/testsuite/wanted-results/parallel-local-ssh2 index f1621547..ab7f2fe4 100644 --- a/testsuite/wanted-results/parallel-local-ssh2 +++ b/testsuite/wanted-results/parallel-local-ssh2 @@ -12,30 +12,26 @@ echo '### --wd no-such-dir - csh' ### --wd no-such-dir - csh stdout parallel --wd /no-such-dir -S csh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? mkdir: cannot create directory ‘/no-such-dir’: Permission denied -/no-such-dir: No such file or directory. +parallel: Cannot chdir to /no-such-dir Exit code 1 echo '### --wd no-such-dir - tcsh' ### --wd no-such-dir - tcsh stdout parallel --wd /no-such-dir -S tcsh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? mkdir: cannot create directory ‘/no-such-dir’: Permission denied -/no-such-dir: No such file or directory. +parallel: Cannot chdir to /no-such-dir Exit code 1 echo '### --wd no-such-dir - bash' ### --wd no-such-dir - bash stdout parallel --wd /no-such-dir -S parallel@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? mkdir: cannot create directory ‘/no-such-dir’: Permission denied -bash: line 0: cd: /no-such-dir: No such file or directory +parallel: Cannot chdir to /no-such-dir Exit code 1 echo '### bug #42725: csh with \n in variables' ### bug #42725: csh with \n in variables not_csh() { echo This is not csh/tcsh; }; export -f not_csh; parallel --env not_csh -S csh@lo not_csh ::: 1; parallel --env not_csh -S tcsh@lo not_csh ::: 1; parallel --env not_csh -S parallel@lo not_csh ::: 1 -CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh -CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh This is not csh/tcsh -Badly placed ()'s. -}: Command not found. -Badly placed ()'s. -}: Command not found. +CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh +CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh echo '### bug #43358: shellshock breaks exporting functions using --env' ### bug #43358: shellshock breaks exporting functions using --env echo shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@localhost funky ::: shellshock-hardened diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index d9fe3fd6..cd889074 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -130,12 +130,10 @@ shell? /bin/bash -c cp `which yash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit e which bash => shell path /bin/bash shell? /bin/bash -c cp `which zsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which; which bash => shell path /bin/bash -MANPATH: Undefined variable. /tmp/SHELL: -c: bad option(s) Local configuration error occurred. Contact the systems administrator for further assistance. SHELL: applet not found -MANPATH: Undefined variable. echo '## Known shells -c' ## Known shells -c parallel -k "\`which {}\` -c 'parallel -Dinit echo ::: 1' | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh @@ -168,12 +166,8 @@ shell? /bin/bash -c `which yash` -c 'parallel -Dinit echo ::: 1' | grep which; which bash => shell path /bin/bash shell? /bin/bash -c `which zsh` -c 'parallel -Dinit echo ::: 1' | grep which; which bash => shell path /bin/bash -MANPATH: Undefined variable. -MANPATH: Undefined variable. Local configuration error occurred. Contact the systems administrator for further assistance. -MANPATH: Undefined variable. -MANPATH: Undefined variable. echo '## Known shells |' ## Known shells | parallel -k "echo 'parallel -Dinit echo ::: 1' | \`which {}\` | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh @@ -197,12 +191,8 @@ which static-sh => shell path /bin/static-sh which tcsh => shell path /usr/bin/tcsh which yash => shell path /usr/bin/yash which zsh => shell path /usr/bin/zsh -MANPATH: Undefined variable. -MANPATH: Undefined variable. Local configuration error occurred. Contact the systems administrator for further assistance. -MANPATH: Undefined variable. -MANPATH: Undefined variable. echo '## Started directly from perl' ## Started directly from perl perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 index c060e764..1a68074d 100644 --- a/testsuite/wanted-results/test15 +++ b/testsuite/wanted-results/test15 @@ -754,7 +754,7 @@ args on cmdline \nice -n1 /bin/bash -c PAR=a\ bash\ -c\ \"echo\ \ \\\$PAR\ b\" a b ### Test --nice remote -ssh one-server 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ \\\\nice\\\ -n1\\\ /bin/bash\\\ -c\\\ PAR=a\\\\\\\ bash\\\\\\\ -c\\\\\\\ \\\\\\\"echo\\\\\\\ \\\\\\\ \\\\\\\\\\\\\\\$PAR\\\\\\\ b\\\\\\\"; +ssh one-server exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex; a b ### Test distribute arguments at EOF to 2 jobslots 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 diff --git a/testsuite/wanted-results/test19 b/testsuite/wanted-results/test19 index 9448ad67..ec355826 100644 --- a/testsuite/wanted-results/test19 +++ b/testsuite/wanted-results/test19 @@ -75,37 +75,29 @@ OK ls: tmp/parallel.file*: No such file or directory OK Input for ssh -parallel@parallel-server1 mkdir -p ./. --l parallel parallel-server1 rsync --server -lDrRze.iLsfx . ./. -parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ X\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=X\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; exec perl -e '$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){setpgrp;exec$ENV{SHELL},"-c",@ARGV;die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)' cat\ tmp/parallel.file.\'' -'\'newlineX\ \>\ tmp/parallel.file.\'' -'\'newlineX.out\;cat\ tmp/parallel.file.\'' -'\'newlineX\ \>\ tmp/parallel.file.\'' -'\'newlineX.out2 --l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' +one-server mkdir -p ./. +-l parallel one-server rsync --server -lDrRze.iLsfx . ./. +one-server exec perl -e '@u=("use","IPC::Open3;");eval"@u";$SIG{CHLD}='IGNORE';my($gzin_fh,$gzout_fh,$eval);my$gzpid=open3($gzin_fh,$gzout_fh,">&STDERR","gzip","-dc");if(my$perlpid=fork){close$gzin_fh;$eval=join"",<$gzout_fh>;close$gzout_fh;}else{close$gzout_fh;print$gzin_fh(pack("H*",join"",@ARGV));close$gzin_fh;exit;}wait;eval$eval;' hex +-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' 'newlineX.out --l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' +-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' 'newlineX.out2 -parallel@parallel-server1 (rm -f ./tmp/parallel.file.' +one-server (rm -f ./tmp/parallel.file.' 'newlineX; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server1 (rm -f ./tmp/parallel.file.' +one-server (rm -f ./tmp/parallel.file.' 'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server1 (rm -f ./tmp/parallel.file.' +one-server (rm -f ./tmp/parallel.file.' 'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server2 mkdir -p ./. --l parallel parallel-server2 rsync --server -lDrRze.iLsfx . ./. -parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ X\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=X\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; exec perl -e '$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){setpgrp;exec$ENV{SHELL},"-c",@ARGV;die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)' cat\ tmp/parallel.file.\'' -'\'newlineX\ \>\ tmp/parallel.file.\'' -'\'newlineX.out\;cat\ tmp/parallel.file.\'' -'\'newlineX\ \>\ tmp/parallel.file.\'' -'\'newlineX.out2 --l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' +one-server mkdir -p ./. +-l parallel one-server rsync --server -lDrRze.iLsfx . ./. +one-server exec perl -e '@u=("use","IPC::Open3;");eval"@u";$SIG{CHLD}='IGNORE';my($gzin_fh,$gzout_fh,$eval);my$gzpid=open3($gzin_fh,$gzout_fh,">&STDERR","gzip","-dc");if(my$perlpid=fork){close$gzin_fh;$eval=join"",<$gzout_fh>;close$gzout_fh;}else{close$gzout_fh;print$gzin_fh(pack("H*",join"",@ARGV));close$gzin_fh;exit;}wait;eval$eval;' hex +-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' 'newlineX.out --l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' +-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.' 'newlineX.out2 -parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +one-server (rm -f ./tmp/parallel.file.' 'newlineX; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +one-server (rm -f ./tmp/parallel.file.' 'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +one-server (rm -f ./tmp/parallel.file.' 'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;) diff --git a/testsuite/wanted-results/test37 b/testsuite/wanted-results/test37 index ecae584b..402aab69 100644 --- a/testsuite/wanted-results/test37 +++ b/testsuite/wanted-results/test37 @@ -42,7 +42,7 @@ echo /dev/fd/62 /dev/fd/62 echo foo foo -ssh parallel@redhat9.tange.dk ssh centos3.tange.dk 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ perl\\\ -pe\\\ \\\"\\\\\\\$a=1\\\;\\\ print\\\ \\\\\\\$a\\\"\\\ \\\<\\\(echo\\\ foo\\\); +perl -pe "\$a=1; print \$a" <(echo foo) 1foo ### Test quoting of $ in command from profile file perl -pe '$a=1; print $a' <(echo foo) diff --git a/testsuite/wanted-results/test61 b/testsuite/wanted-results/test61 index 0d64ab8b..aab1ff81 100644 --- a/testsuite/wanted-results/test61 +++ b/testsuite/wanted-results/test61 @@ -1,15 +1,15 @@ echo '### Test --return of weirdly named file' ### Test --return of weirdly named file -stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b' -ssh parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\>aa\\\\\\\<\\\\\\\$\\\\\\\{\\\\\\\#\\\\\\\}\\\\\\\"\\\\\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status; +stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b' | perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'; rm 'aa<${#}" b' +ssh one-server exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status; echo '### Test if remote login shell is csh' ### Test if remote login shell is csh -stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c -ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ a\\\|\\\ wc\\\ -w; +stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c | perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;' +ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex; 3 -ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ b\\\|\\\ wc\\\ -w; +ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex; 3 -ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ c\\\|\\\ wc\\\ -w; +ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex; 3 echo '### Test {} multiple times in different commands' ### Test {} multiple times in different commands