mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 14:07:55 +00:00
parallel: Commands longer than 1000 chars are bzip2'ed + hex encoded due to csh limit.
This commit is contained in:
Ole Tange
parent
3071e28c15
commit
3d2ecc7651
|
|
@ -236,6 +236,10 @@ Haiku of the month:
|
|||
|
||||
New in this release:
|
||||
|
||||
* Remote jobs now sends stderr (standard error) to stderr (standard error) instead of stdout (standard output)
|
||||
|
||||
* Remote execution is now packed using hex encoded bzip2. That means that big environments (app. 250 KB) can be transferred.
|
||||
|
||||
* A semibig refactoring of big functions. All non-trivial functions are now less than 100 lines. The refactoring makes this release beta quality.
|
||||
|
||||
* GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137
|
||||
|
|
@ -246,8 +250,12 @@ New in this release:
|
|||
|
||||
* GNU Parallel: Open Source For You (OSFY) magazine, October 2013 edition http://www.shakthimaan.com/posts/2014/11/27/gnu-parallel/news.html
|
||||
|
||||
* Multithreaded Encryption and Compression http://www.krazyworks.com/multithreaded-encryption-and-compression/
|
||||
|
||||
* コマンドを並列に実行するGNU parallelがとても便利 http://bicycle1885.hatenablog.com/entry/2014/08/10/143612
|
||||
|
||||
* GNU Parallel on OSX and Linux https://darknightelf.wordpress.com/2015/01/01/gnu-parallel-on-osx/
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
|
|
|||
361
src/parallel
361
src/parallel
|
|
@ -1124,90 +1124,7 @@ sub parse_env_var {
|
|||
# %ENV
|
||||
# Returns: N/A
|
||||
$Global::envvar = "";
|
||||
$Global::envwarn = "";
|
||||
my @vars = ('parallel_bash_environment');
|
||||
for my $varstring (@opt::env) {
|
||||
# Split up --env VAR1,VAR2
|
||||
push @vars, split /,/, $varstring;
|
||||
}
|
||||
if(grep { /^_$/ } @vars) {
|
||||
# --env _
|
||||
# Include all vars that are not in a clean environment
|
||||
if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) {
|
||||
my @ignore = <$vars_fh>;
|
||||
chomp @ignore;
|
||||
my %ignore;
|
||||
@ignore{@ignore} = @ignore;
|
||||
close $vars_fh;
|
||||
push @vars, grep { not defined $ignore{$_} } keys %ENV;
|
||||
@vars = grep { not /^_$/ } @vars;
|
||||
} else {
|
||||
::error("Run '$Global::progname --record-env' in a clean environment first.\n");
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
}
|
||||
# Duplicate vars as BASH functions to include post-shellshock functions.
|
||||
# So --env myfunc should also look for BASH_FUNC_myfunc()
|
||||
@vars = map { $_, "BASH_FUNC_$_()", "BASH_FUNC_$_%%" } @vars;
|
||||
# Keep only defined variables
|
||||
@vars = grep { defined($ENV{$_}) } @vars;
|
||||
# Pre-shellshock style bash function:
|
||||
# myfunc=() { echo myfunc
|
||||
# }
|
||||
# Post-shellshock style bash function:
|
||||
# BASH_FUNC_myfunc()=() { echo myfunc
|
||||
# }
|
||||
# BASH_FUNC_myfunc%%=() { echo myfunc
|
||||
# }
|
||||
my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
|
||||
my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars;
|
||||
if(@bash_functions) {
|
||||
# Functions are not supported for all shells
|
||||
if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
|
||||
::warning("Shell functions may not be supported in $Global::shell\n");
|
||||
}
|
||||
}
|
||||
|
||||
# Pre-shellschock names are without ()
|
||||
my @bash_pre_shellshock = grep { not /\(\)|%%/ } @bash_functions;
|
||||
# Post-shellschock names are with ()
|
||||
my @bash_post_shellshock = grep { /\(\)|%%/ } @bash_functions;
|
||||
|
||||
my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) }
|
||||
grep { not /^parallel_bash_environment$/ } @non_functions);
|
||||
my @qbash = (map { my $a=$_; "export $a=" . env_quote($ENV{$a}) }
|
||||
@non_functions, @bash_pre_shellshock);
|
||||
|
||||
push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_pre_shellshock;
|
||||
push @qbash, map { /BASH_FUNC_(.*)(\(\)|%%)/; "$1 $ENV{$_}" } @bash_post_shellshock;
|
||||
|
||||
#ssh -tt -oLogLevel=quiet lo 'eval `echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ BASH_FUNC_myfunc\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ a\"'
|
||||
#'\"\\\}\ \|\|\ myfunc\(\)\ \{\ \ echo\ a'
|
||||
#'\}\ \;myfunc\ 1;
|
||||
|
||||
# Check if any variables contain \n
|
||||
if(my @v = map { s/BASH_FUNC_(.*)(\(\)|%%)/$1/; $_ } grep { $ENV{$_}=~/\n/ } @vars) {
|
||||
# \n is bad for csh and will cause it to fail.
|
||||
$Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn;
|
||||
}
|
||||
|
||||
if(not @qcsh) { push @qcsh, "true"; }
|
||||
if(not @qbash) { push @qbash, "true"; }
|
||||
# Create lines like:
|
||||
# echo $SHELL | grep "/t\\{0,1\\}csh" >/dev/null && setenv V1 val1 &&
|
||||
# setenv V2 val2 || export V1=val1 && export V2=val2 ; echo "$V1$V2"
|
||||
if(@vars) {
|
||||
$Global::envvar .=
|
||||
join"",
|
||||
(q{echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null && }
|
||||
. join(" && ", @qcsh)
|
||||
. q{ || }
|
||||
. join(" && ", @qbash)
|
||||
.q{;});
|
||||
if($ENV{'parallel_bash_environment'}) {
|
||||
$Global::envvar .= 'eval "$parallel_bash_environment";'."\n";
|
||||
}
|
||||
}
|
||||
$Global::envvarlen = length $Global::envvar;
|
||||
}
|
||||
|
||||
|
|
@ -1554,6 +1471,18 @@ sub shellwords {
|
|||
return Text::ParseWords::shellwords(@_);
|
||||
}
|
||||
|
||||
sub perl_quote_scalar {
|
||||
# Quote the string so perl's eval will not expand any special chars
|
||||
# Inputs:
|
||||
# $string = string to be quoted
|
||||
# Returns:
|
||||
# $shell_quoted = string quoted with \ as needed by perl's eval
|
||||
my $a = $_[0];
|
||||
if(defined $a) {
|
||||
$a =~ s/[\\\"\$\@]/\\$&/go;
|
||||
}
|
||||
return $a;
|
||||
}
|
||||
|
||||
sub __FILEHANDLES__ {}
|
||||
|
||||
|
|
@ -3179,6 +3108,24 @@ sub undef_as_empty {
|
|||
return $a ? $a : "";
|
||||
}
|
||||
|
||||
sub spacefree {
|
||||
# Remove comments and spaces
|
||||
# Inputs:
|
||||
# $spaces = keep 1 space?
|
||||
# $s = string to remove spaces from
|
||||
# Returns:
|
||||
# $s = with spaces removed
|
||||
my $spaces = shift;
|
||||
my $s = shift;
|
||||
$s =~ s/#.*//mg;
|
||||
if($spaces) {
|
||||
$s =~ s/\s+/ /mg;
|
||||
} else {
|
||||
$s =~ s/\s//mg;
|
||||
}
|
||||
return $s;
|
||||
}
|
||||
|
||||
{
|
||||
my $hostname;
|
||||
sub hostname {
|
||||
|
|
@ -5794,7 +5741,7 @@ sub wrapped {
|
|||
# $self->{'wrapped'} = the command wrapped with the above
|
||||
my $self = shift;
|
||||
if(not defined $self->{'wrapped'}) {
|
||||
my $command = $Global::envvar.$self->replaced();
|
||||
my $command = $self->replaced();
|
||||
if($opt::shellquote) {
|
||||
# Prepend echo
|
||||
# and quote twice
|
||||
|
|
@ -5871,8 +5818,73 @@ sub sshlogin {
|
|||
return $self->{'sshlogin'};
|
||||
}
|
||||
|
||||
sub string_zip_hex {
|
||||
# Pipe string through 'bzip2 -9' and hex encode it into 1000 byte
|
||||
# blocks.
|
||||
# 1000 bytes is the largest word size csh supports
|
||||
# Input:
|
||||
# @strings = to be encoded
|
||||
# Returns:
|
||||
# @hex = 1000 byte block
|
||||
my($zipin_fh, $zipout_fh,@hex);
|
||||
::open3($zipin_fh,$zipout_fh,">&STDERR","bzip2 -9");
|
||||
if(fork) {
|
||||
close $zipin_fh;
|
||||
# Chop hex into 1000 byte blocks
|
||||
@hex = unpack("(H1000)*",join"",<$zipout_fh>);
|
||||
close $zipout_fh;
|
||||
} else {
|
||||
close $zipout_fh;
|
||||
print $zipin_fh @_;
|
||||
close $zipin_fh;
|
||||
exit;
|
||||
}
|
||||
::debug("hex","Orig:@_\nAs hex:@hex\n");
|
||||
return @hex;
|
||||
}
|
||||
|
||||
sub hex_zip_eval {
|
||||
# Script that:
|
||||
# * reads hexstrings from @ARGV
|
||||
# * unhexes them
|
||||
# * pipes through 'bzip2 -dc'
|
||||
# * evals the result
|
||||
# Reverse of string_bz_hex + eval
|
||||
# Will be wrapped in ' so single quote is forbidden
|
||||
# Returns:
|
||||
# $script = 1-liner for perl -e
|
||||
my $script = ::spacefree(0,q{
|
||||
@GNU_Parallel=("use","IPC::Open3;");
|
||||
eval "@GNU_Parallel";
|
||||
|
||||
$SIG{CHLD}="IGNORE";
|
||||
# Search for bzip2. Not found => use default path
|
||||
my $zip = (grep { -x $_ } "/usr/local/bin/bzip2")[0] || "bzip2";
|
||||
# $in = stdin on $zip, $out = stdout from $zip
|
||||
my($in, $out,$eval);
|
||||
open3($in,$out,">&STDERR",$zip,"-dc");
|
||||
if(my $perlpid = fork) {
|
||||
close $in;
|
||||
$eval = join "", <$out>;
|
||||
close $out;
|
||||
} else {
|
||||
close $out;
|
||||
# Pipe decoded hex into 'bzip2 -dc'
|
||||
print $in (pack("H*",join"",@ARGV));
|
||||
close $in;
|
||||
exit;
|
||||
}
|
||||
wait;
|
||||
eval $eval;
|
||||
});
|
||||
::debug("hex",$script,"\n");
|
||||
return $script;
|
||||
}
|
||||
|
||||
sub sshlogin_wrap {
|
||||
# Wrap the command with the commands needed to run remotely
|
||||
# Input:
|
||||
# $command = command to run
|
||||
# Returns:
|
||||
# $self->{'sshlogin_wrap'} = command wrapped with ssh+transfer commands
|
||||
sub monitor_parent_sshd_script {
|
||||
|
|
@ -5882,20 +5894,17 @@ sub sshlogin_wrap {
|
|||
# If its parent is ssh: all good
|
||||
# If its parent is init(1): ssh died, so kill children
|
||||
my $monitor_parent_sshd_script;
|
||||
|
||||
if(not $monitor_parent_sshd_script) {
|
||||
$monitor_parent_sshd_script =
|
||||
# This will be packed in ', so only use "
|
||||
q{
|
||||
# If packed as hex, unpack
|
||||
# if($ARGV[0] =~ s/^0x//) {
|
||||
# $ARGV[0] =~ s/(..)/chr(hex $1)/ge;
|
||||
#}
|
||||
::spacefree(0,q{
|
||||
$SIG{CHLD} = sub { $done = 1; };
|
||||
$pid = fork;
|
||||
unless($pid) {
|
||||
# Make own process group to be able to kill HUP it later
|
||||
setpgrp;
|
||||
exec $ENV{SHELL}, "-c", @ARGV;
|
||||
exec $ENV{SHELL}, "-c", ($bashfunc."@ARGV");
|
||||
die "exec: $!\n";
|
||||
}
|
||||
do {
|
||||
|
|
@ -5908,15 +5917,82 @@ sub sshlogin_wrap {
|
|||
kill(SIGHUP, -${pid}) unless $done;
|
||||
wait;
|
||||
exit ($?&127 ? 128+($?&127) : 1+$?>>8)
|
||||
};
|
||||
$monitor_parent_sshd_script =~ s/#.*//mg;
|
||||
$monitor_parent_sshd_script =~ s/\s//mg;
|
||||
});
|
||||
}
|
||||
return $monitor_parent_sshd_script;
|
||||
}
|
||||
#sub hex_pack {
|
||||
#return unpack("H*","@_");
|
||||
#}
|
||||
|
||||
sub vars_to_export {
|
||||
# Uses:
|
||||
# @opt::env
|
||||
my @vars = ("parallel_bash_environment");
|
||||
for my $varstring (@opt::env) {
|
||||
# Split up --env VAR1,VAR2
|
||||
push @vars, split /,/, $varstring;
|
||||
}
|
||||
if(grep { /^_$/ } @vars) {
|
||||
# --env _
|
||||
# Include all vars that are not in a clean environment
|
||||
if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) {
|
||||
my @ignore = <$vars_fh>;
|
||||
chomp @ignore;
|
||||
my %ignore;
|
||||
@ignore{@ignore} = @ignore;
|
||||
close $vars_fh;
|
||||
push @vars, grep { not defined $ignore{$_} } keys %ENV;
|
||||
@vars = grep { not /^_$/ } @vars;
|
||||
} else {
|
||||
::error("Run '$Global::progname --record-env' in a clean environment first.\n");
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
}
|
||||
# Duplicate vars as BASH functions to include post-shellshock functions (v1+v2)
|
||||
# So --env myfunc should also look for BASH_FUNC_myfunc()
|
||||
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
|
||||
map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
|
||||
# Keep only defined variables
|
||||
return grep { defined($ENV{$_}) } @vars;
|
||||
}
|
||||
|
||||
sub env_as_eval {
|
||||
# Returns:
|
||||
# $eval = '$ENV{"..."}=...; ...'
|
||||
my @vars = vars_to_export();
|
||||
my $csh_friendly = not grep { /\n/ } @ENV{@vars};
|
||||
my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
|
||||
my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars;
|
||||
# eval of @envset will set %ENV
|
||||
my $envset = join"", map {
|
||||
'$ENV{"'.::perl_quote_scalar($_).'"}="'. ::perl_quote_scalar($ENV{$_}).'";'; } @non_functions;
|
||||
|
||||
# running @bashfunc on the command line, will set the functions
|
||||
my @bashfunc = map {
|
||||
my $v=$_; s/BASH_FUNC_(.*)(\(\)|%%)/$1/; "$_$ENV{$v};" } @bash_functions;
|
||||
# eval $bashfuncset will set $bashfunc
|
||||
my $bashfuncset;
|
||||
if(@bashfunc) {
|
||||
# Functions are not supported for all shells
|
||||
if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
|
||||
::warning("Shell functions may not be supported in $Global::shell\n");
|
||||
}
|
||||
$bashfuncset =
|
||||
'@bash_functions=qw('."@bash_functions".");".
|
||||
::spacefree(1,q{
|
||||
if($ENV{"SHELL"}=~/csh/) {
|
||||
print STDERR "CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset @bash_functions\n";
|
||||
exec "false";
|
||||
}
|
||||
}).
|
||||
"\n".'$bashfunc = "'.::perl_quote_scalar("@bashfunc").'";';
|
||||
} else {
|
||||
$bashfuncset = '$bashfunc = "";'
|
||||
}
|
||||
if($ENV{"parallel_bash_environment"}) {
|
||||
$bashfuncset .= '$bashfunc .= "eval\ \"\$parallel_bash_environment\"\;";';
|
||||
}
|
||||
::debug("hex",$envset,$bashfuncset,"\n");
|
||||
return $csh_friendly,$envset,$bashfuncset;
|
||||
}
|
||||
|
||||
my $self = shift;
|
||||
my $command = shift;
|
||||
|
|
@ -5925,9 +6001,8 @@ sub sshlogin_wrap {
|
|||
my $sshcmd = $sshlogin->sshcommand();
|
||||
my $serverlogin = $sshlogin->serverlogin();
|
||||
my ($pre,$post,$cleanup)=("","","");
|
||||
|
||||
if($serverlogin eq ":") {
|
||||
# No transfer neeeded
|
||||
# No transfer/environment neeeded
|
||||
$self->{'sshlogin_wrap'} = $command;
|
||||
} else {
|
||||
# --transfer
|
||||
|
|
@ -5944,74 +6019,34 @@ sub sshlogin_wrap {
|
|||
# otherwise use 'export'
|
||||
# We cannot use parse_env_var(), as PARALLEL_SEQ changes
|
||||
# for each command
|
||||
my $parallel_env =
|
||||
($Global::envwarn
|
||||
. q{ 'eval `echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null }
|
||||
. q{ && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; }
|
||||
. q{ setenv PARALLEL_PID '$PARALLEL_PID' }
|
||||
. q{ || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; }
|
||||
. q{ PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' });
|
||||
my $remote_pre = "";
|
||||
my $ssh_options = "";
|
||||
if(($opt::pipe or $opt::pipepart) and $opt::ctrlc
|
||||
or
|
||||
not ($opt::pipe or $opt::pipepart) and not $opt::noctrlc) {
|
||||
# Propagating CTRL-C to kill remote jobs requires
|
||||
# remote jobs to be run with a terminal.
|
||||
# $ssh_options = "-tt -oLogLevel=quiet";
|
||||
$ssh_options = "";
|
||||
# tty - check if we have a tty.
|
||||
# stty:
|
||||
# -onlcr - make output 8-bit clean
|
||||
# isig - pass CTRL-C as signal
|
||||
# -echo - do not echo input
|
||||
# $remote_pre .= ::shell_quote_scalar('tty >/dev/null && stty isig -onlcr -echo;');
|
||||
}
|
||||
$ENV{'PARALLEL_SEQ'} = $self->seq();
|
||||
$ENV{'PARALLEL_PID'} = $$;
|
||||
my $pwd = "";
|
||||
if($opt::workdir) {
|
||||
my $wd = ::shell_quote_file($self->workdir());
|
||||
$remote_pre .= ::shell_quote_scalar("mkdir -p ") . $wd .
|
||||
::shell_quote_scalar("; cd ") . $wd .
|
||||
# exit 255 (instead of exec false) would be the correct thing,
|
||||
# but that fails on tcsh
|
||||
::shell_quote_scalar(qq{ || exec false;});
|
||||
# Create remote workdir if needed. Then cd to it.
|
||||
my $wd = $self->workdir();
|
||||
$pwd = qq{system("mkdir","-p","--","$wd") || chdir "$wd" ||}.
|
||||
qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;};
|
||||
$command = "cd $wd;$command";
|
||||
}
|
||||
my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
|
||||
my $remote_command = $pwd.$envset.$bashfuncset.
|
||||
'@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script();
|
||||
my $quoted_remote_command = "exec perl -e ".::shell_quote_scalar(::shell_quote_scalar($remote_command));
|
||||
if(length $quoted_remote_command > 1000
|
||||
or
|
||||
not $csh_friendly
|
||||
or
|
||||
$command =~ "\n") {
|
||||
# csh does not deal well with > 1000 chars in one word
|
||||
# csh does not deal well with $ENV with \n
|
||||
$quoted_remote_command = "exec perl -e '".::shell_quote_scalar(hex_zip_eval())."' ".
|
||||
join" ",string_zip_hex($remote_command);
|
||||
}
|
||||
my $signal_script = "exec perl -e '".monitor_parent_sshd_script()."' ";
|
||||
# TODO clean this up
|
||||
# TODO Maybe env vars should be set in the perl script
|
||||
# It works on Linux but not Solaris
|
||||
# Finishes on Solaris, but wrong exit code:
|
||||
# $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)};
|
||||
# Hangs on Solaris, but correct exit code on Linux:
|
||||
# $SIG{CHLD} = sub { $done = 1 };
|
||||
# $p->poll;
|
||||
|
||||
# $SIG{CHLD} = sub { $done = 1; ($^O eq "solaris") && exit 0 };
|
||||
# $SIG{CHLD} = sub { $done = 1; (1) && exit ($?&127 ? 128+($?&127) : 1+$?>>8); };
|
||||
# $SIG{CHLD} = sub { $done = 1; exit ($?&127 ? 128+($?&127) : 1+$?>>8); };
|
||||
# -> Linux: "" +script
|
||||
# -> Solaris: -tt -script
|
||||
# --ctrl-c => -tt
|
||||
# q{
|
||||
# ($^O eq "solaris") && exec $ENV{SHELL}, "-c", @ARGV;
|
||||
# $|=1;
|
||||
# use IO::Poll;
|
||||
# $SIG{CHLD} = sub {$done = 1; };
|
||||
# $pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"}
|
||||
# $p = IO::Poll->new;
|
||||
# $p->mask(STDOUT, POLLHUP);
|
||||
# $p->poll;
|
||||
# kill SIGHUP, -${pid} unless $done;
|
||||
# wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8)
|
||||
# } . "' ";
|
||||
# $signal_script =~ s/\s+/ /g;
|
||||
|
||||
$self->{'sshlogin_wrap'} =
|
||||
($pre
|
||||
. "$sshcmd $ssh_options $serverlogin $parallel_env "
|
||||
. $remote_pre
|
||||
. ::shell_quote_scalar($signal_script . ::shell_quote_scalar($command))
|
||||
# . ::shell_quote_scalar($signal_script . hex_pack($command))
|
||||
# . ::shell_quote_scalar($command)
|
||||
. "$sshcmd $serverlogin "
|
||||
. $quoted_remote_command
|
||||
. ";"
|
||||
. $post);
|
||||
}
|
||||
|
|
@ -8527,5 +8562,5 @@ sub mkdir_or_die {
|
|||
}
|
||||
|
||||
# Keep perl -w happy
|
||||
$opt::x = $Semaphore::timeout = $Semaphore::wait =
|
||||
$opt::ctrlc = $opt::x = $Semaphore::timeout = $Semaphore::wait =
|
||||
$Job::file_descriptor_warning_printed = 0;
|
||||
|
|
|
|||
|
|
@ -54,8 +54,6 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
|
|||
export -f funky;
|
||||
parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
|
||||
|
||||
|
||||
|
||||
EOF
|
||||
|
||||
VBoxManage controlvm FreeBSD71 savestate
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@ cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout paral
|
|||
echo '### Stop if all hosts are filtered and there are no hosts left to run on'
|
||||
stdout parallel --filter-hosts -S no-such.host echo ::: 1
|
||||
|
||||
echo '### Can csh propagate a variable containing \n';
|
||||
export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy
|
||||
|
||||
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
|
||||
echo '** test_zsh'
|
||||
FOO=test_zsh parallel --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ echo 'TODO test ssh with > 9 simultaneous'
|
|||
echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1
|
||||
echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2
|
||||
chmod 755 /tmp/myssh1 /tmp/myssh2
|
||||
seq 1 100 | parallel --sshdelay 0.05 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
|
||||
seq 1 100 | parallel --sshdelay 0.01 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj1 -k -L1
|
||||
echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host'
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo '### Test mutex. This should not mix output'
|
||||
parallel -u --semaphore seq 1 10 '|' pv -qL 20
|
||||
parallel -u --semaphore seq 11 20 '|' pv -qL 100
|
||||
parallel --semaphore --wait
|
||||
echo '### Test mutex. This should not mix output';
|
||||
parallel -u --semaphore seq 1 10 '|' pv -qL 20;
|
||||
parallel -u --semaphore seq 11 20 '|' pv -qL 100;
|
||||
parallel --semaphore --wait;
|
||||
echo done
|
||||
|
||||
echo '### Test semaphore 2 jobs running simultaneously'
|
||||
|
|
@ -27,17 +27,17 @@ echo done
|
|||
echo '### Test similar example as from man page - run 2 jobs simultaneously'
|
||||
echo 'Expect done: 1 2 5 3 4'
|
||||
for i in 5 1 2 3 4 ; do
|
||||
sleep 0.2
|
||||
echo Scheduling $i
|
||||
sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i
|
||||
done
|
||||
sleep 0.2;
|
||||
echo Scheduling $i;
|
||||
sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i;
|
||||
done;
|
||||
sem --wait
|
||||
|
||||
echo '### Test --fg followed by --bg'
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30;
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30;
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30;
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30;
|
||||
sem --wait
|
||||
|
||||
echo '### Test bug #33621: --bg -p should give an error message'
|
||||
|
|
|
|||
|
|
@ -241,8 +241,8 @@ echo '### Test --nice locally'
|
|||
parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b
|
||||
|
||||
echo '### Test --nice remote'
|
||||
stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b
|
||||
| perl -pe 's/\S*parallel-server\S*/one-server/'
|
||||
stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b |
|
||||
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
|
||||
|
||||
echo '### Test distribute arguments at EOF to 2 jobslots'
|
||||
seq 1 92 | parallel -j+0 -kX -s 100 echo
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
|
|||
# Should give: No such file or directory
|
||||
stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
|
||||
echo 'Input for ssh'
|
||||
cat /tmp/myssh1-run /tmp/myssh2-run | perl -pe 's/(PID.)\d+/${1}00000/g;s/(SEQ[ =]|line)\d/$1X/g;'
|
||||
cat /tmp/myssh1-run /tmp/myssh2-run | perl -pe 's/(PID.)\d+/${1}00000/g;s/(SEQ[ =]|line)\d/$1X/g;' |
|
||||
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
|
||||
rm /tmp/myssh1-run /tmp/myssh2-run
|
||||
|
||||
|
|
|
|||
|
|
@ -39,27 +39,27 @@ parallel -v -J test_profile ::: <(echo a) <(echo b)
|
|||
echo '### Test ugly quoting from profile file --plain'
|
||||
parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b)
|
||||
|
||||
PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 -v ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1 -v ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
-k --jobs 1 perl -pe \'\\\$a=1; print \\\$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 '<(echo {})' ::: foo
|
||||
parallel -v -J test_profile -S ssh\ parallel\@$SERVER1 '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from profile file --plain'
|
||||
parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 'cat <(echo {})' ::: foo
|
||||
parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1 'cat <(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from $PARALLEL'
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from $PARALLEL --plain'
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1 -v 'cat <(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in arguments (-S) from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
-k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\'
|
||||
-k --jobs 1 -S ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile '<(echo {})' ::: foo
|
||||
|
||||
|
|
@ -67,17 +67,17 @@ echo '### Test quoting of space in arguments (-S) from profile file --plain'
|
|||
parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in arguments (-S) from $PARALLEL'
|
||||
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in long arguments (--sshlogin) from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
# testprofile
|
||||
-k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\'
|
||||
-k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in arguments (-S) from $PARALLEL'
|
||||
PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test merging of profiles - sort needed because -k only works on the single machine'
|
||||
echo --tag > ~/.parallel/test_tag
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@
|
|||
SERVER1=parallel-server3
|
||||
SERVER2=parallel-server2
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1
|
||||
echo '### Test --return of weirdly named file'
|
||||
stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b'
|
||||
stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b' |
|
||||
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'; rm 'aa<${#}" b'
|
||||
|
||||
echo '### Test if remote login shell is csh'
|
||||
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c
|
||||
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c |
|
||||
perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
|
||||
|
||||
echo '### Test {} multiple times in different commands'
|
||||
seq 10 | parallel -v -Xj1 echo {} \; echo {}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ echo '### Stop if all hosts are filtered and there are no hosts left to run on'
|
|||
stdout parallel --filter-hosts -S no-such.host echo ::: 1
|
||||
parallel: Warning: Removed no-such.host
|
||||
parallel: Error: There are no hosts left to run on.
|
||||
echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy
|
||||
### Can csh propagate a variable containing \n
|
||||
1
|
||||
2
|
||||
3
|
||||
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
|
||||
### bug #41805: Idea: propagate --env for parallel --number-of-cores
|
||||
echo '** test_zsh'
|
||||
|
|
|
|||
|
|
@ -12,30 +12,26 @@ echo '### --wd no-such-dir - csh'
|
|||
### --wd no-such-dir - csh
|
||||
stdout parallel --wd /no-such-dir -S csh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $?
|
||||
mkdir: cannot create directory ‘/no-such-dir’: Permission denied
|
||||
/no-such-dir: No such file or directory.
|
||||
parallel: Cannot chdir to /no-such-dir
|
||||
Exit code 1
|
||||
echo '### --wd no-such-dir - tcsh'
|
||||
### --wd no-such-dir - tcsh
|
||||
stdout parallel --wd /no-such-dir -S tcsh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $?
|
||||
mkdir: cannot create directory ‘/no-such-dir’: Permission denied
|
||||
/no-such-dir: No such file or directory.
|
||||
parallel: Cannot chdir to /no-such-dir
|
||||
Exit code 1
|
||||
echo '### --wd no-such-dir - bash'
|
||||
### --wd no-such-dir - bash
|
||||
stdout parallel --wd /no-such-dir -S parallel@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $?
|
||||
mkdir: cannot create directory ‘/no-such-dir’: Permission denied
|
||||
bash: line 0: cd: /no-such-dir: No such file or directory
|
||||
parallel: Cannot chdir to /no-such-dir
|
||||
Exit code 1
|
||||
echo '### bug #42725: csh with \n in variables'
|
||||
### bug #42725: csh with \n in variables
|
||||
not_csh() { echo This is not csh/tcsh; }; export -f not_csh; parallel --env not_csh -S csh@lo not_csh ::: 1; parallel --env not_csh -S tcsh@lo not_csh ::: 1; parallel --env not_csh -S parallel@lo not_csh ::: 1
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
|
||||
This is not csh/tcsh
|
||||
Badly placed ()'s.
|
||||
}: Command not found.
|
||||
Badly placed ()'s.
|
||||
}: Command not found.
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
|
||||
echo '### bug #43358: shellshock breaks exporting functions using --env'
|
||||
### bug #43358: shellshock breaks exporting functions using --env
|
||||
echo shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@localhost funky ::: shellshock-hardened
|
||||
|
|
|
|||
|
|
@ -130,12 +130,10 @@ shell? /bin/bash -c cp `which yash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit e
|
|||
which bash => shell path /bin/bash
|
||||
shell? /bin/bash -c cp `which zsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
|
||||
which bash => shell path /bin/bash
|
||||
MANPATH: Undefined variable.
|
||||
/tmp/SHELL: -c: bad option(s)
|
||||
Local configuration error occurred.
|
||||
Contact the systems administrator for further assistance.
|
||||
SHELL: applet not found
|
||||
MANPATH: Undefined variable.
|
||||
echo '## Known shells -c'
|
||||
## Known shells -c
|
||||
parallel -k "\`which {}\` -c 'parallel -Dinit echo ::: 1' | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
|
||||
|
|
@ -168,12 +166,8 @@ shell? /bin/bash -c `which yash` -c 'parallel -Dinit echo ::: 1' | grep which;
|
|||
which bash => shell path /bin/bash
|
||||
shell? /bin/bash -c `which zsh` -c 'parallel -Dinit echo ::: 1' | grep which;
|
||||
which bash => shell path /bin/bash
|
||||
MANPATH: Undefined variable.
|
||||
MANPATH: Undefined variable.
|
||||
Local configuration error occurred.
|
||||
Contact the systems administrator for further assistance.
|
||||
MANPATH: Undefined variable.
|
||||
MANPATH: Undefined variable.
|
||||
echo '## Known shells |'
|
||||
## Known shells |
|
||||
parallel -k "echo 'parallel -Dinit echo ::: 1' | \`which {}\` | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
|
||||
|
|
@ -197,12 +191,8 @@ which static-sh => shell path /bin/static-sh
|
|||
which tcsh => shell path /usr/bin/tcsh
|
||||
which yash => shell path /usr/bin/yash
|
||||
which zsh => shell path /usr/bin/zsh
|
||||
MANPATH: Undefined variable.
|
||||
MANPATH: Undefined variable.
|
||||
Local configuration error occurred.
|
||||
Contact the systems administrator for further assistance.
|
||||
MANPATH: Undefined variable.
|
||||
MANPATH: Undefined variable.
|
||||
echo '## Started directly from perl'
|
||||
## Started directly from perl
|
||||
perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which
|
||||
|
|
|
|||
|
|
@ -754,7 +754,7 @@ args on cmdline
|
|||
\nice -n1 /bin/bash -c PAR=a\ bash\ -c\ \"echo\ \ \\\$PAR\ b\"
|
||||
a b
|
||||
### Test --nice remote
|
||||
ssh one-server 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ \\\\nice\\\ -n1\\\ /bin/bash\\\ -c\\\ PAR=a\\\\\\\ bash\\\\\\\ -c\\\\\\\ \\\\\\\"echo\\\\\\\ \\\\\\\ \\\\\\\\\\\\\\\$PAR\\\\\\\ b\\\\\\\";
|
||||
ssh one-server exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
|
||||
a b
|
||||
### Test distribute arguments at EOF to 2 jobslots
|
||||
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
|
||||
|
|
|
|||
|
|
@ -75,37 +75,29 @@ OK
|
|||
ls: tmp/parallel.file*: No such file or directory
|
||||
OK
|
||||
Input for ssh
|
||||
parallel@parallel-server1 mkdir -p ./.
|
||||
-l parallel parallel-server1 rsync --server -lDrRze.iLsfx . ./.
|
||||
parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ X\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=X\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; exec perl -e '$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){setpgrp;exec$ENV{SHELL},"-c",@ARGV;die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)' cat\ tmp/parallel.file.\''
|
||||
'\'newlineX\ \>\ tmp/parallel.file.\''
|
||||
'\'newlineX.out\;cat\ tmp/parallel.file.\''
|
||||
'\'newlineX\ \>\ tmp/parallel.file.\''
|
||||
'\'newlineX.out2
|
||||
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
one-server mkdir -p ./.
|
||||
-l parallel one-server rsync --server -lDrRze.iLsfx . ./.
|
||||
one-server exec perl -e '@u=("use","IPC::Open3;");eval"@u";$SIG{CHLD}='IGNORE';my($gzin_fh,$gzout_fh,$eval);my$gzpid=open3($gzin_fh,$gzout_fh,">&STDERR","gzip","-dc");if(my$perlpid=fork){close$gzin_fh;$eval=join"",<$gzout_fh>;close$gzout_fh;}else{close$gzout_fh;print$gzin_fh(pack("H*",join"",@ARGV));close$gzin_fh;exit;}wait;eval$eval;' hex
|
||||
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
'newlineX.out
|
||||
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
'newlineX.out2
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
one-server (rm -f ./tmp/parallel.file.'
|
||||
'newlineX; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
one-server (rm -f ./tmp/parallel.file.'
|
||||
'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
one-server (rm -f ./tmp/parallel.file.'
|
||||
'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 mkdir -p ./.
|
||||
-l parallel parallel-server2 rsync --server -lDrRze.iLsfx . ./.
|
||||
parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ X\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=X\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; exec perl -e '$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){setpgrp;exec$ENV{SHELL},"-c",@ARGV;die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)' cat\ tmp/parallel.file.\''
|
||||
'\'newlineX\ \>\ tmp/parallel.file.\''
|
||||
'\'newlineX.out\;cat\ tmp/parallel.file.\''
|
||||
'\'newlineX\ \>\ tmp/parallel.file.\''
|
||||
'\'newlineX.out2
|
||||
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
one-server mkdir -p ./.
|
||||
-l parallel one-server rsync --server -lDrRze.iLsfx . ./.
|
||||
one-server exec perl -e '@u=("use","IPC::Open3;");eval"@u";$SIG{CHLD}='IGNORE';my($gzin_fh,$gzout_fh,$eval);my$gzpid=open3($gzin_fh,$gzout_fh,">&STDERR","gzip","-dc");if(my$perlpid=fork){close$gzin_fh;$eval=join"",<$gzout_fh>;close$gzout_fh;}else{close$gzout_fh;print$gzin_fh(pack("H*",join"",@ARGV));close$gzin_fh;exit;}wait;eval$eval;' hex
|
||||
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
'newlineX.out
|
||||
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfx . ./parallel.file.'
|
||||
'newlineX.out2
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
one-server (rm -f ./tmp/parallel.file.'
|
||||
'newlineX; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
one-server (rm -f ./tmp/parallel.file.'
|
||||
'newlineX.out; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
one-server (rm -f ./tmp/parallel.file.'
|
||||
'newlineX.out2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ echo /dev/fd/62
|
|||
/dev/fd/62
|
||||
echo foo
|
||||
foo
|
||||
ssh parallel@redhat9.tange.dk ssh centos3.tange.dk 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ perl\\\ -pe\\\ \\\"\\\\\\\$a=1\\\;\\\ print\\\ \\\\\\\$a\\\"\\\ \\\<\\\(echo\\\ foo\\\);
|
||||
perl -pe "\$a=1; print \$a" <(echo foo)
|
||||
1foo
|
||||
### Test quoting of $ in command from profile file
|
||||
perl -pe '$a=1; print $a' <(echo foo)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
echo '### Test --return of weirdly named file'
|
||||
### Test --return of weirdly named file
|
||||
stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b'
|
||||
ssh parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\>aa\\\\\\\<\\\\\\\$\\\\\\\{\\\\\\\#\\\\\\\}\\\\\\\"\\\\\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
|
||||
stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b' | perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'; rm 'aa<${#}" b'
|
||||
ssh one-server exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
|
||||
echo '### Test if remote login shell is csh'
|
||||
### Test if remote login shell is csh
|
||||
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c
|
||||
ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ a\\\|\\\ wc\\\ -w;
|
||||
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c | perl -pe 's/\S*parallel-server\S*/one-server/;s/[a-f0-9]{500,}/hex/;'
|
||||
ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
|
||||
3
|
||||
ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ b\\\|\\\ wc\\\ -w;
|
||||
ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
|
||||
3
|
||||
ssh csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' exec\ perl\ -e\ \'\$SIG\{CHLD\}=sub\{\$done=1\;\}\;\$pid=fork\;unless\(\$pid\)\{setpgrp\;exec\$ENV\{SHELL\},\"-c\",@ARGV\;die\"exec:\$\!\\n\"\;\}do\{\$s=\$s\<1\?0.001+\$s\*1.03:\$s\;select\(undef,undef,undef,\$s\)\;\}until\(\$done\|\|getppid==1\)\;kill\(SIGHUP,-\$\{pid\}\)unless\$done\;wait\;exit\(\$\?\&127\?128+\(\$\?\&127\):1+\$\?\>\>8\)\'\ echo\\\ \\\$PARALLEL_PID\\\ \\\$PARALLEL_SEQ\\\ c\\\|\\\ wc\\\ -w;
|
||||
ssh csh@localhost exec\ perl\ -e\ \'@u=\(\"use\",\"IPC::Open3\;\"\)\;eval\"@u\"\;\$SIG\{CHLD\}=\'IGNORE\'\;my\(\$gzin_fh,\$gzout_fh,\$eval\)\;my\$gzpid=open3\(\$gzin_fh,\$gzout_fh,\"\>\&STDERR\",\"gzip\",\"-dc\"\)\;if\(my\$perlpid=fork\)\{close\$gzin_fh\;\$eval=join\"\",\<\$gzout_fh\>\;close\$gzout_fh\;\}else\{close\$gzout_fh\;print\$gzin_fh\(pack\(\"H\*\",join\"\",@ARGV\)\)\;close\$gzin_fh\;exit\;\}wait\;eval\$eval\;\'\ hex;
|
||||
3
|
||||
echo '### Test {} multiple times in different commands'
|
||||
### Test {} multiple times in different commands
|
||||
|
|
|
|||
Loading…
Reference in a new issue