tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #54352: env_parallel -Slo --nonall myfunc broken in 20180722.

Browse source
This commit is contained in:
Ole Tange 2018-08-27 01:54:11 +02:00
parent a6b96a3472
commit b7e789c02f
12 changed files with 193 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
doc/release_new_version
View file

@ -206,21 +206,20 @@ to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20180822 ('Genova') released [alpha] <<[stable]>> Subject: GNU Parallel 20180922 ('') released <<[stable]>>
GNU Parallel 20180822 ('Genova') [alpha] <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20180922 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
Quote of the month: Quote of the month:
GNU parallel is a thing of magic. I know I'm late to the party but Gnu Parallel is truly amazing!
-- Sam Diaz-Munoz @sociovirology
-- Josh Meyer @joshmeyerphd@twitter
New in this release: New in this release:
* How to install GNU Parallel https://www.techrepublic.com/videos/how-to-install-gnu-parallel/ *
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>> <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>

5
src/env_parallel.pod
View file

@ -20,9 +20,6 @@ environment to GNU B<parallel>.
If the shell function is not loaded, a dummy script will be run If the shell function is not loaded, a dummy script will be run
instead that explains how to install the function. instead that explains how to install the function.
B<env_parallel> is beta quality and not production ready, but please
use it for everyday use and report bugs.
B<env_parallel> is 100 ms slower at startup than pure GNU B<env_parallel> is 100 ms slower at startup than pure GNU
B<parallel>, and takes up to 30% longer to start a job (typically 15 ms). B<parallel>, and takes up to 30% longer to start a job (typically 15 ms).
@ -448,7 +445,7 @@ array with the given name. Multiple B<--env>s can be given.
=back =back
=head2 mksh (beta testing) =head2 mksh
=head3 Installation =head3 Installation

2
src/niceload
View file

@ -24,7 +24,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20180822; $Global::version = 20180824;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

168
src/parallel
View file

@ -160,7 +160,8 @@ sub set_input_source_header {
::debug("init", "Colname: '$s'"); ::debug("init", "Colname: '$s'");
# Replace {colname} with {2} # Replace {colname} with {2}
for(@command,@Global::ret_files,@Global::transfer_files, for(@command,@Global::ret_files,@Global::transfer_files,
$opt::tagstring, $opt::workdir, $opt::results, $opt::retries) { $opt::tagstring, $opt::workdir, $opt::results,
$opt::retries) {
# Skip if undefined # Skip if undefined
$_ or next; $_ or next;
s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
@ -202,7 +203,8 @@ sub halt {
if(not defined $Global::halt_exitstatus) { if(not defined $Global::halt_exitstatus) {
if($Global::halt_pct) { if($Global::halt_pct) {
$Global::halt_exitstatus = $Global::halt_exitstatus =
::ceil($Global::total_failed / $Global::total_started * 100); ::ceil($Global::total_failed /
$Global::total_started * 100);
} elsif($Global::halt_count) { } elsif($Global::halt_count) {
$Global::halt_exitstatus = $Global::halt_exitstatus =
::min(undef_as_zero($Global::total_failed),101); ::min(undef_as_zero($Global::total_failed),101);
@ -226,7 +228,7 @@ sub pipepart_setup {
if($opt::tee) { if($opt::tee) {
# Prepend each command with # Prepend each command with
# < file # < file
my $cat_string = "< ".::shell_quote_scalar($opt::a[0]); my $cat_string = "< ".Q($opt::a[0]);
for(1..$Global::JobQueue->total_jobs()) { for(1..$Global::JobQueue->total_jobs()) {
push @Global::cat_appends, $cat_string; push @Global::cat_appends, $cat_string;
push @Global::cat_prepends, ""; push @Global::cat_prepends, "";
@ -337,7 +339,8 @@ sub find_header {
if($opt::header eq ":") { $opt::header = "(.*\n)"; } if($opt::header eq ":") { $opt::header = "(.*\n)"; }
# Number = number of lines # Number = number of lines
$opt::header =~ s/^(\d+)$/"(.*\n)"x$1/e; $opt::header =~ s/^(\d+)$/"(.*\n)"x$1/e;
while(read($fh,substr($$buf_ref,length $$buf_ref,0),$Global::blocksize)) { while(read($fh,substr($$buf_ref,length $$buf_ref,0),
$Global::blocksize)) {
if($$buf_ref=~s/^($opt::header)//) { if($$buf_ref=~s/^($opt::header)//) {
$header = $1; $header = $1;
last; last;
@ -441,7 +444,7 @@ sub cat_partial {
} }
} }
}); });
return "<". shell_quote_scalar($file) . return "<". Q($file) .
" perl -e '$script' @start_len |"; " perl -e '$script' @start_len |";
} }
@ -667,7 +670,8 @@ sub recstartrecend {
} }
if($opt::regexp) { if($opt::regexp) {
# If $recstart/$recend contains '|' this should only apply to the regexp # If $recstart/$recend contains '|'
# this should only apply to the regexp
$recstart = "(?:".$recstart.")"; $recstart = "(?:".$recstart.")";
$recend = "(?:".$recend.")"; $recend = "(?:".$recend.")";
} else { } else {
@ -726,7 +730,8 @@ sub nindex {
if($job->block_length() > 0) { if($job->block_length() > 0) {
$written += $job->non_blocking_write(); $written += $job->non_blocking_write();
} else { } else {
$job->set_block($header_ref,$buffer_ref,$endpos,$recstart,$recend); $job->set_block($header_ref, $buffer_ref,
$endpos, $recstart, $recend);
$block_passed = 1; $block_passed = 1;
$job->set_virgin(0); $job->set_virgin(0);
$written += $job->non_blocking_write(); $written += $job->non_blocking_write();
@ -740,8 +745,8 @@ sub nindex {
if($job->block_length() > 0) { if($job->block_length() > 0) {
$written += $job->non_blocking_write(); $written += $job->non_blocking_write();
} else { } else {
$job->set_block($header_ref,$buffer_ref, $job->set_block($header_ref, $buffer_ref,
$endpos,$recstart,$recend); $endpos, $recstart, $recend);
$block_passed = 1; $block_passed = 1;
$job->set_virgin(0); $job->set_virgin(0);
$written += $job->non_blocking_write(); $written += $job->non_blocking_write();
@ -864,12 +869,14 @@ sub write_record_to_pipe {
# @Global::virgin_jobs # @Global::virgin_jobs
# Returns: # Returns:
# Number of chunks written (0 or 1) # Number of chunks written (0 or 1)
my ($chunk_number,$header_ref,$buffer_ref,$recstart,$recend,$endpos) = @_; my ($chunk_number, $header_ref, $buffer_ref,
$recstart, $recend, $endpos) = @_;
if($endpos == 0) { return 0; } if($endpos == 0) { return 0; }
if(vec($Global::job_already_run,$chunk_number,1)) { return 1; } if(vec($Global::job_already_run,$chunk_number,1)) { return 1; }
if($opt::roundrobin) { if($opt::roundrobin) {
# Write the block to one of the already running jobs # Write the block to one of the already running jobs
return round_robin_write($header_ref,$buffer_ref,$recstart,$recend,$endpos); return round_robin_write($header_ref, $buffer_ref,
$recstart, $recend, $endpos);
} }
# If no virgin found, backoff # If no virgin found, backoff
my $sleep = 0.0001; # 0.01 ms - better performance on highend my $sleep = 0.0001; # 0.01 ms - better performance on highend
@ -889,7 +896,8 @@ sub write_record_to_pipe {
# Copy $buffer[0..$endpos] to $job->{'block'} # Copy $buffer[0..$endpos] to $job->{'block'}
# Remove rec_sep # Remove rec_sep
# Run $job->add_transfersize # Run $job->add_transfersize
$job->set_block($header_ref,$buffer_ref,$endpos,$recstart,$recend); $job->set_block($header_ref, $buffer_ref, $endpos,
$recstart, $recend);
if(fork()) { if(fork()) {
# Skip # Skip
} else { } else {
@ -908,7 +916,7 @@ sub write_record_to_pipe {
substr($$buffer_ref,$endpos,length $$buffer_ref) = ""; substr($$buffer_ref,$endpos,length $$buffer_ref) = "";
# Remove rec_sep # Remove rec_sep
if($opt::remove_rec_sep) { if($opt::remove_rec_sep) {
Job::remove_rec_sep($buffer_ref,$recstart,$recend); Job::remove_rec_sep($buffer_ref, $recstart, $recend);
} }
$job->write($header_ref); $job->write($header_ref);
$job->write($buffer_ref); $job->write($buffer_ref);
@ -931,7 +939,8 @@ sub acquire_semaphore {
# Returns: # Returns:
# The semaphore to be released when jobs is complete # The semaphore to be released when jobs is complete
$Global::host{':'} = SSHLogin->new(":"); $Global::host{':'} = SSHLogin->new(":");
my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running()); my $sem = Semaphore->new($Semaphore::name,
$Global::host{':'}->max_jobs_running());
$sem->acquire(); $sem->acquire();
if($Semaphore::fg) { if($Semaphore::fg) {
# skip # skip
@ -974,7 +983,8 @@ sub options_hash {
"group" => \$opt::group, "group" => \$opt::group,
"g" => \$opt::retired, "g" => \$opt::retired,
"ungroup|u" => \$opt::ungroup, "ungroup|u" => \$opt::ungroup,
"linebuffer|linebuffered|line-buffer|line-buffered|lb" => \$opt::linebuffer, "linebuffer|linebuffered|line-buffer|line-buffered|lb"
=> \$opt::linebuffer,
"tmux" => \$opt::tmux, "tmux" => \$opt::tmux,
"tmuxpane" => \$opt::tmuxpane, "tmuxpane" => \$opt::tmuxpane,
"null|0" => \$opt::null, "null|0" => \$opt::null,
@ -1001,8 +1011,10 @@ sub options_hash {
"number-of-sockets" => \$opt::number_of_sockets, "number-of-sockets" => \$opt::number_of_sockets,
"number-of-cores" => \$opt::number_of_cores, "number-of-cores" => \$opt::number_of_cores,
"number-of-threads" => \$opt::number_of_threads, "number-of-threads" => \$opt::number_of_threads,
"use-sockets-instead-of-threads" => \$opt::use_sockets_instead_of_threads, "use-sockets-instead-of-threads"
"use-cores-instead-of-threads" => \$opt::use_cores_instead_of_threads, => \$opt::use_sockets_instead_of_threads,
"use-cores-instead-of-threads"
=> \$opt::use_cores_instead_of_threads,
"use-cpus-instead-of-cores" => \$opt::use_cpus_instead_of_cores, "use-cpus-instead-of-cores" => \$opt::use_cpus_instead_of_cores,
"shellquote|shell_quote|shell-quote" => \$opt::shellquote, "shellquote|shell_quote|shell-quote" => \$opt::shellquote,
"nice=i" => \$opt::nice, "nice=i" => \$opt::nice,
@ -1030,7 +1042,8 @@ sub options_hash {
"rsync-opts|rsyncopts=s" => \$opt::rsync_opts, "rsync-opts|rsyncopts=s" => \$opt::rsync_opts,
"tmpdir|tempdir=s" => \$opt::tmpdir, "tmpdir|tempdir=s" => \$opt::tmpdir,
"use-compress-program|compress-program=s" => \$opt::compress_program, "use-compress-program|compress-program=s" => \$opt::compress_program,
"use-decompress-program|decompress-program=s" => \$opt::decompress_program, "use-decompress-program|decompress-program=s"
=> \$opt::decompress_program,
"compress" => \$opt::compress, "compress" => \$opt::compress,
"tty" => \$opt::tty, "tty" => \$opt::tty,
"T" => \$opt::retired, "T" => \$opt::retired,
@ -1100,7 +1113,8 @@ sub options_hash {
"wait" => \$opt::wait, "wait" => \$opt::wait,
# Shebang #!/usr/bin/parallel --shebang # Shebang #!/usr/bin/parallel --shebang
"shebang|hashbang" => \$opt::shebang, "shebang|hashbang" => \$opt::shebang,
"internal-pipe-means-argfiles" => \$opt::internal_pipe_means_argfiles, "internal-pipe-means-argfiles"
=> \$opt::internal_pipe_means_argfiles,
"Y" => \$opt::retired, "Y" => \$opt::retired,
"skip-first-line" => \$opt::skip_first_line, "skip-first-line" => \$opt::skip_first_line,
"bug" => \$opt::bug, "bug" => \$opt::bug,
@ -1541,7 +1555,7 @@ sub check_invalid_option_combinations {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20180822; $Global::version = 20180824;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -2046,7 +2060,7 @@ sub read_options {
# remove --hashbang if it is set # remove --hashbang if it is set
$opt::shebang .= ($ARGV[0] =~ s/^--hashbang *//); $opt::shebang .= ($ARGV[0] =~ s/^--hashbang *//);
if($opt::shebang) { if($opt::shebang) {
my $argfile = shell_quote_scalar(pop @ARGV); my $argfile = Q(pop @ARGV);
# exec myself to split $ARGV[0] into separate fields # exec myself to split $ARGV[0] into separate fields
exec "$0 --skip-first-line -a $argfile @ARGV"; exec "$0 --skip-first-line -a $argfile @ARGV";
} }
@ -2069,7 +2083,7 @@ sub read_options {
} else { } else {
@options = shift @ARGV; @options = shift @ARGV;
} }
my $script = shell_quote_scalar(shift @ARGV); my $script = Q(shift @ARGV);
# exec myself to split $ARGV[0] into separate fields # exec myself to split $ARGV[0] into separate fields
exec "$0 --internal-pipe-means-argfiles @options @parser $script ". exec "$0 --internal-pipe-means-argfiles @options @parser $script ".
"::: @ARGV"; "::: @ARGV";
@ -2352,6 +2366,7 @@ sub shell_quote_scalar {
sub Q { sub Q {
# Q alias for ::shell_quote_scalar # Q alias for ::shell_quote_scalar
no warnings 'redefine';
*Q = \&::shell_quote_scalar; *Q = \&::shell_quote_scalar;
return Q(@_); return Q(@_);
} }
@ -2372,7 +2387,7 @@ sub shell_quote_file {
$a = "./".$a; $a = "./".$a;
} }
} }
return ::Q($a); return Q($a);
} }
sub shell_words { sub shell_words {
@ -3766,7 +3781,7 @@ sub parallelized_host_filtering {
my $cmd = my $cmd =
"$0 -j0 --timeout 10 --joblog - --plain --delay 0.1 --retries 3 ". "$0 -j0 --timeout 10 --joblog - --plain --delay 0.1 --retries 3 ".
"--tag --tagstring '{1}' -0 --colsep '\t' -k eval '{2}' && true "; "--tag --tagstring '{1}' -0 --colsep '\t' -k eval '{2}' && true ";
$cmd = $Global::shell." -c ".::shell_quote_scalar($cmd); $cmd = $Global::shell." -c ".Q($cmd);
::debug("init", $cmd, "\n"); ::debug("init", $cmd, "\n");
my @out; my @out;
my $prepend = ""; my $prepend = "";
@ -3895,16 +3910,16 @@ sub onall {
((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""), ((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""),
((defined $opt::ungroup) ? "-u" : ""), ((defined $opt::ungroup) ? "-u" : ""),
((defined $opt::tee) ? "--tee" : ""), ((defined $opt::tee) ? "--tee" : ""),
((defined $opt::workdir) ? "--wd ".::shell_quote_scalar($opt::workdir) : ""), ((defined $opt::workdir) ? "--wd ".Q($opt::workdir) : ""),
(@Global::transfer_files ? map { "--tf ".::shell_quote_scalar($_) } (@Global::transfer_files ? map { "--tf ".Q($_) }
@Global::transfer_files : ""), @Global::transfer_files : ""),
(@Global::ret_files ? map { "--return ".::shell_quote_scalar($_) } (@Global::ret_files ? map { "--return ".Q($_) }
@Global::ret_files : ""), @Global::ret_files : ""),
(@opt::env ? map { "--env ".::shell_quote_scalar($_) } @opt::env : ""), (@opt::env ? map { "--env ".Q($_) } @opt::env : ""),
(map { "-v" } @opt::v), (map { "-v" } @opt::v),
); );
::debug("init", "| $0 $options\n"); ::debug("init", "| $0 $options\n");
open(my $parallel_fh, "|-", "$0 --will-cite -j0 $options") || open(my $parallel_fh, "|-", "$0 -0 --will-cite -j0 $options") ||
::die_bug("This does not run GNU Parallel: $0 $options"); ::die_bug("This does not run GNU Parallel: $0 $options");
my @joblogs; my @joblogs;
for my $host (sort keys %Global::host) { for my $host (sort keys %Global::host) {
@ -3915,16 +3930,20 @@ sub onall {
$joblog = "--joblog $joblog"; $joblog = "--joblog $joblog";
} }
my $quad = $opt::arg_file_sep || "::::"; my $quad = $opt::arg_file_sep || "::::";
::debug("init", "$0 $suboptions -j1 $joblog ", # If PARALLEL_ENV is set: Pass it on
my $penv=$Global::parallel_env ?
"PARALLEL_ENV=".Q($Global::parallel_env) :
'';
::debug("init", "$penv $0 $suboptions -j1 $joblog ",
((defined $opt::tag) ? ((defined $opt::tag) ?
"--tagstring ".shell_quote_scalar($sshlogin->string()) : ""), "--tagstring ".Q($sshlogin->string()) : ""),
" -S ", shell_quote_scalar($sshlogin->string())," ", " -S ", Q($sshlogin->string())," ",
join(" ",shell_quote(@command))," $quad @argfiles\n"); join(" ",shell_quote(@command))," $quad @argfiles\n");
print $parallel_fh "$0 $suboptions -j1 $joblog ", print $parallel_fh "$penv $0 $suboptions -j1 $joblog ",
((defined $opt::tag) ? ((defined $opt::tag) ?
"--tagstring ".shell_quote_scalar($sshlogin->string()) : ""), "--tagstring ".Q($sshlogin->string()) : ""),
" -S ", shell_quote_scalar($sshlogin->string())," ", " -S ", Q($sshlogin->string())," ",
join(" ",shell_quote(@command))," $quad @argfiles\n"; join(" ",shell_quote(@command))," $quad @argfiles\0";
} }
close $parallel_fh; close $parallel_fh;
$Global::exitstatus = $? >> 8; $Global::exitstatus = $? >> 8;
@ -5528,7 +5547,7 @@ sub memfree_recompute {
$perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}'; $perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}';
} }
$perlscript =~ s/[\t\n ]+/ /g; $perlscript =~ s/[\t\n ]+/ /g;
$script = "perl -e " . ::shell_quote_scalar($perlscript); $script = "perl -e " . ::Q($perlscript);
} }
return $script; return $script;
} }
@ -5647,7 +5666,7 @@ sub swap_activity {
$swap_activity = swapactivityscript(); $swap_activity = swapactivityscript();
if($self->{'string'} ne ":") { if($self->{'string'} ne ":") {
$swap_activity = $self->sshcommand() . " " . $self->serverlogin() . " " . $swap_activity = $self->sshcommand() . " " . $self->serverlogin() . " " .
::shell_quote_scalar($swap_activity); ::Q($swap_activity);
} }
# Run swap_activity measuring. # Run swap_activity measuring.
# As the command can take long to run if run remote # As the command can take long to run if run remote
@ -5784,7 +5803,7 @@ sub swap_activity {
$perlscript .= 'if($^O eq "'.$os.'") { print `'.$vmstat{$os}[0].' | awk "{print ' . $perlscript .= 'if($^O eq "'.$os.'") { print `'.$vmstat{$os}[0].' | awk "{print ' .
$vmstat{$os}[1] . '}"` }'; $vmstat{$os}[1] . '}"` }';
} }
$script = "perl -e " . ::shell_quote_scalar($perlscript); $script = "perl -e " . ::Q($perlscript);
} }
return $script; return $script;
} }
@ -5944,7 +5963,7 @@ sub loadavg {
my $cmd = ""; my $cmd = "";
if($self->{'string'} ne ":") { if($self->{'string'} ne ":") {
$cmd = $self->sshcommand() . " " . $self->serverlogin() . " " . $cmd = $self->sshcommand() . " " . $self->serverlogin() . " " .
::shell_quote_scalar(loadavg_cmd()); ::Q(loadavg_cmd());
} else { } else {
$cmd .= loadavg_cmd(); $cmd .= loadavg_cmd();
} }
@ -6102,8 +6121,8 @@ sub compute_number_of_processes {
# The child takes one process slot # The child takes one process slot
# It will be killed later # It will be killed later
$SIG{'TERM'} = $Global::original_sig{'TERM'}; $SIG{'TERM'} = $Global::original_sig{'TERM'};
if($^O eq "cygwin" or $^O eq "msys") { if($^O eq "cygwin" or $^O eq "msys" or $^O eq "nto") {
# The exec does not work on Cygwin # The exec does not work on Cygwin and QNX
sleep 10101010; sleep 10101010;
} else { } else {
# 'exec sleep' takes less RAM than sleeping in perl # 'exec sleep' takes less RAM than sleeping in perl
@ -6932,7 +6951,7 @@ sub sshcommand_of_sshlogin {
open(STDIN,"<","/dev/null"); open(STDIN,"<","/dev/null");
# Run a sleep that outputs data, so it will discover # Run a sleep that outputs data, so it will discover
# if the ssh connection closes. # if the ssh connection closes.
my $sleep = ::shell_quote_scalar my $sleep = ::Q
('$|=1;while(1){sleep 1;print "foo\n"}'); ('$|=1;while(1){sleep 1;print "foo\n"}');
my @master = ($opt::ssh, "-MTS", my @master = ($opt::ssh, "-MTS",
$control_path, $serverlogin, "--", "perl", "-e", $control_path, $serverlogin, "--", "perl", "-e",
@ -6995,7 +7014,7 @@ sub rsync_transfer_cmd {
$file = ::shell_quote_file($file); $file = ::shell_quote_file($file);
my $sshcmd = $self->sshcommand(); my $sshcmd = $self->sshcommand();
my $rsync_opts = $ENV{'PARALLEL_RSYNC_OPTS'}. my $rsync_opts = $ENV{'PARALLEL_RSYNC_OPTS'}.
" -e".::shell_quote_scalar($sshcmd); " -e".::Q($sshcmd);
my $serverlogin = $self->serverlogin(); my $serverlogin = $self->serverlogin();
# Make dir if it does not exist # Make dir if it does not exist
return "$sshcmd $serverlogin -- mkdir -p $rsync_destdir && " . return "$sshcmd $serverlogin -- mkdir -p $rsync_destdir && " .
@ -7028,15 +7047,15 @@ sub cleanup_cmd {
$dir .= $_."/"; $dir .= $_."/";
unshift @rmdir, ::shell_quote_file($dir); unshift @rmdir, ::shell_quote_file($dir);
} }
my $rmdir = @rmdir ? "sh -c ".::shell_quote_scalar("rmdir @rmdir 2>/dev/null;") : ""; my $rmdir = @rmdir ? "sh -c ".::Q("rmdir @rmdir 2>/dev/null;") : "";
if(defined $opt::workdir and $opt::workdir eq "...") { if(defined $opt::workdir and $opt::workdir eq "...") {
$rmdir .= ::shell_quote_scalar("rm -rf " . ::shell_quote_file($workdir).';'); $rmdir .= ::Q("rm -rf " . ::shell_quote_file($workdir).';');
} }
$f = ::shell_quote_file($f); $f = ::shell_quote_file($f);
my $sshcmd = $self->sshcommand(); my $sshcmd = $self->sshcommand();
my $serverlogin = $self->serverlogin(); my $serverlogin = $self->serverlogin();
return "$sshcmd $serverlogin -- ".::shell_quote_scalar("rm -f $f; $rmdir"); return "$sshcmd $serverlogin -- ".::Q("rm -f $f; $rmdir");
} }
{ {
@ -7511,7 +7530,7 @@ sub empty_input_wrapper {
exit ($?&127 ? 128+($?&127) : 1+$?>>8) exit ($?&127 ? 128+($?&127) : 1+$?>>8)
} }
}); });
::debug("run",'Empty wrap: perl -e '.::shell_quote_scalar($script)."\n"); ::debug("run",'Empty wrap: perl -e '.::Q($script)."\n");
if($Global::cshell if($Global::cshell
and and
length $command > 499) { length $command > 499) {
@ -7519,12 +7538,12 @@ sub empty_input_wrapper {
# $command = "perl -e '".base64_zip_eval()."' ". # $command = "perl -e '".base64_zip_eval()."' ".
# join" ",string_zip_base64( # join" ",string_zip_base64(
# 'exec "'.::perl_quote_scalar($command).'"'); # 'exec "'.::perl_quote_scalar($command).'"');
return 'perl -e '.::shell_quote_scalar($script)." ". return 'perl -e '.::Q($script)." ".
base64_wrap("exec \"$Global::shell\",'-c',\"". base64_wrap("exec \"$Global::shell\",'-c',\"".
::perl_quote_scalar($command).'"'); ::perl_quote_scalar($command).'"');
} else { } else {
return 'perl -e '.::shell_quote_scalar($script)." ". return 'perl -e '.::Q($script)." ".
$Global::shell." -c ".::shell_quote_scalar($command); $Global::shell." -c ".::Q($command);
} }
} }
@ -7911,7 +7930,7 @@ sub wrapped {
# Prepend /bin/echo (echo no-/bin is wrong in csh) # Prepend /bin/echo (echo no-/bin is wrong in csh)
# and quote twice # and quote twice
$command = "/bin/echo " . $command = "/bin/echo " .
::shell_quote_scalar(::shell_quote_scalar($command)); ::Q(::Q($command));
} }
if($Global::parallel_env) { if($Global::parallel_env) {
# If $PARALLEL_ENV set, put that in front of the command # If $PARALLEL_ENV set, put that in front of the command
@ -7919,7 +7938,7 @@ sub wrapped {
if($Global::shell =~ /zsh/) { if($Global::shell =~ /zsh/) {
# The extra 'eval' will make aliases work, too # The extra 'eval' will make aliases work, too
$command = $Global::parallel_env."\n". $command = $Global::parallel_env."\n".
"eval ".::shell_quote_scalar($command); "eval ".::Q($command);
} else { } else {
$command = $Global::parallel_env."\n".$command; $command = $Global::parallel_env."\n".$command;
} }
@ -7946,10 +7965,7 @@ sub wrapped {
# wait; rm {} # wait; rm {}
# without affecting $? # without affecting $?
$command = fifo_wrap(). " ". $command = fifo_wrap(). " ".
$Global::shell. " ". $Global::shell. " ". ::Q($command). ' $PARALLEL_TMP'. ';';
::shell_quote_scalar($command).
' $PARALLEL_TMP'.
';';
} }
# Wrap with ssh + tranferring of files # Wrap with ssh + tranferring of files
$command = $self->sshlogin_wrap($command); $command = $self->sshlogin_wrap($command);
@ -8104,7 +8120,7 @@ sub base64_wrap {
my $eval_string = shift; my $eval_string = shift;
return return
"perl -e ". "perl -e ".
::shell_quote_scalar(base64_zip_eval())." ". ::Q(base64_zip_eval())." ".
join" ",::shell_quote(string_zip_base64($eval_string)); join" ",::shell_quote(string_zip_base64($eval_string));
} }
@ -8304,7 +8320,7 @@ sub sshlogin_wrap {
# csh does not deal well with $ENV with \n # csh does not deal well with $ENV with \n
$self->{'sshlogin_wrap'} = base64_wrap($perl_code); $self->{'sshlogin_wrap'} = base64_wrap($perl_code);
} else { } else {
$self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($perl_code); $self->{'sshlogin_wrap'} = "perl -e ".::Q($perl_code);
} }
} else { } else {
$self->{'sshlogin_wrap'} = $command; $self->{'sshlogin_wrap'} = $command;
@ -8321,10 +8337,8 @@ sub sshlogin_wrap {
my $remote_command = $pwd.$envset.$bashfuncset. my $remote_command = $pwd.$envset.$bashfuncset.
'@ARGV="'.::perl_quote_scalar($command).'";'. '@ARGV="'.::perl_quote_scalar($command).'";'.
monitor_parent_sshd_script(); monitor_parent_sshd_script();
$quoted_remote_command = "perl -e ". $quoted_remote_command = "perl -e ". ::Q($remote_command);
::shell_quote_scalar($remote_command); my $dq_remote_command = ::Q($quoted_remote_command);
my $dq_remote_command =
::shell_quote_scalar($quoted_remote_command);
if(length $dq_remote_command > 999 if(length $dq_remote_command > 999
or or
not $csh_friendly not $csh_friendly
@ -8333,8 +8347,7 @@ sub sshlogin_wrap {
# csh does not deal well with > 1000 chars in one word # csh does not deal well with > 1000 chars in one word
# csh does not deal well with $ENV with \n # csh does not deal well with $ENV with \n
$quoted_remote_command = $quoted_remote_command =
"perl -e ". "perl -e ". ::Q(::Q(base64_zip_eval()))." ".
::shell_quote_scalar(::shell_quote_scalar(base64_zip_eval()))." ".
join" ",::shell_quote(::shell_quote(string_zip_base64($remote_command))); join" ",::shell_quote(::shell_quote(string_zip_base64($remote_command)));
} else { } else {
$quoted_remote_command = $dq_remote_command; $quoted_remote_command = $dq_remote_command;
@ -8449,8 +8462,7 @@ sub sshreturn {
my $sshlogin = $self->sshlogin(); my $sshlogin = $self->sshlogin();
my $sshcmd = $sshlogin->sshcommand(); my $sshcmd = $sshlogin->sshcommand();
my $serverlogin = $sshlogin->serverlogin(); my $serverlogin = $sshlogin->serverlogin();
my $rsync_opts = $ENV{'PARALLEL_RSYNC_OPTS'}. my $rsync_opts = $ENV{'PARALLEL_RSYNC_OPTS'}. " -e". ::Q($sshcmd);
" -e".::shell_quote_scalar($sshcmd);
my $pre = ""; my $pre = "";
for my $file ($self->return()) { for my $file ($self->return()) {
$file =~ s:^\./::g; # Remove ./ if any $file =~ s:^\./::g; # Remove ./ if any
@ -8471,9 +8483,8 @@ sub sshreturn {
my $nobasedir = $file; my $nobasedir = $file;
$nobasedir =~ s:.*/\./::; $nobasedir =~ s:.*/\./::;
$cd = ::shell_quote_file(::dirname($nobasedir)); $cd = ::shell_quote_file(::dirname($nobasedir));
my $rsync_cd = '--rsync-path='.::shell_quote_scalar("cd $wd$cd; rsync"); my $rsync_cd = '--rsync-path='.::Q("cd $wd$cd; rsync");
my $basename = my $basename = ::Q(::shell_quote_file(::basename($file)));
::shell_quote_scalar(::shell_quote_file(::basename($file)));
# --return # --return
# mkdir -p /home/tange/dir/subdir/; # mkdir -p /home/tange/dir/subdir/;
# rsync (--protocol 30) -rlDzR # rsync (--protocol 30) -rlDzR
@ -8502,7 +8513,7 @@ sub sshcleanup {
$cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";"; $cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";";
} }
if(defined $opt::workdir and $opt::workdir eq "...") { if(defined $opt::workdir and $opt::workdir eq "...") {
$cleancmd .= "$sshcmd $serverlogin -- rm -rf " . ::shell_quote_scalar($workdir).';'; $cleancmd .= "$sshcmd $serverlogin -- rm -rf " . ::Q($workdir).';';
} }
return $cleancmd; return $cleancmd;
} }
@ -8817,7 +8828,7 @@ sub interactive_start {
# ; causes problems # ; causes problems
# ascii 194-245 annoys tmux # ascii 194-245 annoys tmux
$title =~ tr/[\011-\016;\302-\365]/ /s; $title =~ tr/[\011-\016;\302-\365]/ /s;
$title = ::shell_quote_scalar($title); $title = ::Q($title);
my $l_act = length($actual_command); my $l_act = length($actual_command);
my $l_tit = length($title); my $l_tit = length($title);
@ -8825,7 +8836,7 @@ sub interactive_start {
# The line to run contains a 118 chars extra code + the title 2x # The line to run contains a 118 chars extra code + the title 2x
my $l_tot = 2 * $l_tit + $l_act + $l_fifo; my $l_tot = 2 * $l_tit + $l_act + $l_fifo;
my $quoted_space75 = ::shell_quote_scalar(" ")x75; my $quoted_space75 = ::Q(" ")x75;
while($l_tit < 1000 and while($l_tit < 1000 and
( (
(890 < $l_tot and $l_tot < 1350) (890 < $l_tot and $l_tot < 1350)
@ -8869,7 +8880,7 @@ sub interactive_start {
return "mkfifo $tmpfifo && $tmux ". return "mkfifo $tmpfifo && $tmux ".
# Run in tmux # Run in tmux
::shell_quote_scalar ::Q
( (
"(".$actual_command.');'. "(".$actual_command.');'.
# The triple print is needed - otherwise the testsuite fails # The triple print is needed - otherwise the testsuite fails
@ -10158,7 +10169,7 @@ sub replace_placeholders {
if($_ eq "\0spc" or $_ eq "\0end") { if($_ eq "\0spc" or $_ eq "\0end") {
# \0spc splits quotable groups # \0spc splits quotable groups
if($quote) { if($quote) {
CORE::push @quoted, ::shell_quote_scalar(join"",@quotegroup); CORE::push @quoted, ::Q(join"",@quotegroup);
} else { } else {
CORE::push @quoted, join"",@quotegroup; CORE::push @quoted, join"",@quotegroup;
} }
@ -11167,10 +11178,9 @@ sub new {
sub Q { sub Q {
# Q alias for ::shell_quote_scalar # Q alias for ::shell_quote_scalar
# Run shell_quote_scalar once to set the reference to the sub no warnings 'redefine';
my $a = ::shell_quote_scalar(@_);
*Q = \&::shell_quote_scalar; *Q = \&::shell_quote_scalar;
return $a; return Q(@_);
} }
sub pQ { sub pQ {
@ -11239,7 +11249,7 @@ sub total_jobs {
$self->{'cache'}{$perlexpr} = $_; $self->{'cache'}{$perlexpr} = $_;
} }
# Return the value quoted if needed # Return the value quoted if needed
return($quote ? ::shell_quote_scalar($self->{'cache'}{$perlexpr}) return($quote ? Q($self->{'cache'}{$perlexpr})
: $self->{'cache'}{$perlexpr}); : $self->{'cache'}{$perlexpr});
} }
} }

16
src/parallel.pod
View file

@ -1460,20 +1460,20 @@ on remote computers).
Print the number of physical CPU cores and exit. Print the number of physical CPU cores and exit.
=item B<--number-of-cores> (beta testing) =item B<--number-of-cores>
Print the number of physical CPU cores and exit (used by GNU B<parallel> itself Print the number of physical CPU cores and exit (used by GNU B<parallel> itself
to determine the number of physical CPU cores on remote computers). to determine the number of physical CPU cores on remote computers).
=item B<--number-of-sockets> (beta testing) =item B<--number-of-sockets>
Print the number of filled CPU sockets and exit (used by GNU Print the number of filled CPU sockets and exit (used by GNU
B<parallel> itself to determine the number of filled CPU sockets on B<parallel> itself to determine the number of filled CPU sockets on
remote computers). remote computers).
=item B<--number-of-threads> (beta testing) =item B<--number-of-threads>
Print the number of hyperthreaded CPU cores and exit (used by GNU Print the number of hyperthreaded CPU cores and exit (used by GNU
B<parallel> itself to determine the number of hyperthreaded CPU cores B<parallel> itself to determine the number of hyperthreaded CPU cores
@ -1619,9 +1619,9 @@ it to the command.
Only used with B<--pipe>. Only used with B<--pipe>.
=item B<--results> I<name> (beta testing) =item B<--results> I<name>
=item B<--res> I<name> (beta testing) =item B<--res> I<name>
Save the output into files. Save the output into files.
@ -2058,7 +2058,7 @@ Use the replacement string I<replace-str> instead of B<{#}> for
job sequence number. job sequence number.
=item B<--session> (beta testing) =item B<--session>
Record names in current environment in B<$PARALLEL_IGNORED_NAMES> and Record names in current environment in B<$PARALLEL_IGNORED_NAMES> and
exit. Only used with B<env_parallel>. Aliases, functions, and exit. Only used with B<env_parallel>. Aliases, functions, and
@ -2571,9 +2571,9 @@ Use the replacement string I<replace-str> instead of B<{.}> for input
line without extension. line without extension.
=item B<--use-sockets-instead-of-threads> (beta testing) =item B<--use-sockets-instead-of-threads>
=item B<--use-cores-instead-of-threads> (beta testing) =item B<--use-cores-instead-of-threads>
=item B<--use-cpus-instead-of-cores> (obsolete) =item B<--use-cpus-instead-of-cores> (obsolete)

6
src/parset.pod
View file

@ -28,7 +28,9 @@ If I<variablename> is a single variable name, this will be treated as
the destination variable and made into an array. the destination variable and made into an array.
If I<variablename> contains multiple names separated by ',' or space, If I<variablename> contains multiple names separated by ',' or space,
the names will be the destination variables. the names will be the destination variables. The number of names must
be at least the number of jobs - otherwise some tmp files will not be
cleaned up.
=head1 OPTIONS =head1 OPTIONS
@ -39,7 +41,7 @@ variable.
=head1 SUPPORTED SHELLS =head1 SUPPORTED SHELLS
=head2 Bash/Zsh/Ksh/(Mksh beta testing) =head2 Bash/Zsh/Ksh/Mksh
=head3 Examples =head3 Examples

2
src/sql
View file

@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20180822; $Global::version = 20180824;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

8
testsuite/tests-to-run/parallel-local-ssh5.sh
View file

@ -48,6 +48,14 @@ par_pipe_retries() {
stdout parallel --retries 2 --roundrobin echo ::: should fail stdout parallel --retries 2 --roundrobin echo ::: should fail
} }
par_env_parallel_onall() {
echo "bug #54352: env_parallel -Slo --nonall myfunc broken in 20180722"
. `which env_parallel.bash`
doit() { echo Myfunc "$@"; }
env_parallel -Slo --onall doit ::: works
env_parallel -Slo --nonall doit works
}
export -f $(compgen -A function | grep par_) export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort -r | compgen -A function | grep par_ | sort -r |
parallel --joblog /tmp/jl-`basename $0` -j3 --tag -k --delay 0.1 --retries 3 '{} 2>&1' parallel --joblog /tmp/jl-`basename $0` -j3 --tag -k --delay 0.1 --retries 3 '{} 2>&1'

28
testsuite/tests-to-run/parallel-local-ssh7.sh
View file

@ -2032,9 +2032,9 @@ par_ash_parset() {
# env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo # env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
# echo "${myarray[*]}" # echo "${myarray[*]}"
# echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" # echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOF _EOF
) )
@ -2090,9 +2090,9 @@ par_bash_parset() {
env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
echo "${myarray[*]}" echo "${myarray[*]}"
echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOF _EOF
) )
@ -2157,9 +2157,9 @@ par_dash_parset() {
# env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo # env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
# echo "${myarray[*]}" # echo "${myarray[*]}"
# echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" # echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOF _EOF
) )
@ -2219,9 +2219,9 @@ par_ksh_parset() {
env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
echo "${myarray[*]}" echo "${myarray[*]}"
echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOF _EOF
) )
@ -2277,9 +2277,9 @@ par_mksh_parset() {
env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
echo "${myarray[*]}" echo "${myarray[*]}"
echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOF _EOF
) )
@ -2363,9 +2363,9 @@ par_sh_parset() {
# env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo # env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
# echo "${myarray[*]}" # echo "${myarray[*]}"
# echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" # echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOF _EOF
) )
@ -2433,9 +2433,9 @@ par_zsh_parset() {
env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo
echo "${myarray[*]}" echo "${myarray[*]}"
echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" echo "${myarray[4]} ${myarray[5]} ${myarray[6]}"
parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
env_parset a,b,c 'echo {};exit {}' ::: 0 1 1 0 env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0
echo Exit value 2 = $? echo Exit value 2 = $?
_EOS`" _EOS`"
_EOF _EOF

37
testsuite/tests-to-run/parallel-polarhome.sh
View file

@ -5,21 +5,21 @@ unset TIMEOUT
. `which env_parallel.bash` . `which env_parallel.bash`
env_parallel --session env_parallel --session
P_ALL="qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax ultrix minix irix hurd beaglebone cubieboard2" P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax ultrix minix irix hurd beaglebone cubieboard2"
# P_NOTWORKING="vax alpha openstep"
# P_NOTWORKING_YET="ultrix irix"
#
# P_WORKING="openbsd tru64 debian freebsd redhat netbsd macosx miros centos unixware pidora ubuntu scosysv raspbian solaris-x86 aix mandriva debian-ppc suse solaris hpux openindiana hpux-ia64"
# P_WORKING="openbsd tru64 debian redhat netbsd macosx miros centos unixware pidora scosysv raspbian solaris-x86 aix mandriva debian-ppc suse solaris hpux hurd freebsd ubuntu openindiana"
# P_TEMPORARILY_BROKEN="minix dragonfly hpux-ia64 beaglebone cubieboard2"
P="$P_ALL" P="$P_ALL"
# 2018-04-22 MAXTIME=20 # 2018-04-22 MAXTIME=20
MAXTIME=25 MAXTIME=25
RETRIES=4 RETRIES=4
MAXPROC=150
parallel --retries $RETRIES rsync -a /usr/local/bin/{parallel,env_parallel,env_parallel.*,parcat} ::: ubuntu.polarhome.com:bin/ # select a running master (debian or ubuntu)
MASTER=$(parallel -j0 --halt now,success=1 ssh {} echo {} \
::: {suse,ubuntu,debian}.polarhome.com)
parallel -j0 --delay 0.1 --retries $RETRIES \
rsync -a /usr/local/bin/{parallel,env_parallel,env_parallel.*[^~],parcat} \
::: $MASTER:bin/
doit() { doit() {
# Avoid the stupid /etc/issue.net banner at Polarhome: -oLogLevel=quiet # Avoid the stupid /etc/issue.net banner at Polarhome: -oLogLevel=quiet
@ -27,12 +27,14 @@ doit() {
export PARALLEL_SSH export PARALLEL_SSH
export MAXTIME export MAXTIME
export RETRIES export RETRIES
export MAXPROC
echo MAXTIME=$MAXTIME RETRIES=$RETRIES export RET_TIME_K="-k --retries $RETRIES --timeout $MAXTIME"
echo MAXTIME=$MAXTIME RETRIES=$RETRIES MAXPROC=$MAXPROC
echo '### Filter out working servers' echo '### Filter out working servers'
POLAR="`bin/parallel -j0 -k --retries $RETRIES --timeout $MAXTIME $PARALLEL_SSH {} echo {} ::: $P`" POLAR="` bin/parallel -j0 $RET_TIME_K $PARALLEL_SSH {} echo {} ::: $P`"
S_POLAR=`bin/parallel -j0 -k --retries $RETRIES --timeout $MAXTIME echo -S 1/{} ::: $POLAR` S_POLAR=`bin/parallel -j0 $RET_TIME_K echo -S 1/{} ::: $POLAR`
copy() { copy() {
# scp, but atomic (avoid half files if disconnected) # scp, but atomic (avoid half files if disconnected)
@ -43,17 +45,16 @@ doit() {
stdout ssh -oLogLevel=quiet $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'" stdout ssh -oLogLevel=quiet $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'"
} }
export -f copy export -f copy
par_nonall() { par_nonall() {
parallel -j150 -k --retries $RETRIES --timeout $MAXTIME --delay 0.1 --tag \ parallel -j$MAXPROC $RET_TIME_K --delay 0.1 --tag \
--nonall $S_POLAR --argsep ,:- \ --nonall $S_POLAR --argsep ,:- \
'source setupenv >&/dev/null || . `pwd`/setupenv;' "$@" 'source setupenv >&/dev/null || . `pwd`/setupenv;' "$@"
} }
export -f par_nonall export -f par_nonall
echo '### Copy commands to servers' echo '### Copy commands to servers'
parallel -vkj150 --retries $RETRIES --timeout $MAXTIME --delay 0.03 --tag \ env_parallel -vj$MAXPROC $RET_TIME_K --delay 0.03 --tag copy {2} {1} {1/} \
copy {2} {1} {1/} \
::: bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \ ::: bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \
::: $POLAR ::: $POLAR
echo Done copying echo Done copying
@ -130,7 +131,7 @@ doit() {
par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1 par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1
} }
env_parallel -u -Subuntu.polarhome.com doit ::: 1 env_parallel -u -S$MASTER doit ::: 1
# eval 'myfunc() { echo '$(perl -e 'print "x"x20000')'; }' # eval 'myfunc() { echo '$(perl -e 'print "x"x20000')'; }'
# env_parallel myfunc ::: a | wc # OK # env_parallel myfunc ::: a | wc # OK

3
testsuite/wanted-results/parallel-local-ssh5
View file

@ -12,6 +12,9 @@ par_lsh parallel: Warning: Could not figure out number of cpus on lo (). Using 1
par_lsh OK par_lsh OK
par_input_loss_pipe ### bug #36595: silent loss of input with --pipe and --sshlogin par_input_loss_pipe ### bug #36595: silent loss of input with --pipe and --sshlogin
par_input_loss_pipe 1 10000 48894 par_input_loss_pipe 1 10000 48894
par_env_parallel_onall bug #54352: env_parallel -Slo --nonall myfunc broken in 20180722
par_env_parallel_onall Myfunc works
par_env_parallel_onall Myfunc works
par_controlmaster_eats bug #36707: --controlmaster eats jobs par_controlmaster_eats bug #36707: --controlmaster eats jobs
par_controlmaster_eats OK1 par_controlmaster_eats OK1
par_controlmaster_eats OK2 par_controlmaster_eats OK2

54
testsuite/wanted-results/parallel-polarhome
View file

@ -1,4 +1,4 @@
MAXTIME=25 RETRIES=4 MAXTIME=25 RETRIES=4 MAXPROC=150
### Filter out working servers ### Filter out working servers
### Copy commands to servers ### Copy commands to servers
bin/parallel qnx copy qnx bin/parallel parallel bin/parallel qnx copy qnx bin/parallel parallel
@ -348,8 +348,9 @@ netbsd Works on netbsd.polarhome.com
openbsd Works on openbsd.polarhome.com openbsd Works on openbsd.polarhome.com
openindiana Works on openindiana openindiana Works on openindiana
pidora Works on pidora pidora Works on pidora
qnx Works on qnx
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian Works on raspbian raspbian Works on raspbian
redhat Works on redhat.polarhome.com redhat Works on redhat.polarhome.com
scosysv Works on scosysv.polarhome.com scosysv Works on scosysv.polarhome.com
@ -376,7 +377,7 @@ mandriva ### Test if empty command in process list causes problems
mandriva OK_with_empty_cmd mandriva OK_with_empty_cmd
miros ### Test if empty command in process list causes problems miros ### Test if empty command in process list causes problems
miros OK_with_empty_cmd miros OK_with_empty_cmd
netbsd CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset test_empty_cmd test_empty_cmd netbsd CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset test_empty_cmd test_empty_cmd test_empty_cmd test_empty_cmd
openbsd ### Test if empty command in process list causes problems openbsd ### Test if empty command in process list causes problems
openbsd OK_with_empty_cmd openbsd OK_with_empty_cmd
openindiana ### Test if empty command in process list causes problems openindiana ### Test if empty command in process list causes problems
@ -428,8 +429,9 @@ openindiana Error in tempfile() using /XXXXXXXX.arg: Could not create temp file
openindiana OK readonly tmp openindiana OK readonly tmp
pidora Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. pidora Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000.
pidora OK readonly tmp pidora OK readonly tmp
qnx Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000
qnx OK readonly tmp
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory
raspbian Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 raspbian Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000
raspbian OK readonly tmp raspbian OK readonly tmp
redhat Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 redhat Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000
@ -464,8 +466,9 @@ netbsd 1
openbsd 1 openbsd 1
openindiana 1 openindiana 1
pidora 1 pidora 1
qnx 1
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian 1 raspbian 1
redhat 1 redhat 1
scosysv 1 scosysv 1
@ -488,8 +491,9 @@ netbsd 1
openbsd 1 openbsd 1
openindiana 1 openindiana 1
pidora 1 pidora 1
qnx 1
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian 1 raspbian 1
redhat 2 redhat 2
scosysv 1 scosysv 1
@ -512,8 +516,9 @@ netbsd 1
openbsd 1 openbsd 1
openindiana 1 openindiana 1
pidora 1 pidora 1
qnx 1
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian 1 raspbian 1
redhat 2 redhat 2
scosysv 1 scosysv 1
@ -536,8 +541,9 @@ netbsd 1
openbsd 1 openbsd 1
openindiana 1 openindiana 1
pidora 1 pidora 1
qnx 1
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian 1 raspbian 1
redhat 2 redhat 2
scosysv 1 scosysv 1
@ -586,6 +592,7 @@ netbsd bash only B
openbsd bash only B openbsd bash only B
openindiana bash only B openindiana bash only B
pidora bash only B pidora bash only B
qnx bash only B
raspbian bash only B raspbian bash only B
redhat bash only B redhat bash only B
scosysv bash only B scosysv bash only B
@ -731,8 +738,18 @@ pidora /home/t/tange/.profile
pidora /home/t/tange/.cshrc pidora /home/t/tange/.cshrc
pidora /home/t/tange/.tcshrc pidora /home/t/tange/.tcshrc
pidora install-OK pidora install-OK
qnx Installed env_parallel in:
qnx /home/t/tange/.bashrc
qnx /home/t/tange/.shrc
qnx /home/t/tange/.zshenv
qnx /home/t/tange/.config/fish/config.fish
qnx /home/t/tange/.kshrc
qnx /home/t/tange/.mkshrc
qnx /home/t/tange/.profile
qnx /home/t/tange/.cshrc
qnx /home/t/tange/.tcshrc
qnx install-OK
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory
raspbian Installed env_parallel in: raspbian Installed env_parallel in:
raspbian /home/t/tange/.bashrc raspbian /home/t/tange/.bashrc
raspbian /home/t/tange/.shrc raspbian /home/t/tange/.shrc
@ -825,8 +842,9 @@ netbsd env_parallel run-OK
openbsd env_parallel run-OK openbsd env_parallel run-OK
openindiana env_parallel run-OK openindiana env_parallel run-OK
pidora env_parallel run-OK pidora env_parallel run-OK
qnx env_parallel run-OK
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian env_parallel run-OK raspbian env_parallel run-OK
redhat env_parallel run-OK redhat env_parallel run-OK
scosysv env_parallel run-OK scosysv env_parallel run-OK
@ -872,8 +890,10 @@ netbsd arr: Undefined variable.
openbsd openbsd
openindiana 1 2 1 2 3 1 2 3 4 openindiana 1 2 1 2 3 1 2 3 4
pidora 1 2 1 2 3 1 2 3 4 pidora 1 2 1 2 3 1 2 3 4
qnx
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
qnx /bin/sh: syntax error: `(' unexpected
raspbian 1 2 1 2 3 1 2 3 4 raspbian 1 2 1 2 3 1 2 3 4
redhat 1 2 1 2 3 1 2 3 4 redhat 1 2 1 2 3 1 2 3 4
scosysv 1 2 1 2 3 1 2 3 4 scosysv 1 2 1 2 3 1 2 3 4
@ -899,8 +919,10 @@ netbsd arr: Undefined variable.
openbsd openbsd
openindiana 2 2 3 2 3 4 openindiana 2 2 3 2 3 4
pidora 2 2 3 2 3 4 pidora 2 2 3 2 3 4
qnx
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
qnx /bin/sh: syntax error: `(' unexpected
raspbian 2 2 3 2 3 4 raspbian 2 2 3 2 3 4
redhat 2 2 3 2 3 4 redhat 2 2 3 2 3 4
scosysv 2 2 3 2 3 4 scosysv 2 2 3 2 3 4
@ -928,8 +950,10 @@ netbsd var1: Undefined variable.
openbsd ,, openbsd ,,
openindiana 1 2,1 2 3,1 2 3 4 openindiana 1 2,1 2 3,1 2 3 4
pidora 1 2,1 2 3,1 2 3 4 pidora 1 2,1 2 3,1 2 3 4
qnx ,,
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian 1 2,1 2 3,1 2 3 4 raspbian 1 2,1 2 3,1 2 3 4
redhat 1 2,1 2 3,1 2 3 4 redhat 1 2,1 2 3,1 2 3 4
scosysv 1 2,1 2 3,1 2 3 4 scosysv 1 2,1 2 3,1 2 3 4
@ -955,8 +979,10 @@ netbsd var1: Undefined variable.
openbsd ,, openbsd ,,
openindiana 2,2 3,2 3 4 openindiana 2,2 3,2 3 4
pidora 2,2 3,2 3 4 pidora 2,2 3,2 3 4
qnx ,,
qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: >&/dev/null : illegal file descriptor name
qnx /bin/sh: .: /home/t/tange/setupenv: No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
qnx parallel: Warning: Cannot figure out number of cpus. Using 1.
raspbian 2,2 3,2 3 4 raspbian 2,2 3,2 3 4
redhat 2,2 3,2 3 4 redhat 2,2 3,2 3 4
scosysv 2,2 3,2 3 4 scosysv 2,2 3,2 3 4