From b14ca075634294439bc7f5ed508091fdcca0312a Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 30 Nov 2015 00:47:21 +0100 Subject: [PATCH] Fixed bug #46520: --basefile cleans up without --cleanup. qqx() is qx{} with stderr ignored. {= $_ = total_jobs() =} can now be used. --- NEWS | 2 - doc/release_new_version | 35 +----- src/parallel | 115 ++++++++++-------- src/parallel.pod | 49 +++++--- testsuite/tests-to-run/parallel-freebsd.sh | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 8 +- testsuite/tests-to-run/parallel-local-ssh6.sh | 71 +++++++++++ testsuite/wanted-results/parallel-freebsd | 2 +- testsuite/wanted-results/parallel-local-0.3s | 8 +- testsuite/wanted-results/parallel-local12 | 4 +- testsuite/wanted-results/parallel-local9 | 2 +- testsuite/wanted-results/parallel-tutorial | 4 +- 12 files changed, 188 insertions(+), 114 deletions(-) create mode 100644 testsuite/tests-to-run/parallel-local-ssh6.sh diff --git a/NEWS b/NEWS index 54b72e9b..6440c5b0 100644 --- a/NEWS +++ b/NEWS @@ -12,8 +12,6 @@ * GNU Parallel was cited in: A Detailed Characterization of Errors in Logic Circuits due to Single-Event Transients http://ieeexplore.ieee.org/xpl/articleDetails.jsp?tp=&arnumber=7302348&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7302348 -* GNU Parallel was cited in: Exploiting Multiple Levels of Parallelism in Sparse Matrix-Matrix Multiplication http://arxiv.org/pdf/1510.00844.pdf - * GNU Parallel was cited in: Arabic Speaker Emotion Classification Using Rhythm Metrics and Neural Networks http://www.eurasip.org/Proceedings/Eusipco/Eusipco2015/papers/1570104855.pdf * GNU Parallel was cited in: Stride Search: a general algorithm for storm detection in high resolution climate data http://www.geosci-model-dev-discuss.net/8/7727/2015/gmdd-8-7727-2015.pdf diff --git a/doc/release_new_version b/doc/release_new_version index 2386aa3e..11de86f0 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -4,7 +4,7 @@ Check that documentation is updated (compare to web): -Fixet for 20151022 +Fixet for 20151122 git diff last-release-commit Unmodified beta since last version => production Unmodified alpha since last version => beta @@ -212,9 +212,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20151122 ('Kronan/Sharm el-Sheik/Bataclan') released <<[stable]>> +Subject: GNU Parallel 20151222 ('') released <<[stable]>> -GNU Parallel 20151122 ('Kronan/Sharm el-Sheik/Bataclan') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20151222 ('') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> @@ -236,34 +236,9 @@ http://www.researchgate.net/profile/Christoph_Junghans/publication/276178326_TAD * << Update forventet juni Rachel har lige svaret >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/ -* GNU Parallel packaged for CERN CentOS: http://linuxsoft.cern.ch/cern/centos/7/cern/x86_64/repoview/parallel.html - -* GNU Parallel was cited in: The Outer Solar System Origins Survey: I. Design and First-Quarter Discoveries http://arxiv.org/pdf/1511.02895.pdf - -* GNU Parallel was cited in: Contrasting genetic architectures of schizophrenia and other complex diseases using fast variance-components analysis http://www.nature.com/ng/journal/vaop/ncurrent/full/ng.3431.html - -* GNU Parallel was cited in: Named-Entity Chunking for Norwegian Text using Support Vector Machines http://ojs.bibsys.no/index.php/NIK/article/viewFile/248/211 - -* GNU Parallel was cited in: Multiscale Estimation of Binding Kinetics Using Brownian Dynamics, Molecular Dynamics and Milestoning http://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1004381#pcbi.1004381.ref072 - -* GNU Parallel was cited in: A Detailed Characterization of Errors in Logic Circuits due to Single-Event Transients http://ieeexplore.ieee.org/xpl/articleDetails.jsp?tp=&arnumber=7302348&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7302348 - -* GNU Parallel was cited in: Exploiting Multiple Levels of Parallelism in Sparse Matrix-Matrix Multiplication http://arxiv.org/pdf/1510.00844.pdf - -* GNU Parallel was cited in: Arabic Speaker Emotion Classification Using Rhythm Metrics and Neural Networks http://www.eurasip.org/Proceedings/Eusipco/Eusipco2015/papers/1570104855.pdf - -* GNU Parallel was cited in: Stride Search: a general algorithm for storm detection in high resolution climate data http://www.geosci-model-dev-discuss.net/8/7727/2015/gmdd-8-7727-2015.pdf - -* GNU Parallel was cited in: Decomposing Digital Paintings into Layers via RGB-space Geometry http://arxiv.org/pdf/1509.03335.pdf - -* GNU Parallel was cited in: Structure and evolutionary history of a large family of NLR proteins in the zebrafish http://www.biorxiv.org/content/biorxiv/early/2015/09/18/027151.full.pdf - -* Automating large numbers of tasks https://rcc.uchicago.edu/docs/tutorials/kicp-tutorials/running-jobs.html - -* Max out your IOPs with GNU Parallel http://blog.bitratchet.com/2015/11/11/max-out-your-iops-with-gnu-parallel/ - -* GNU Parallel was cited in: Evolution of movement strategies under competitive interactions http://digital.csic.es/bitstream/10261/115973/1/evolution_movement_strategies_Kiziridis.pdf +* GNU Parallel was cited in: Achieving Consistent Doppler Measurements from SDO/HMI Vector Field Inversions http://arxiv.org/pdf/1511.06500.pdf +* 使用 GNU parallel 來平行運算http://mutolisp.logdown.com/posts/316959-using-gnu-parallel-to-parallel-computing * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index 89f06df2..14fb7d28 100755 --- a/src/parallel +++ b/src/parallel @@ -1118,7 +1118,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20151122; + $Global::version = 20151123; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1150,7 +1150,7 @@ sub init_globals { '{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::', '{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::', # {##} = number of jobs - '{##}' => '$_=$Global::JobQueue->total_jobs()', + '{##}' => '$_=total_jobs()', ); # Modifiable copy of %Global::replace %Global::rpl = %Global::replace; @@ -1682,7 +1682,7 @@ sub cleanup { # Returns: N/A unlink keys %Global::unlink; map { rmdir $_ } keys %Global::unlink; - if(@opt::basefile) { cleanup_basefile(); } + if(@opt::basefile and $opt::cleanup) { cleanup_basefile(); } } sub __QUOTING_ARGUMENTS_FOR_SHELL__ {} @@ -2508,7 +2508,7 @@ sub progress { $last_column_time = time; $columns = $ENV{'COLUMNS'}; if(not $columns) { - my $stty = qx{ stty -a /dev/null' }; + my $resize = ::qqx("resize"); $resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; }; } $columns ||= 80; @@ -3116,7 +3116,7 @@ sub onall { close $parallel_fh; $Global::exitstatus = $? >> 8; debug("init", "--onall exitvalue ", $?); - if(@opt::basefile) { cleanup_basefile(); } + if(@opt::basefile and $opt::cleanup) { cleanup_basefile(); } $Global::debug or unlink(@argfiles); my %seen; for my $joblog (@joblogs) { @@ -3150,11 +3150,11 @@ sub save_original_signal_handler { # %Global::original_sig # Returns: N/A $SIG{INT} = sub { - if($opt::tmux) { qx { tmux kill-session -t p$$ }; } + if($opt::tmux) { ::qqx("tmux kill-session -t p$$"); } wait_and_exit(255); }; $SIG{TERM} = sub { - if($opt::tmux) { qx { tmux kill-session -t p$$ }; } + if($opt::tmux) { ::qqx("tmux kill-session -t p$$"); } wait_and_exit(255); }; %Global::original_sig = %SIG; @@ -3399,7 +3399,7 @@ sub usage { " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", " ;login: The USENIX Magazine, February 2011:42-47.", "", - "This helps funding further development; and it won't cost you a cent.", + "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.\n", ""); } @@ -3427,7 +3427,7 @@ sub citation_notice { " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,\n", " ;login: The USENIX Magazine, February 2011:42-47.\n", "\n", - "This helps funding further development; and it won't cost you a cent.\n", + "This helps funding further development; AND IT WON'T COST YOU A CENT.\n", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.\n", "\n", "To silence the citation notice: run 'parallel --bibtex'.\n\n", @@ -3507,7 +3507,7 @@ sub bibtex { "", "(Feel free to use \\nocite{Tange2011a})", "", - "This helps funding further development; and it won't cost you a cent.", + "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "", "If you send a copy of your published article to tange\@gnu.org, it will be", @@ -3600,6 +3600,19 @@ sub tmpfifo { return $tmpfifo; } +sub qqx { + # Like qx but with clean environment (except for $PATH) + # and STDERR ignored + # This is needed if the environment contains functions + # that /bin/sh does not understand + my $devnull = $Global::debug ? "" : "exec 2>/dev/null;"; + return qx{ $devnull @_ }; + my $PATH = $ENV{'PATH'}; + local(%ENV); + $ENV{'PATH'} = $PATH; + return qx{ ( @_ ) 2>/dev/null }; +} + sub uniq { # Remove duplicates and return unique values return keys %{{ map { $_ => 1 } @_ }}; @@ -4247,8 +4260,8 @@ sub memfree_recompute { my $script = memfreescript(); # TODO add sshlogin and backgrounding - # Run it twice if it gives 0 - $self->{'memfree'} = qx{ $script } || qx{ $script }; + # Run the script twice if it gives 0 (typically intermittent error) + $self->{'memfree'} = ::qqx($script) || ::qqx($script); if(not $self->{'memfree'}) { ::die_bug("Less than 1 byte free"); } @@ -4366,7 +4379,7 @@ sub swap_activity { my $file = $self->{'swap_activity_file'}; my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".swp"); ::debug("swap", "\n", $swap_activity, "\n"); - qx{ ($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile) & }; + ::qqx("($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile) &"); } return $self->{'swap_activity'}; } @@ -4664,7 +4677,7 @@ sub loadavg { ::debug("load", "Cmd: ", $cmd); my $file = $self->{'loadavg_file'}; my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa"); - qx{ ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & }; + ::qqx(" ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & "); } return $self->{'loadavg'}; } @@ -5096,10 +5109,10 @@ sub ncpus { my $ncpu; my $sqe = ::shell_quote_scalar($Global::envvar); if($opt::use_cpus_instead_of_cores) { - $ncpu = qx(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cpus); + $ncpu = ::qqx("echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cpus"); } else { ::debug("init",qq(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores\n)); - $ncpu = qx(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores); + $ncpu = ::qqx("echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores"); } chomp $ncpu; if($ncpu =~ /^\s*[0-9]+\s*$/s) { @@ -5232,7 +5245,7 @@ sub no_of_cores { sub nproc { # Returns: # Number of cores using `nproc` - my $no_of_cores = qx{ sh -c 'nproc 2>/dev/null' }; + my $no_of_cores = ::qqx("nproc"); return $no_of_cores; } @@ -5308,9 +5321,9 @@ sub no_of_cpus_freebsd { # Number of physical CPUs on FreeBSD # undef if not FreeBSD my $no_of_cpus = - (qx{ sh -c 'sysctl -a dev.cpu 2>/dev/null' | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }' } + (::qqx(qq{ sysctl -a dev.cpu | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }' }) or - qx{ sh -c 'sysctl hw.ncpu 2>/dev/null' | awk '{ print \$2 }' }); + ::qqx(qq{ sysctl hw.ncpu | awk '{ print \$2 }' })); chomp $no_of_cpus; return $no_of_cpus; } @@ -5320,9 +5333,9 @@ sub no_of_cores_freebsd { # Number of CPU cores on FreeBSD # undef if not FreeBSD my $no_of_cores = - (qx{ sh -c 'sysctl hw.ncpu 2>/dev/null' | awk '{ print \$2 }' } + (::qqx(qq{ sysctl hw.ncpu | awk '{ print \$2 }' }) or - qx{ sh -c 'sysctl -a hw 2>/dev/null' | grep [^a-z]logicalcpu[^a-z] | awk '{ print \$2 }' }); + ::qqx(qq{ sysctl -a dev.cpu | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }' })); chomp $no_of_cores; return $no_of_cores; } @@ -5331,7 +5344,7 @@ sub no_of_cpus_netbsd { # Returns: # Number of physical CPUs on NetBSD # undef if not NetBSD - my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; + my $no_of_cpus = ::qqx("sysctl -n hw.ncpu"); chomp $no_of_cpus; return $no_of_cpus; } @@ -5340,7 +5353,7 @@ sub no_of_cores_netbsd { # Returns: # Number of CPU cores on NetBSD # undef if not NetBSD - my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; + my $no_of_cores = ::qqx("sysctl -n hw.ncpu"); chomp $no_of_cores; return $no_of_cores; } @@ -5349,7 +5362,7 @@ sub no_of_cpus_openbsd { # Returns: # Number of physical CPUs on OpenBSD # undef if not OpenBSD - my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; + my $no_of_cpus = ::qqx('sysctl -n hw.ncpu'); chomp $no_of_cpus; return $no_of_cpus; } @@ -5358,7 +5371,7 @@ sub no_of_cores_openbsd { # Returns: # Number of CPU cores on OpenBSD # undef if not OpenBSD - my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; + my $no_of_cores = ::qqx('sysctl -n hw.ncpu'); chomp $no_of_cores; return $no_of_cores; } @@ -5367,7 +5380,7 @@ sub no_of_cpus_hurd { # Returns: # Number of physical CPUs on HURD # undef if not HURD - my $no_of_cpus = qx{ nproc }; + my $no_of_cpus = ::qqx("nproc"); chomp $no_of_cpus; return $no_of_cpus; } @@ -5376,19 +5389,19 @@ sub no_of_cores_hurd { # Returns: # Number of physical CPUs on HURD # undef if not HURD - my $no_of_cores = `nproc`; + my $no_of_cores = ::qqx("nproc"); chomp $no_of_cores; return $no_of_cores; } sub no_of_cpus_darwin { # Returns: - # Number of physical CPUs on Mac Darwin - # undef if not Mac Darwin + # Number of physical CPUs on MacOSX Darwin + # undef if not MacOSX Darwin my $no_of_cpus = - (qx{ sh -c 'sysctl -n hw.physicalcpu 2>/dev/null' } + (::qqx('sysctl -n hw.physicalcpu') or - qx{ sh -c 'sysctl -a hw 2>/dev/null' | grep [^a-z]physicalcpu[^a-z] | awk '{ print \$2 }' }); + ::qqx(qq{ sysctl -a hw | grep [^a-z]physicalcpu[^a-z] | awk '{ print \$2 }' })); return $no_of_cpus; } @@ -5397,9 +5410,9 @@ sub no_of_cores_darwin { # Number of CPU cores on Mac Darwin # undef if not Mac Darwin my $no_of_cores = - (qx{ sh -c 'sysctl -n hw.logicalcpu 2>/dev/null' } + (::qqx('sysctl -n hw.logicalcpu') or - qx{ sh -c 'sysctl -a hw 2>/dev/null' | grep [^a-z]logicalcpu[^a-z] | awk '{ print \$2 }' }); + ::qqx(qq{ sysctl -a hw | grep [^a-z]logicalcpu[^a-z] | awk '{ print \$2 }' })); return $no_of_cores; } @@ -5408,13 +5421,13 @@ sub no_of_cpus_solaris { # Number of physical CPUs on Solaris # undef if not Solaris if(-x "/usr/sbin/psrinfo") { - my @psrinfo = `/usr/sbin/psrinfo`; + my @psrinfo = ::qqx("/usr/sbin/psrinfo"); if($#psrinfo >= 0) { return $#psrinfo +1; } } if(-x "/usr/sbin/prtconf") { - my @prtconf = qx{ /usr/sbin/prtconf | grep cpu..instance }; + my @prtconf = ::qqx("/usr/sbin/prtconf | grep cpu..instance"); if($#prtconf >= 0) { return $#prtconf +1; } @@ -5427,13 +5440,13 @@ sub no_of_cores_solaris { # Number of CPU cores on Solaris # undef if not Solaris if(-x "/usr/sbin/psrinfo") { - my @psrinfo = `/usr/sbin/psrinfo`; + my @psrinfo = ::qqx("/usr/sbin/psrinfo"); if($#psrinfo >= 0) { return $#psrinfo +1; } } if(-x "/usr/sbin/prtconf") { - my @prtconf = qx{ /usr/sbin/prtconf | grep cpu..instance }; + my @prtconf = ::qqx("/usr/sbin/prtconf | grep cpu..instance"); if($#prtconf >= 0) { return $#prtconf +1; } @@ -5476,7 +5489,7 @@ sub no_of_cpus_hpux { # Number of physical CPUs on HP-UX # undef if not HP-UX my $no_of_cpus = - qx{ sh -c '/usr/bin/mpsched -s 2>&1' | grep 'Locality Domain Count' | awk '{ print \$4 }'}; + ::qqx(qq{ /usr/bin/mpsched -s 2>&1 | grep 'Locality Domain Count' | awk '{ print \$4 }'}); return $no_of_cpus; } @@ -5485,7 +5498,7 @@ sub no_of_cores_hpux { # Number of CPU cores on HP-UX # undef if not HP-UX my $no_of_cores = - qx{ sh -c '/usr/bin/mpsched -s 2>&1' | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'}; + ::qqx(qq{ /usr/bin/mpsched -s 2>&1 | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'}); return $no_of_cores; } @@ -5513,7 +5526,7 @@ sub no_of_cpus_openserver { # undef if not SCO OpenServer my $no_of_cpus = 0; if(-x "/usr/sbin/psrinfo") { - my @psrinfo = `/usr/sbin/psrinfo`; + my @psrinfo = ::qqx("/usr/sbin/psrinfo"); if($#psrinfo >= 0) { return $#psrinfo +1; } @@ -5527,7 +5540,7 @@ sub no_of_cores_openserver { # undef if not SCO OpenServer my $no_of_cores = 0; if(-x "/usr/sbin/psrinfo") { - my @psrinfo = `/usr/sbin/psrinfo`; + my @psrinfo = ::qqx("/usr/sbin/psrinfo"); if($#psrinfo >= 0) { return $#psrinfo +1; } @@ -5539,7 +5552,7 @@ sub no_of_cpus_irix { # Returns: # Number of physical CPUs on IRIX # undef if not IRIX - my $no_of_cpus = `hinv | grep HZ | grep Processor | awk '{print \$1}'`; + my $no_of_cpus = ::qqx(qq{ hinv | grep HZ | grep Processor | awk '{print \$1}' }); return $no_of_cpus; } @@ -5547,7 +5560,7 @@ sub no_of_cores_irix { # Returns: # Number of CPU cores on IRIX # undef if not IRIX - my $no_of_cores = `hinv | grep HZ | grep Processor | awk '{print \$1}'`; + my $no_of_cores = ::qqx(qq{ hinv | grep HZ | grep Processor | awk '{print \$1}' }); return $no_of_cores; } @@ -5555,7 +5568,7 @@ sub no_of_cpus_tru64 { # Returns: # Number of physical CPUs on Tru64 # undef if not Tru64 - my $no_of_cpus = `sizer -pr`; + my $no_of_cpus = ::qqx("sizer -pr"); return $no_of_cpus; } @@ -5563,7 +5576,7 @@ sub no_of_cores_tru64 { # Returns: # Number of CPU cores on Tru64 # undef if not Tru64 - my $no_of_cores = `sizer -pr`; + my $no_of_cores = ::qqx("sizer -pr"); return $no_of_cores; } @@ -8777,9 +8790,9 @@ sub tmux_length { my @out; for my $l (1, 2020, 16320, 100000, $len) { my $tmpfile = ::tmpname("tms"); - my $tmuxcmd = "sh -c '".$ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l". - ("x"x$l). " 2>/dev/null' && echo $l; rm -f $tmpfile"; - push @out, qx{ $tmuxcmd }; + my $tmuxcmd = $ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l". + ("x"x$l). " && echo $l; rm -f $tmpfile"; + push @out, ::qqx($tmuxcmd); unlink $tmpfile; } ::debug("tmux","tmux-length ",@out); @@ -9191,6 +9204,10 @@ sub Q { return $a; } +sub total_jobs { + return $Global::JobQueue->total_jobs() +} + { my %perleval; diff --git a/src/parallel.pod b/src/parallel.pod index 01b89a18..be6b43b1 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -426,7 +426,7 @@ string that is not in the command line. See also: B<:::>. -=item B<--bar> (beta testing) +=item B<--bar> Show progress as a progress bar. In the bar is shown: % of jobs completed, estimated seconds left, and number of jobs started. @@ -513,7 +513,7 @@ I defaults to 1M. See B<--pipe> and B<--pipepart> for use of this. -=item B<--cat> (beta testing) +=item B<--cat> Create a temporary file with content. Normally B<--pipe>/B<--pipepart> will give data to the program on stdin (standard input). With B<--cat> @@ -613,7 +613,7 @@ I seconds after starting each job. I can be less than 1 second. -=item B<--dry-run> (beta testing) +=item B<--dry-run> Print the job to run on stdout (standard output), but do not run the job. Use B<-v -v> to include the wrapping that GNU Parallel generates @@ -672,7 +672,7 @@ and functions) use env_parallel as described under the option I. See also: B<--record-env>. -=item B<--eta> (beta testing) +=item B<--eta> Show the estimated number of seconds before finishing. This forces GNU B to read all jobs before starting to find the number of @@ -693,7 +693,7 @@ Implies B<--semaphore>. See also B<--bg>, B. -=item B<--fifo> (beta testing) +=item B<--fifo> Create a temporary fifo with content. Normally B<--pipe> and B<--pipepart> will give data to the program on stdin (standard @@ -725,7 +725,7 @@ over B<--tollef>. The B<--tollef> option is now retired, and therefore may not be used. B<--gnu> is kept for compatibility. -=item B<--group> (beta testing) +=item B<--group> Group output. Output from each jobs is grouped together and is only printed when the command is finished. stderr (standard error) first @@ -747,9 +747,9 @@ See also: B<--line-buffer> B<--ungroup> Print a summary of the options to GNU B and exit. -=item B<--halt-on-error> I (beta testing) +=item B<--halt-on-error> I -=item B<--halt> I (beta testing) +=item B<--halt> I When should GNU B terminate? In some situations it makes no sense to run all jobs. GNU B should simply give up as soon @@ -1172,7 +1172,7 @@ performance is important use B<--pipepart>. See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>. -=item B<--pipepart> (beta testing) +=item B<--pipepart> Pipe parts of a physical file. B<--pipepart> works similar to B<--pipe>, but is much faster. It has a few limitations: @@ -1198,7 +1198,7 @@ control on the command line (used by GNU B internally when called with B<--sshlogin>). -=item B<--plus> (beta testing) +=item B<--plus> Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} {...} {/..} {/...} {##}. The idea being that '{+foo}' matches the opposite of @@ -1288,7 +1288,7 @@ Overrides an earlier B<--keep-order> (e.g. if set in B<~/.parallel/config>). -=item B<--nice> I (beta testing) +=item B<--nice> I Run the command at this niceness. For simple commands you can just add B in front of the command. But if the command consists of more @@ -1576,6 +1576,19 @@ it possible to define your own replacement strings. GNU B's --rpl '{/.} s:.*/::; s:\.[^/.]+$::;' --rpl '{.} s:\.[^/.]+$::' +The <--plus> replacement strings are implemented as: + + --rpl '{+/} s:/[^/]*$::' + --rpl '{+.} s:.*\.::' + --rpl '{+..} s:.*\.([^.]*\.):$1:' + --rpl '{+...} s:.*\.([^.]*\.[^.]*\.):$1:' + --rpl '{..} s:\.[^/.]+$::; s:\.[^/.]+$::' + --rpl '{...} s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::' + --rpl '{/..} s:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::' + --rpl '{/...} s:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::' + --rpl '{##} $_=total_jobs()' + + If the user defined replacement string starts with '{' it can also be used as a positional replacement string (like B<{2.}>). @@ -1737,7 +1750,7 @@ Does not run the command but quotes it. Useful for making quoted composed commands for GNU B. -=item B<--shuf> (beta testing) +=item B<--shuf> Shuffle jobs. When having multiple input sources it is hard to randomize jobs. --shuf will generate all jobs, and shuffle them before @@ -1765,13 +1778,13 @@ I seconds after starting each ssh. I can be less than 1 seconds. -=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> (beta testing) +=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> -=item B<-S> I<@hostgroup> (beta testing) +=item B<-S> I<@hostgroup> -=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> (beta testing) +=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> -=item B<--sshlogin> I<@hostgroup> (beta testing) +=item B<--sshlogin> I<@hostgroup> Distribute jobs to remote computers. The jobs will be run on a list of remote computers. @@ -2088,9 +2101,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely. Print the version GNU B and exit. -=item B<--workdir> I (beta testing) +=item B<--workdir> I -=item B<--wd> I (beta testing) +=item B<--wd> I Files transferred using B<--transfer> and B<--return> will be relative to I on remote computers, and the command will be executed in diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index 6aea1bd4..2fd7fe69 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -36,7 +36,7 @@ echo 'bug #40134: FreeBSD: --shebang not working' echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap; - chmod 755 ./shebang-wrap; ./shebang-wrap wrap works + chmod 755 ./shebang-wrap; ./shebang-wrap wrap works | sort -r echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/bin/perl -w :::'; echo 'print @ARGV,"\n";') > shebang-wrap-opt; diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index bd202c66..eb1b37e1 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -249,11 +249,11 @@ echo '**' echo '{##} bug #45841: Replacement string for total no of jobs' - parallel --plus echo {##} ::: {a..j}; + parallel -k --plus echo {##} ::: {a..j}; parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10} - parallel -N7 --plus echo {#} {##} ::: {1..14} - parallel -N7 --plus echo {#} {##} ::: {1..15} - parallel -X --plus echo {#} {##} ::: {1..15} + parallel -k -N7 --plus echo {#} {##} ::: {1..14} + parallel -k -N7 --plus echo {#} {##} ::: {1..15} + parallel -k -X --plus echo {#} {##} ::: {1..15} echo '**' diff --git a/testsuite/tests-to-run/parallel-local-ssh6.sh b/testsuite/tests-to-run/parallel-local-ssh6.sh new file mode 100644 index 00000000..e4d084e1 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-ssh6.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +SSHLOGIN1=parallel@lo +SSHLOGIN2=csh@lo +mkdir -p tmp + +# -L1 will join lines ending in ' ' +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj5 -k --joblog /tmp/jl-`basename $0` -L1 +echo '### Test --onall'; + parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 + +echo '### Test | --onall'; + seq 3 | parallel --onall --tag -k -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - + +echo '### Test --onall -u'; + parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort + +echo '### Test --nonall'; + parallel --nonall -k -S $SSHLOGIN1,$SSHLOGIN2 pwd | sort + +echo '### Test --nonall -u - should be interleaved x y x y'; + parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 -u 'pwd|grep -q csh && sleep 3; pwd;sleep 12;pwd;' + +echo '### Test read sshloginfile from STDIN'; + echo $SSHLOGIN1,$SSHLOGIN2 | parallel -S - -k --nonall pwd; + echo $SSHLOGIN1,$SSHLOGIN2 | parallel --sshloginfile - -k --onall pwd\; echo ::: foo + +echo '**' + +echo '### Test --nonall --basefile'; + touch tmp/nonall--basefile; + stdout parallel --nonall --basefile tmp/nonall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile; + stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile; + stdout rm tmp/nonall--basefile + +echo '**' + +echo '### Test --onall --basefile'; + touch tmp/onall--basefile; + stdout parallel --onall --basefile tmp/onall--basefile -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile; + stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile; + stdout rm tmp/onall--basefile + +echo '**' + +echo '### Test --nonall --basefile --cleanup (rm should fail)'; + touch tmp/nonall--basefile--clean; + stdout parallel --nonall --basefile tmp/nonall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls tmp/nonall--basefile--clean; + stdout parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 rm tmp/nonall--basefile--clean; + stdout rm tmp/nonall--basefile--clean + +echo '**' + +echo '### Test --onall --basefile --cleanup (rm should fail)'; + touch tmp/onall--basefile--clean; + stdout parallel --onall --basefile tmp/onall--basefile--clean --cleanup -S $SSHLOGIN1,$SSHLOGIN2 ls {} ::: tmp/onall--basefile--clean; + stdout parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 rm {} ::: tmp/onall--basefile--clean; + stdout rm tmp/onall--basefile--clean + +echo '**' + +echo '### Test --workdir .'; + ssh $SSHLOGIN1 mkdir -p mydir; + mkdir -p $HOME/mydir; cd $HOME/mydir; + parallel --workdir . -S $SSHLOGIN1 ::: pwd + +echo '### Test --wd .'; + ssh $SSHLOGIN2 mkdir -p mydir; + mkdir -p $HOME/mydir; cd $HOME/mydir; + parallel --workdir . -S $SSHLOGIN2 ::: pwd +EOF diff --git a/testsuite/wanted-results/parallel-freebsd b/testsuite/wanted-results/parallel-freebsd index 4641378d..5f3e75d3 100644 --- a/testsuite/wanted-results/parallel-freebsd +++ b/testsuite/wanted-results/parallel-freebsd @@ -44,7 +44,7 @@ It worked echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' bug #40134: FreeBSD: --shebang(-wrap) not working - (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap; chmod 755 ./shebang-wrap; ./shebang-wrap wrap works + (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap; chmod 755 ./shebang-wrap; ./shebang-wrap wrap works | sort -r wrap works echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 1053c8b3..f206a4ac 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -521,7 +521,7 @@ echo '**' ** echo '{##} bug #45841: Replacement string for total no of jobs' {##} bug #45841: Replacement string for total no of jobs - parallel --plus echo {##} ::: {a..j}; parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10} + parallel -k --plus echo {##} ::: {a..j}; parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10} 10 10 10 @@ -542,14 +542,14 @@ echo '{##} bug #45841: Replacement string for total no of jobs' 10 10 10 - parallel -N7 --plus echo {#} {##} ::: {1..14} + parallel -k -N7 --plus echo {#} {##} ::: {1..14} 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 - parallel -N7 --plus echo {#} {##} ::: {1..15} + parallel -k -N7 --plus echo {#} {##} ::: {1..15} 1 3 3 3 3 3 3 3 2 3 3 3 3 3 3 3 3 3 - parallel -X --plus echo {#} {##} ::: {1..15} + parallel -k -X --plus echo {#} {##} ::: {1..15} 1 15 15 2 15 15 3 15 15 diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 001d6570..d8820f91 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -6,7 +6,7 @@ please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. -This helps funding further development; and it won't cost you a cent. +This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence the citation notice: run 'parallel --bibtex'. @@ -27,7 +27,7 @@ please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. -This helps funding further development; and it won't cost you a cent. +This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence the citation notice: run 'parallel --bibtex'. diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index c6dc6804..24d79973 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -176,7 +176,7 @@ please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. -This helps funding further development; and it won't cost you a cent. +This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index faa2dd8a..9eecb504 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -980,7 +980,7 @@ please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. -This helps funding further development; and it won't cost you a cent. +This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. parallel --version @@ -1019,7 +1019,7 @@ please cite: (Feel free to use \nocite{Tange2011a}) -This helps funding further development; and it won't cost you a cent. +This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. If you send a copy of your published article to tange@gnu.org, it will be