diff --git a/doc/release_new_version b/doc/release_new_version index d9112efd..b23995fc 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -258,6 +258,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * GNU Parallel was cited in: An Operational Radiometric Landsat Preprocessing Framework for Large-Area Time Series Applications https://www.uni-trier.de/fileadmin/fb6/prof/FER/Publikationen/frantz_et_al_ieee-tgrs-2016-post-print.pdf +* A basic demo of how GNU Parallel can speed up execution of commands https://www.youtube.com/watch?v=kl8LO2jcvMc + * Downloading a list of URLs http://blog.gypsydave5.com/2016/02/04/xargs-and-curl/ * qbatch uses GNU Parallel: https://pypi.python.org/pypi/qbatch/1.0rc2 diff --git a/src/env_parallel.csh b/src/env_parallel.csh new file mode 100755 index 00000000..fe912092 --- /dev/null +++ b/src/env_parallel.csh @@ -0,0 +1,28 @@ +#!/bin/csh + +# This file must be sourced in csh: +# +# source `which env_parallel.csh` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2016 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV' diff --git a/src/parallel b/src/parallel index 8b892641..0dd076e5 100755 --- a/src/parallel +++ b/src/parallel @@ -3563,7 +3563,8 @@ sub usage { "", "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", - ""); + "", + "",); } @@ -3615,13 +3616,13 @@ sub status_no_nl { sub warning { my @w = @_; my $prog = $Global::progname || "parallel"; - status(map { ($prog, ": Warning: ", $_, "\n"); } @w); + status_no_nl(map { ($prog, ": Warning: ", $_, "\n"); } @w); } sub error { my @w = @_; my $prog = $Global::progname || "parallel"; - status(map { ($prog, ": Error: ", $_, "\n"); } @w); + status(map { ($prog.": Error: ". $_); } @w); } sub die_bug { @@ -3711,7 +3712,7 @@ sub citation { "GNU parallel is indirectly financed through citations, so if users", "do not know they should cite then you are making it harder to finance", "development. However, if you pay 10000 EUR, you should feel free to", - "use --will-cite.", + "use '--will-cite' in scripts.", ""); last; } @@ -3798,9 +3799,7 @@ sub qqx { if($Global::debug) { return qx{ @_ && true }; } else { - local *STDERR; - open (STDERR, ">", "/dev/null"); - return qx{ @_ }; + return qx{ ( @_ ) 2>/dev/null }; } } @@ -4580,7 +4579,7 @@ sub swap_activity { my $file = $self->{'swap_activity_file'}; my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".swp"); ::debug("swap", "\n", $swap_activity, "\n"); - ::qqx("($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile) &"); + ::qqx("($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile &)"); } return $self->{'swap_activity'}; } @@ -4868,10 +4867,10 @@ sub loadavg { } # As the command can take long to run if run remote # save it to a tmp file before moving it to the correct file - ::debug("load", "Cmd: ", $cmd); + ::debug("load", "Cmd: ", $cmd,"\n"); my $file = $self->{'loadavg_file'}; my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa"); - ::qqx(" ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & "); + ::qqx("($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile & )"); } return $self->{'loadavg'}; } @@ -6860,11 +6859,14 @@ sub wrapped { and length $command > 499) { # csh does not like words longer than 1000 (499 quoted) - # bzip2 breaks --sql mysql://... # $command = "perl -e '".base64_zip_eval()."' ". - # join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"'); - $command = "perl -e '".base64_eval()."' ". - join" ",string_base64('exec "'.::perl_quote_scalar($command).'"'); + # join" ",string_zip_base64( + # 'exec "'.::perl_quote_scalar($command).'"'); + $command = "perl -e ". + ::shell_quote_scalar(base64_eval())." ". + join(" ", ::shell_quote( + string_base64('exec "'. + ::perl_quote_scalar($command).'"'))); } $self->{'wrapped'} = $command; } @@ -6888,7 +6890,7 @@ sub sshlogin { } sub string_base64 { - # Base64 encode it into 1000 byte blocks. + # Base64 encode strings into 1000 byte blocks. # 1000 bytes is the largest word size csh supports # Input: # @strings = to be encoded @@ -7153,8 +7155,9 @@ sub sshlogin_wrap { # bzip2 breaks --sql mysql://... # $env_command = "perl -e '".base64_zip_eval()."' ". # join" ",string_zip_base64($env_command); - $env_command = "perl -e '".base64_eval()."' ". - join" ",string_base64($env_command); + $env_command = "perl -e ". + ::shell_quote_scalar(base64_eval())." ". + join" ",::shell_quote(string_base64($env_command)); $self->{'sshlogin_wrap'} = $env_command; } else { $self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($env_command); @@ -7172,9 +7175,12 @@ sub sshlogin_wrap { } my ($csh_friendly,$envset,$bashfuncset) = env_as_eval(); my $remote_command = $pwd.$envset.$bashfuncset. - '@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script(); - $quoted_remote_command = "perl -e ".::shell_quote_scalar($remote_command); - my $dq_remote_command = ::shell_quote_scalar($quoted_remote_command); + '@ARGV="'.::perl_quote_scalar($command).'";'. + monitor_parent_sshd_script(); + $quoted_remote_command = "perl -e ". + ::shell_quote_scalar($remote_command); + my $dq_remote_command = + ::shell_quote_scalar($quoted_remote_command); if(length $dq_remote_command > 999 or not $csh_friendly @@ -7182,11 +7188,9 @@ sub sshlogin_wrap { $command =~ /\n/) { # csh does not deal well with > 1000 chars in one word # csh does not deal well with $ENV with \n - # bzip2 breaks --sql mysql://... - # $quoted_remote_command = "perl -e \\''".base64_zip_eval()."'\\' "."". - # join" ",string_zip_base64($remote_command); - $quoted_remote_command = "perl -e \\''".base64_eval()."'\\' ". - join" ",string_base64($remote_command); + $quoted_remote_command = "perl -e ". + ::shell_quote_scalar(::shell_quote_scalar(base64_eval()))." ". + join" ",::shell_quote(string_base64($remote_command)); } else { $quoted_remote_command = $dq_remote_command; } @@ -8173,13 +8177,14 @@ sub set_exitsignal { $Global::halt_exitstatus = $job->exitstatus(); } } - ::debug("halt","Pct: ",$Global::halt_pct," count: ",$Global::halt_count,"\n"); + ::debug("halt","Pct: ",$Global::halt_pct, + " count: ",$Global::halt_count,"\n"); if($Global::halt_when eq "soon" and scalar(keys %Global::running) > 0) { ::status - ("$Global::progname: Starting no more jobs. ", - "Waiting for ", scalar(keys %Global::running), - " jobs to finish.\n"); + ("$Global::progname: Starting no more jobs. ". + "Waiting for ". (keys %Global::running). + " jobs to finish."); $Global::start_no_new_jobs ||= 1; } return($Global::halt_when); @@ -8199,9 +8204,9 @@ sub set_exitsignal { if($Global::halt_when eq "soon" and scalar(keys %Global::running) > 0) { ::status - ("$Global::progname: Starting no more jobs. ", - "Waiting for ", scalar(keys %Global::running), - " jobs to finish.\n"); + ("$Global::progname: Starting no more jobs. ". + "Waiting for ". (keys %Global::running). + " jobs to finish."); $Global::start_no_new_jobs ||= 1; } return($Global::halt_when); @@ -9158,7 +9163,7 @@ sub tmux_length { push @out, ::qqx($tmuxcmd); unlink $tmpfile; } - ::debug("tmux","tmux-length ",@out); + ::debug("tmux","tmux-out ",@out); chomp @out; # The arguments is given 3 times on the command line # and the wrapping is around 30 chars diff --git a/src/parallel.pod b/src/parallel.pod index f47b7aa7..83c5f4f5 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -464,9 +464,9 @@ B<--will-cite>. If you use B<--will-cite> in scripts to be run by others you are making it harder for others to see the citation notice. The development of GNU B is indirectly financed through -citations, so if users do not know they should cite then you are +citations, so if your users do not know they should cite then you are making it harder to finance development. However, if you pay 10000 -EUR, you should feel free to use B<--will-cite>. +EUR, you should feel free to use B<--will-cite> in scripts. =item B<--block> I diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index be1b6100..1b0d9d07 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -1276,7 +1276,7 @@

And a graphic bar can be shown with --bar and zenity:

-
  seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill)
+
  seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill --auto-close)

A logfile of the jobs completed so far can be generated with --joblog:

diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 4aa4cb63..803ac3cc 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -1208,7 +1208,7 @@ A progress bar can be shown with B<--bar>: And a graphic bar can be shown with B<--bar> and B: - seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill) + seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill --auto-close) A logfile of the jobs completed so far can be generated with B<--joblog>: diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index c389a67d..628a4608 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -53,6 +53,10 @@ echo '### Do children receive --termseq signals' echo | stdout parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals; sleep 3; +echo '**' +echo '### bug #47644: Wrong slot number replacement when resuming' + seq 0 20 | parallel -kj 4 --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==10 and exit =}'; + seq 0 20 | parallel -kj 4 --resume --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==110 and exit =}' EOF diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index edd579ac..3cafdb36 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -64,6 +64,21 @@ echo '### csh2' setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' + +echo '### rc' + echo "3 big vars run remotely - length(base64) > 1000" + stdout ssh rc@lo 'A=`{seq 200}; + B=`{seq 200 -1 1}; + C=`{seq 300 -2 1}; + parallel -Src@lo --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' + +echo '### rc2' + echo "3 big vars run locally" + stdout ssh rc@lo 'A=`{seq 200}; + B=`{seq 200 -1 1}; + C=`{seq 300 -2 1}; + parallel --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' + echo '### Test tmux works on different shells' (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output'; (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | grep -v 'See output'; diff --git a/testsuite/tests-to-run/parallel-local12.sh b/testsuite/tests-to-run/parallel-local12.sh index 32c2e489..2bbb3f45 100644 --- a/testsuite/tests-to-run/parallel-local12.sh +++ b/testsuite/tests-to-run/parallel-local12.sh @@ -36,4 +36,6 @@ echo ZZZZ | script -q -f -c /tmp/parallel-script-for-script3 /dev/null sleep 2 rm /tmp/parallel-script-for-script3 +stdout parallel --citation < /dev/null + touch ~/.parallel/will-cite diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 93b03601..d428ce38 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -14,7 +14,7 @@ export PARALLEL=-k perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/parallel_tutorial.pod | egrep -v 'curl|tty|parallel_tutorial|interactive|example.(com|net)|shellquote|works' | perl -pe 's/username@//;s/user@//; - s/zenity/zenity --timeout=12/; + s/zenity/zenity --timeout=15/; s:/usr/bin/time:/usr/bin/time -f %e:; s:ignored_vars:ignored_vars|sort:; # Remove \n to join all joblogs into the previous block diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 1c280e50..49b06519 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -34,9 +34,34 @@ bash---pstree ** echo '### Do children receive --termseq signals' ### Do children receive --termseq signals - show_signals() { perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print STDERR \"Got $_\\n\"; }";} while(1){sleep 1}'; }; export -f show_signals; echo | stdout parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals; echo | stdout parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals; sleep 3; + show_signals() { perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print STDERR \"Got $_\\n\"; }";} while(1){sleep 1}'; }; export -f show_signals; echo | stdout parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals; echo | stdout parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals; sleep 3; echo '**' Got TERM Got TERM Got TERM Got INT Got TERM +** +echo '### bug #47644: Wrong slot number replacement when resuming' +### bug #47644: Wrong slot number replacement when resuming + seq 0 20 | parallel -kj 4 --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==10 and exit =}'; seq 0 20 | parallel -kj 4 --resume --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==110 and exit =}' +1 0 +2 1 +3 2 +4 3 +1 4 +2 5 +3 6 +4 7 +1 8 +2 9 +3 10 +4 11 +1 12 +2 13 +3 14 +4 15 +1 16 +2 17 +3 18 +4 19 +1 20 diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 20750bd0..eafb3575 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -59,6 +59,22 @@ echo '### csh2' 1 200 692 1 200 692 1 150 547 +echo '### rc' +### rc + echo "3 big vars run remotely - length(base64) > 1000" +3 big vars run remotely - length(base64) > 1000 + stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel -Src@lo --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' + 1 200 692 + 1 200 692 + 1 150 547 +echo '### rc2' +### rc2 + echo "3 big vars run locally" +3 big vars run locally + stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' + 1 200 692 + 1 200 692 + 1 150 547 echo '### Test tmux works on different shells' ### Test tmux works on different shells (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output'; diff --git a/testsuite/wanted-results/parallel-local-ssh5 b/testsuite/wanted-results/parallel-local-ssh5 index bf226497..35c7cb49 100644 --- a/testsuite/wanted-results/parallel-local-ssh5 +++ b/testsuite/wanted-results/parallel-local-ssh5 @@ -34,7 +34,7 @@ Environment variables are: stderr rsync: connection unexpectedly closed (0 bytes received so far) [sender] -rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.1] +rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.0] /usr/lib/autossh/autossh: invalid option -- '-' usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS] @@ -66,7 +66,7 @@ Environment variables are: stderr rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] -rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.1] +rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.0] echo '### bug #46520: --basefile cleans up without --cleanup' ### bug #46520: --basefile cleans up without --cleanup touch bug_46520; parallel -S parallel@lo --bf bug_46520 ls ::: bug_46520; ssh parallel@lo ls bug_46520; parallel -S parallel@lo --cleanup --bf bug_46520 ls ::: bug_46520; stdout ssh parallel@lo ls bug_46520 # should not exist diff --git a/testsuite/wanted-results/parallel-local114 b/testsuite/wanted-results/parallel-local114 index cb6bfc75..bac6acee 100644 Binary files a/testsuite/wanted-results/parallel-local114 and b/testsuite/wanted-results/parallel-local114 differ diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index d8820f91..eea55dd8 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -9,7 +9,7 @@ please cite: This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence the citation notice: run 'parallel --bibtex'. +To silence the citation notice: run 'parallel --citation'. echo a a @@ -30,7 +30,36 @@ please cite: This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence the citation notice: run 'parallel --bibtex'. +To silence the citation notice: run 'parallel --citation'. 10 files to edit -[?1049h[?1h=[?12;25h[?12l[?25h[?25l"file1" [New File]~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ [?12l[?25h[?25lE173: 9 more files to edit[?12l[?25h[?1l>[?1049l \ No newline at end of file +[?1049h[?1h=[?12;25h[?12l[?25h[?25l"file1" [New File]~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ [?12l[?25h[?25lE173: 9 more files to edit[?12l[?25h[?1l>[?1049lAcademic tradition requires you to cite works you base your article on. +When using programs that use GNU Parallel to process data for publication +please cite: + +@article{Tange2011a, + title = {GNU Parallel - The Command-Line Power Tool}, + author = {O. Tange}, + address = {Frederiksberg, Denmark}, + journal = {;login: The USENIX Magazine}, + month = {Feb}, + number = {1}, + volume = {36}, + url = {http://www.gnu.org/s/parallel}, + year = {2011}, + pages = {42-47}, + doi = {10.5281/zenodo.16303} +} + +(Feel free to use \nocite{Tange2011a}) + +This helps funding further development; AND IT WON'T COST YOU A CENT. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. + +If you send a copy of your published article to tange@gnu.org, it will be +mentioned in the release notes of next version of GNU Parallel. + + + +Type: 'will cite' and press enter. +> \ No newline at end of file diff --git a/testsuite/wanted-results/parallel-local13 b/testsuite/wanted-results/parallel-local13 index 628a01c7..d1479feb 100644 --- a/testsuite/wanted-results/parallel-local13 +++ b/testsuite/wanted-results/parallel-local13 @@ -456,7 +456,7 @@ echo '### true < 32767-ys.xi' ### true < 32767-ys.xi stdout xargs true < 32767-ys.xi stdout parallel -k true < 32767-ys.xi -parallel: Error: Command line too long (98306 >= 65535) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ... +parallel: Error: Command line too long (98306 >= 65528) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ... echo '### true < 16383-ys.xi' ### true < 16383-ys.xi stdout xargs true < 16383-ys.xi diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index f4046b28..81ad0343 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -111,7 +111,7 @@ echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | par 20 179960 1286702 echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1 ### Test -X with 60000 args -af6c8c69409b5cbc55aeb95174e23c37 - +97715240fa65309902932877d24273db - 21 60000 1346682 echo '### Test -X with 60000 args and 5 expansions' ### Test -X with 60000 args and 5 expansions diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 13b451fd..0ed7fdfd 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -1,16 +1,16 @@ echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; CHAR=$(cat /tmp/a$$ | wc -c); LINES=$(cat /tmp/a$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/awc$$; rm /tmp/a$$ /tmp/awc$$ ### Test of xargs -m command lines > 130k -cb2184f70b8e9058cae1413426f8a9ae - +14bacad229d8b0d32be0a2339c2a6af7 - Chars per line: 63437 11 119989 697810 echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$ ### Test of xargs -X command lines > 130k -514f33181f346d416a1ada982b43a542 - +81c0a85162c989c07f666b827a30ce52 - Chars per line: 62906 13 60000 817788 echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum ### Test of xargs -m command lines > 130k -1e763b036fe666d229520dad45610e6c - +4c821f9aa0e42e53b4b5ff8cc809490b - echo '### This causes problems if we kill child processes'; # 2>/dev/null to avoid parallel: Warning: Starting 45 processes took > 2 sec. ### This causes problems if we kill child processes seq 2 40 | parallel -j 0 seq 1 10 2>/dev/null | sort | md5sum diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 7afbc647..617b251e 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -82,13 +82,13 @@ GNU parallel comes with no warranty. Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication -please cite as described in 'parallel --bibtex'. +please cite as described in 'parallel --citation'. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' -65526 -65526 -37842 +65520 +65520 +37854 echo '### --delay should grow by 3 sec per arg' ### --delay should grow by 3 sec per arg stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 24d79973..ba52850c 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -407,8 +407,8 @@ seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit echo | $NICEPAR --show-limits -Maximal size of command: 131071 -Maximal used size of command: 65535 +Maximal size of command: 131048 +Maximal used size of command: 65528 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 834b34e8..7adffaeb 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -407,13 +407,13 @@ C outdir/1/C/seq outdir/1/C/stderr outdir/1/C/stdout -/bin/bash: outdir/1/A/seq: No such file or directory +/bin/bash: outdir/1/A/seq: Permission denied /bin/bash: line 1: outdir/1/A/stderr: No such file or directory /bin/bash: line 2: outdir/1/A/stdout: No such file or directory -/bin/bash: line 3: outdir/1/B/seq: No such file or directory +/bin/bash: line 3: outdir/1/B/seq: Permission denied /bin/bash: line 4: outdir/1/B/stderr: No such file or directory /bin/bash: line 5: outdir/1/B/stdout: No such file or directory -/bin/bash: line 6: outdir/1/C/seq: No such file or directory +/bin/bash: line 6: outdir/1/C/seq: Permission denied /bin/bash: line 7: outdir/1/C/stderr: No such file or directory /bin/bash: line 8: outdir/1/C/stdout: No such file or directory parallel --header : --results outdir echo ::: f1 A B ::: f2 C D @@ -522,7 +522,7 @@ Computers / CPU cores / Max jobs to run Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete - seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill) + seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=15 --progress --auto-kill --auto-close) BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log; parallel --joblog /tmp/log exit ::: 1 2 3 0 @@ -708,6 +708,36 @@ parallel@lo foo bar alias myecho=echo myvar="Joe's var is" env_parallel -S $SERVER1 'myecho $myvar' ::: green +env_parallel only works if it is a function. Do the below and restart your shell. + +bash: Put this in /home/tange/.bashrc: . /usr/local/bin/env_parallel.bash + E.g. by doing: echo '. /usr/local/bin/env_parallel.bash' >> /home/tange/.bashrc + Supports: aliases, functions, variables, arrays + +zsh: Put this in /home/tange/.zshrc: . /usr/local/bin/env_parallel.zsh + E.g. by doing: echo '. /usr/local/bin/env_parallel.zsh' >> /home/tange/.zshenv + Supports: functions, variables, arrays + +fish: Put this in /home/tange/.config/fish/config.fish: + source (which env_parallel.fish) + E.g. by doing: + echo 'source (which env_parallel.fish)' >> /home/tange/.config/fish/config.fish + Supports: aliases, functions, variables, arrays + +ksh: Put this in /home/tange/.kshrc: source /usr/local/bin/env_parallel.ksh + E.g. by doing: echo 'source /usr/local/bin/env_parallel.ksh' >> /home/tange/.kshrc + Supports: aliases, functions, variables, arrays + +pdksh: Put this in /home/tange/.profile: source /usr/local/bin/env_parallel.pdksh + E.g. by doing: echo 'source /usr/local/bin/env_parallel.pdksh' >> /home/tange/.profile + Supports: aliases, functions, variables, arrays + +csh: Put this in /home/tange/.cshrc: source /usr/local/bin/env_parallel.csh + E.g. by doing: echo 'source /usr/local/bin/env_parallel.csh' >> /home/tange/.cshrc + Supports: aliases + +For details: see man env_parallel + MYVAR='foo bar' export MYVAR parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz @@ -827,7 +857,7 @@ _ } export -f my_func3 parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file -( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64 BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status; +( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; BASE64 BASE64\=;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status; parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14 cat /tmp/log.csv 10 @@ -1238,7 +1268,7 @@ GNU parallel comes with no warranty. Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication -please cite as described in 'parallel --bibtex'. +please cite as described in 'parallel --citation'. parallel --minversion VERSION && echo Your version is at least VERSION. VERSION Your version is at least VERSION. @@ -1269,8 +1299,7 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing. If you send a copy of your published article to tange@gnu.org, it will be mentioned in the release notes of next version of GNU Parallel. - parallel --max-line-length-allowed -131071 + parallel --number-of-cpus parallel --number-of-cores 1