testsuite: --shebang-wrap for Octave, Python. Perl, GnuPlot, R, Ruby, Bash.

10seconds_install: Try curl.
Makefile: parallel.html dependency.
parallel: Printing joblog => print_joblog.
parallel.pod: Removed TinyOgg.
testsuite: Minor fixups.

10seconds_install

@@ -18,13 +18,15 @@
 TAIL=$(echo | tail -n 1 2>/dev/null && echo tail || (echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail))
 # grep on openindiana must be /usr/xpg4/bin/grep
 GREP=$(echo | grep -vE . 2>/dev/null && echo grep || (echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep))
+# wget is not default on MacOS, so try curl
+GET=$(curl -h >/dev/null && echo curl -L || echo wget -qO -)
 
-LATEST=$(wget -qO- http://ftpmirror.gnu.org/parallel | perl -ne '/(parallel-\d{8})/ and print $1."\n"' | sort | $TAIL -n1)
+LATEST=$($GET http://ftpmirror.gnu.org/parallel | perl -ne '/(parallel-\d{8})/ and print $1."\n"' | sort | $TAIL -n1)
 if test \! -d $LATEST/src/; then
   # Source dir does not exist
   rm -f $LATEST.tar.bz2 $LATEST.tar.bz2.sig
-  wget http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2
+  $GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2 > $LATEST.tar.bz2
-  wget http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig
+  $GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig > $LATEST.tar.bz2.sig
 fi
 
 # Check signature - in case ftpmirror.gnu.org is compromised

doc/promo

@@ -1,3 +1,9 @@
+=head1 top photos
+
+http://www.flickr.com/photos/dexxus/5499821986/in/photostream/
+https://www.google.com/search?lr=&safe=images&hl=en&tbs=sur:fmc&tbm=isch&q=top+nature+photos&revid=600471240&biw=1024&bih=569
+
+
 =head1 What is GNU Parallel used for
 
 Searching for transit planets using data from the Kepler space telescope.

doc/release_new_version

@@ -230,9 +230,15 @@ New in this release:
 * Compression of files in parallel using GNU parallel
   http://codextechnicanum.blogspot.dk/2013/07/compression-of-files-in-parallel-using.html
 
+* Using GNU Parallel to roll-your-own Map Reduce!
+  http://www.rankfocus.com/?p=1
+
 * TimeMachineっぽいバックアップスクリプト
   http://rio.tc/2013/07/timemachine-1.html
 
+* Using GNU Parallel with Amazon
+  media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf
+
 * Some useful comments on GNU Parallel
   https://news.ycombinator.com/item?id=6209767
 

src/Makefile.in

@@ -508,6 +508,10 @@ niceload.1: niceload.pod Makefile
 	pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
 		--section=1 $(srcdir)/niceload.pod > $(srcdir)/niceload.1
 
+parallel.html: parallel.pod Makefile
+	pod2html $(srcdir)/parallel.pod > $(srcdir)/parallel.html
+	rm -f $(srcdir)/pod2htm*
+
 # Depending on parallel.html to avoid stupid pod2html race condition
 parallel_tutorial.html: parallel_tutorial.pod Makefile parallel.html
 	pod2html $(srcdir)/parallel_tutorial.pod > $(srcdir)/parallel_tutorial.html

src/parallel

@@ -4477,23 +4477,7 @@ sub print {
 	unlink $self->fd_file_name(1);
     }
 
-    if($Global::joblog) {
+    if($Global::joblog) { $self->print_joblog() }
-	my $cmd;
-	if($Global::verbose <= 1) {
-	    $cmd = $self->replaced();
-	} else {
-	    # Verbose level > 1: Print the rsync and stuff
-	    $cmd = $command;
-	}
-	print $Global::joblog
-	    join("\t", $self->seq(), $self->sshlogin()->string(),
-		 $self->starttime(), $self->runtime(),
-		 $self->transfersize(), $self->returnsize(),
-		 $self->exitstatus(), $self->exitsignal(), $cmd
-		 ). "\n";
-	flush $Global::joblog;
-	$self->set_job_in_joblog();
-    }
 
     # Printing is only relevant for grouped output.
     $Global::grouped or return;
@@ -4560,7 +4544,6 @@ sub print {
 		}
 	    }
 
-
 	    if(defined $self->{'exitstatus'} and length $$partial > 0) {
 		# If the job is dead: print the remaining partial line
 		if($opt::tag or defined $opt::tagstring) {
@@ -4611,6 +4594,25 @@ sub print {
     }
 }
 
+sub print_joblog {
+    my $self = shift;
+    my $cmd;
+    if($Global::verbose <= 1) {
+	$cmd = $self->replaced();
+    } else {
+	# Verbose level > 1: Print the rsync and stuff
+	$cmd = $command;
+    }
+    print $Global::joblog
+	join("\t", $self->seq(), $self->sshlogin()->string(),
+	     $self->starttime(), $self->runtime(),
+	     $self->transfersize(), $self->returnsize(),
+	     $self->exitstatus(), $self->exitsignal(), $cmd
+	). "\n";
+    flush $Global::joblog;
+    $self->set_job_in_joblog();
+}
+
 sub tag {
     my $self = shift;
     if(not defined $self->{'tag'}) {
@@ -4726,9 +4728,10 @@ sub seq {
 sub populate {
     # Add arguments from arg_queue until the number of arguments or
     # max line length is reached
+    # Returns: N/A
     my $self = shift;
     if($opt::pipe) {
-	# Do no read any args
+	# --pipe => Do no read any args
 	$self->push([Arg->new("")]);
 	return;
     }
@@ -4788,6 +4791,7 @@ sub populate {
 
 sub push {
     # Add one or more records as arguments
+    # Returns: N/A
     my $self = shift;
     my $record = shift;
     push @{$self->{'arg_list'}}, $record;
@@ -4818,6 +4822,8 @@ sub push {
 
 sub pop {
     # Remove last argument
+    # Returns:
+    #   the last record
     my $self = shift;
     my $record = pop @{$self->{'arg_list'}};
     for my $arg (@$record) {
@@ -4832,7 +4838,9 @@ sub pop {
 }
 
 sub pop_all {
-    # Remove all arguments
+    # Remove all arguments and zeros the length of replacement strings
+    # Returns:
+    #   all records
     my $self = shift;
     my @popped = @{$self->{'arg_list'}};
     for my $replacement_string (keys %{$self->{'replacecount'}}) {
@@ -4843,8 +4851,37 @@ sub pop_all {
 }
 
 sub number_of_args {
+    # The number of records
+    # Returns:
+    #   number of records
     my $self = shift;
-    # This is really number of records
+    # Ftq rudef oaawuq ime dqxqmeqp az 2011-01-24 mzp ime iaz nk MQhmd
+    # Mdzrvadp Nvmdymeaz az 2011-04-10. Ftue oaawuq dqxqmeqp az
+    # 2013-08-18 ue m nuf tmdpqd me kag tmhq fa geq daf14. Bxqmeq
+    # qymux oaawuq@fmzsq.pw itqz kag dqmp ftue.
+    # 
+    # U my ftq ymuzfmuzqd ar m buqoq ar rdqq earfimdq omxxqp SZG
+    # Bmdmxxqx. Rdqq earfimdq sgmdmzfqqe kag mooqee fa ftq eagdoq
+    # oapq, ngf U tmhq nqqz iazpqduzs tai ymzk mofgmxxk _dqmp_ ftq
+    # eagdoq oapq.
+    # 
+    # Fa fqef ftue U bgf uz m oayyqzf fqxxuzs bqabxq fa qymux yq itqz
+    # ftqk dqmp ftue. Ftq oayyqzf ime bgf uz m eqofuaz ar ftq oapq
+    # ftmf za azq iagxp xaaw fa ruj ad uybdahq ftq earfimdq - ea ftq
+    # eagdoq oapq qcguhmxqzf fa m pgefk oadzqd. Fa ymwq egdq ftq
+    # oayyqzf iagxp zaf etai gb ur eayq azq vgef sdqbbqp ftdagst ftq
+    # eagdoq oapq U daf13'qp ftq eagdoq oapq
+    # tffb://qz.iuwubqpum.ads/iuwu/DAF13
+    # 
+    # 2.5 yazfte xmfqd U dqoquhqp mz qymux rday eayqazq ita zaf azxk
+    # ymzmsqp fa ruzp ftq oayyqzf, ngf mxea ymzmsqp fa sgqee ftq oapq
+    # tmp fa nq daf13'qp.
+    # 
+    # Ftue nduzse yq fa ftq oazoxgeuaz ftmf ftqdq _mdq_ bqabxq, ita
+    # mdq zaf mrruxumfqp iuft ftq bdavqof, ftmf iuxx dqmp ftq eagdoq
+    # oapq - ftagst uf ymk zaf tmbbqz hqdk arfqz.
+    #
+    # This is really the number of records
     return $#{$self->{'arg_list'}}+1;
 }
 

src/parallel.pod

@@ -49,8 +49,6 @@ after the list of options. That will give you an idea of what GNU
 B<parallel> is capable of.
 
 You can also watch the intro video for a quick introduction:
-http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and
-http://tinyogg.com/watch/YQuXd/ or
 http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1
 
 =head1 OPTIONS

testsuite/input-files/shebang/shebangwrap.gp

@@ -0,0 +1,4 @@
+#!/usr/bin/parallel --shebang-wrap -k A={} /usr/bin/gnuplot
+
+name=system("echo $A")
+print name

testsuite/input-files/shebang/shebangwrap.oct

@@ -0,0 +1,6 @@
+#!/usr/bin/parallel --shebang-wrap -k /usr/bin/octave -qf
+
+arg_list = argv ();
+filename = arg_list{1};
+printf(filename);
+printf("\n");

testsuite/input-files/shebang/shebangwrap.pl

@@ -0,0 +1,3 @@
+#!/usr/bin/parallel --shebang-wrap -k /usr/bin/perl
+
+print @ARGV,"\n";

testsuite/input-files/shebang/shebangwrap.py

@@ -0,0 +1,5 @@
+#!/usr/bin/parallel --shebang-wrap -k /usr/bin/python
+
+import sys
+
+print str(sys.argv[1])

testsuite/input-files/shebang/shebangwrap.r

@@ -0,0 +1,4 @@
+#!/usr/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave
+
+options <- commandArgs(trailingOnly = TRUE)
+options

testsuite/input-files/shebang/shebangwrap.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/parallel --shebang-wrap -k /usr/bin/ruby
+
+p ARGV

testsuite/input-files/shebang/shebangwrap.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/parallel --shebang-wrap -k /bin/sh
+
+echo "$@"

testsuite/tests-to-run/parallel-local1.sh

@@ -155,10 +155,10 @@ echo '### Bug introduce by fixing bug #37042'
   parallel --xapply -a <(printf 'abc') --colsep '\t' echo {1}
 
 echo "### Test --header with -N"
-  (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '\n.*\n' echo Start\;cat \; echo Stop
+  (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '.*\n.*\n' echo Start\;cat \; echo Stop
 
 echo "### Test --header with --block 1k"
-  (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '\n.*\n' echo Start\;cat \; echo Stop
+  (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '.*\n.*\n' echo Start\;cat \; echo Stop
 
 echo "### Test --header with multiple :::"
   parallel --header : echo {a} {b} {1} {2} ::: b b1 ::: a a2

testsuite/tests-to-run/parallel-local22.sh

@@ -39,6 +39,6 @@ echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop (I
   echo a | parallel -qX echo  "'{}'"
 
 echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL'; 
-  seq 1 2 | SHELL=tcsh stdout parallel -k --nice 8 setenv a b\;echo \$SHELL
+  seq 1 2 | SHELL=tcsh MANPATH=. stdout parallel -k --nice 8 setenv a b\;echo \$SHELL
 
 EOF

testsuite/tests-to-run/parallel-local23.sh

@@ -6,7 +6,7 @@ cp -a input-files/testdir2 tmp
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1
 echo '### Test filenames containing UTF-8'; 
   cd tmp; 
-  find . -name '*.jpg' | nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; 
+  find . -name '*.jpg' | nice nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; 
   find |grep -v CVS | sort; 
 
 echo '### bug #39554: Feature request: line buffered output'; 
@@ -24,7 +24,7 @@ echo '### --version must have higher priority than retired options'
    parallel --version -g -Y -U -W -T | tail
 
 echo '### bug #39787: --xargs broken'
-  perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo  | perl -ne 'print length $_,"\n"'
+  nice perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo  | perl -ne 'print length $_,"\n"'
 
 echo '### --delay should grow by 2 sec per arg'
 stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 | perl -ne '$_ >= 2 and $_ <= 5 and print "OK\n"'
@@ -40,10 +40,13 @@ echo '### --header num'
   (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat
 
 echo '### --header regexp --round-robin'
-  (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB{#}\;cat
+  (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort
 
 echo '### --header num --round-robin'
-  (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2  echo JOB{#}\;cat
+  (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2  echo JOB{#}\;wc | sort
+
+echo '### shebang-wrap'
+  nice nice parallel -k {} A B C ::: ./input-files/shebang/shebangwrap.*[^~]
 
 EOF
 

testsuite/tests-to-run/parallel-remote1.sh

@@ -12,9 +12,11 @@ chmod 755 /tmp/myssh1 /tmp/myssh2
 seq 1 100 | parallel --sshlogin "/tmp/myssh1 $SSHLOGIN1, /tmp/myssh2 $SSHLOGIN2" \
   -j10000% -k echo
 
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/ | parallel -j2 -k -L1
 echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host'
   parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname
 
 echo '### test --workdir . in $HOME'
-  cd && mkdir parallel-test && cd parallel-test && 
+  cd && mkdir -p parallel-test && cd parallel-test && 
     echo OK > testfile && parallel --workdir . --transfer -S $SSHLOGIN1 cat {} ::: testfile
+EOF

testsuite/wanted-results/parallel-local23

@@ -83,26 +83,34 @@ JOB3
 %head2
 5
 ### --header regexp --round-robin
-JOB1
+      4       4      18
-%head1
+      5       5      20
-%head2
+JOB
-3
+JOB
-4
-JOB2
-%head1
-%head2
-1
-2
-5
 ### --header num --round-robin
+      4       4      18
+      5       5      20
 JOB1
-%head1
-%head2
-1
-2
-5
 JOB2
-%head1
+### shebang-wrap
-%head2
+A
-3
+B
-4
+C
+A
+B
+C
+A
+B
+C
+[1] "A"
+[1] "B"
+[1] "C"
+["A"]
+["B"]
+["C"]
+A
+B
+C
+A
+B
+C

testsuite/wanted-results/parallel-remote1

@@ -101,3 +101,5 @@
 100
 ### --filter-hosts - OK, non-such-user, connection refused, wrong host
 hk
+### test --workdir . in $HOME
+OK