parcat: Small fixup.

testsuite: Fixups for --line-buffer/--files.

doc/release_new_version

@@ -219,18 +219,15 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
    Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
    Jesse Alama <jesse.alama@gmail.com>
 
-Subject: GNU Parallel 20160922 ('Umbria') released <<[stable]>>
+Subject: GNU Parallel 20160922 ('Christiania') released <<[stable]>>
 
-GNU Parallel 20160922 ('Umbria') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20160922 ('Christiania') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
 Haiku of the month:
 
-  Want to monitor
+    -- Ole Tange
-  many job outputs at once:
-  --tmuxpane
-    --ole-tange
 
 New in this release:
 
@@ -266,8 +263,26 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
 
 * <<Possible: http://link.springer.com/article/10.1007/s12021-015-9290-5 http://link.springer.com/protocol/10.1007/978-1-4939-3578-9_14>>
 
+* parcat - cat files or fifos in parallel
+
+* GNU Parallel was cited in: Velocity: A Netflow Based Optimized Geo-IP Lookup Tool https://mospace.umsystem.edu/xmlui/bitstream/handle/10355/50842/Thesis_2016_Pakhare.pdf?sequence=1&isAllowed=y
+
+* GNU Parallel was cited in: Viral deep sequencing needs an adaptive approach: IRMA, the iterative refinement meta-assembler http://www.ncbi.nlm.nih.gov/pmc/articles/PMC5011931/
+
+* GNU Parallel is used in Gene Set Omic Analysis: https://bitbucket.org/srp33/gsoa
+
+* Using GNU Parallel and ImageMagick for Conversions https://blog.fractalcomic.com/tutorials/using-gnu-parallel-and-imagemagick-for-conversions/
+
+* Training in GNU Parallel http://www.cism.ucl.ac.be/Services/Formations/parcomp.pdf
+
 * Chown Ceph OSD data directory using GNU Parallel http://blog.widodh.nl/2016/08/chown-ceph-osd-data-directory-using-gnu-parallel/
 
+* FastGBS uses GNU Parallel https://bitbucket.org/jerlar73/fastgbs
+
+* dDocent: A guide to ddRAD sequencing https://ddocent.wordpress.com/ddocent-pipeline-user-guide/
+
+* SOFTWARE GNU Parallel http://msi-riss.readthedocs.io/en/latest/software/gnu-parallel.html
+
 * Bug fixes and man page updates.
 
 GNU Parallel - For people who live life in the parallel lane.

src/parallel.pod

@@ -264,11 +264,11 @@ shell quote a string
 
 number of jobs in total
 
-=item Z<> B<$job->slot()>
+=item Z<> B<$job->>B<slot()>
 
 slot number of job
 
-=item Z<> B<$job->seq()>
+=item Z<> B<$job->>B<seq()>
 
 sequence number of job
 

src/parcat

@@ -32,15 +32,8 @@ GNU B<parcat> is part of GNU B<parallel>. Report bugs to <bug-parallel@gnu.org>.
 
 =head1 AUTHOR
 
-When using GNU B<sql> for a publication please cite:
+Copyright (C) 2016 Ole Tange, http://ole.tange.dk and Free Software
-
+Foundation, Inc.
-O. Tange (2011): GNU SQL - A Command Line Tool for Accessing Different
-Databases Using DBURLs, ;login: The USENIX Magazine, April 2011:29-32.
-
-Copyright (C) 2008,2009,2010 Ole Tange http://ole.tange.dk
-
-Copyright (C) 2010,2011 Ole Tange, http://ole.tange.dk and Free
-Software Foundation, Inc.
 
 =head1 LICENSE
 
@@ -203,14 +196,12 @@ sub producer {
 }
 
 my $s = IO::Select->new();
-my %fhr;
 my %buffer;
 
 sub add_file {
     my $fd = shift;
     open(my $fh, "<&=", $fd) || die;
     $s->add($fh);
-    $fhr{$fh}++;
     # Tell the producer now opened here and can be released
     $okq->enqueue($fd);
     # Initialize the buffer
@@ -236,9 +227,9 @@ my (@ready,$file,$rv,$buf);
 do {
     # Wait until at least one file is opened
     add_files_block();
-    while($q->pending or keys %fhr) {
+    while($q->pending or keys %buffer) {
 	add_files();
-	while(keys %fhr) {
+	while(keys %buffer) {
 	    @ready = $s->can_read(0.01);
 	    if(not @ready) {
 		add_files();
@@ -252,7 +243,6 @@ do {
 		    } else {
 			# This file is done
 			$s->remove($file);
-			delete $fhr{$file};
 			print @{$buffer{$file}};
 			delete $buffer{$file};
 			# Closing the $file causes it to block

testsuite/Makefile

@@ -58,7 +58,7 @@ startvm:
 	# If they are already running: Don't fail
 	VBoxManage startvm CentOS3-root:centos3 || VBoxManage controlvm CentOS3-root:centos3 resume || true
 	VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true
-	parallel ping -c 3 ::: redhat9.tange.dk centos3.tange.dk oracle11.tange.dk
+	parallel 'ping {} | read' ::: redhat9.tange.dk centos3.tange.dk
 
 stopvm:
 	# || true - because this should not fail if the VM is not running
@@ -73,6 +73,7 @@ startdb:
 	#echo shutdown abort | sudo su - oracle -c "sqlplus / as sysdba"
 	sudo parallel /etc/init.d/{} status '||' /etc/init.d/{} restart ::: postgresql mysql # oracle-xe
 	VBoxManage startvm OracleXE || true
+	parallel 'ping {} | read' :::  oracle11.tange.dk
 
 clean:
 	rm -rf input-files/random_dirs_no_newline

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -643,7 +643,7 @@ echo '**'
 
 echo '### bug #48658: --linebuffer --files'
 
-  parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
+  stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
 
 echo '**'
 

testsuite/tests-to-run/parallel-local-100s.sh

@@ -33,23 +33,21 @@ measure() {
     # Input:
     #   $1 = iterations
     #   $2 = sleep 1 sec for every $2
-    seq $1 |
+    seq $1 | ramusage parallel -u sleep '{= $_=$_%'$2'?0:1 =}'
-	stdout time -v parallel -u sleep '{= $_=$_%'$2'?0:1 =}' |
-	grep Maximum | field 6;
 }
 export -f measure
 
 no_mem_leak() {
     # Return false if leaking
-    max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 |
+    max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 1000 1000 1000 1000 |
     		       sort -n | tail -n 1)
-    min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 |
+    min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 |
     			sort -n | head -n 1)
     if [ $max1000 -gt $min30000 ] ; then
 	# Make sure there are a few sleeps
-	max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 |
+	max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 1000 1000 1000 1000 |
 			   sort -n | tail -n 1)
-	min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 |
+	min30000=$(parallel measure {} 100 ::: 30000 30000 30000 |
 			    sort -n | head -n 1)
 	if [ $max1000 -gt $min30000 ] ; then
 	    echo $max1000 -gt $min30000 = no leak

testsuite/tests-to-run/parallel-local-1s.sh

@@ -82,6 +82,20 @@ echo 'bug #41613: --compress --line-buffer no newline';
 
   perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
 
+echo 'bug #48658: --linebuffer --files';
+
+  doit() { parallel --files --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; 
+  export -f doit; 
+  parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
+
+  doit() { parallel --results /tmp/par48658$1 --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; rm -rf "/tmp/par48658$1"; }; 
+  export -f doit; 
+  parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
+
+  doit() { parallel --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; 
+  export -f doit; 
+  parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
+
 echo '**'
 
 echo "### Test -I"; 

testsuite/wanted-results/parallel-install

@@ -119,18 +119,32 @@ Warning: pod2html not found. Using old parallel_tutorial.html
 rm -f ./pod2htm*
 pod2texi --output=./parallel.texi ./parallel.pod \
 || echo "Warning: pod2texi not found. Using old parallel.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parallel.texi
 pod2texi --output=./env_parallel.texi ./env_parallel.pod \
 || echo "Warning: pod2texi not found. Using old env_parallel.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old env_parallel.texi
 pod2texi --output=./sem.texi ./sem.pod \
 || echo "Warning: pod2texi not found. Using old sem.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old sem.texi
 pod2texi --output=./sql.texi ./sql \
 || echo "Warning: pod2texi not found. Using old sql.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old sql.texi
 pod2texi --output=./niceload.texi ./niceload.pod \
 || echo "Warning: pod2texi not found. Using old niceload.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old niceload.texi
 pod2texi --output=./parallel_tutorial.texi ./parallel_tutorial.pod \
 || echo "Warning: pod2texi not found. Using old parallel_tutorial.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parallel_tutorial.texi
 pod2texi --output=./parallel_design.texi ./parallel_design.pod \
 || echo "Warning: pod2texi not found. Using old parallel_design.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parallel_design.texi
 pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
 || echo "Warning: pod2pdf not found. Using old parallel.pdf"
 /bin/bash: pod2pdf: command not found
@@ -161,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
 Warning: pod2pdf not found. Using old parallel_design.pdf
 make[0]: Entering directory '/tmp/parallel-00000000/src'
  /bin/mkdir -p '/usr/local/bin'
- /usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin'
+ /usr/bin/install -c parallel sql niceload parcat env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin'
 make  install-exec-hook
 make[0]: Entering directory '/tmp/parallel-00000000/src'
 rm /usr/local/bin/sem || true
@@ -209,6 +223,34 @@ pod2html --title "GNU Parallel tutorial" ./parallel_tutorial.pod > ./parallel_tu
 /bin/bash: pod2html: command not found
 Warning: pod2html not found. Using old parallel_tutorial.html
 rm -f ./pod2htm*
+pod2texi --output=./parallel.texi ./parallel.pod \
+|| echo "Warning: pod2texi not found. Using old parallel.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parallel.texi
+pod2texi --output=./env_parallel.texi ./env_parallel.pod \
+|| echo "Warning: pod2texi not found. Using old env_parallel.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old env_parallel.texi
+pod2texi --output=./sem.texi ./sem.pod \
+|| echo "Warning: pod2texi not found. Using old sem.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old sem.texi
+pod2texi --output=./sql.texi ./sql \
+|| echo "Warning: pod2texi not found. Using old sql.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old sql.texi
+pod2texi --output=./niceload.texi ./niceload.pod \
+|| echo "Warning: pod2texi not found. Using old niceload.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old niceload.texi
+pod2texi --output=./parallel_tutorial.texi ./parallel_tutorial.pod \
+|| echo "Warning: pod2texi not found. Using old parallel_tutorial.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parallel_tutorial.texi
+pod2texi --output=./parallel_design.texi ./parallel_design.pod \
+|| echo "Warning: pod2texi not found. Using old parallel_design.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parallel_design.texi
 pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
 || echo "Warning: pod2pdf not found. Using old parallel.pdf"
 /bin/bash: pod2pdf: command not found
@@ -238,7 +280,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
 /bin/bash: pod2pdf: command not found
 Warning: pod2pdf not found. Using old parallel_design.pdf
  /bin/mkdir -p '/usr/local/share/doc/parallel'
- /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf '/usr/local/share/doc/parallel'
+ /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parcat.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parcat.pdf '/usr/local/share/doc/parallel'
 pod2man --release='00000000' --center='parallel' \
 	--section=1 ./parallel.pod > ./parallel.1n \
 && mv ./parallel.1n ./parallel.1 \
@@ -282,7 +324,7 @@ pod2man --release='00000000' --center='parallel' \
 /bin/bash: pod2man: command not found
 Warning: pod2man not found. Using old parallel_design.7
  /bin/mkdir -p '/usr/local/share/man/man1'
- /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 '/usr/local/share/man/man1'
+ /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 '/usr/local/share/man/man1'
  /bin/mkdir -p '/usr/local/share/man/man7'
  /usr/bin/install -c -m 644 parallel_tutorial.7 parallel_design.7 '/usr/local/share/man/man7'
 make[0]: Leaving directory '/tmp/parallel-00000000/src'

testsuite/wanted-results/parallel-local-0.3s

@@ -1761,9 +1761,9 @@ echo '**'
 **
 echo '### bug #48658: --linebuffer --files'
 ### bug #48658: --linebuffer --files
-  parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
+  stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
 10
 echo '**'
 **
 ### 1 .par file from --files expected
-0
+10

testsuite/wanted-results/parallel-local-1s

@@ -74,8 +74,6 @@ parallel: Error: false failed.
   (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1
 1
 parallel: Error: false failed.
-parallel: Error: false failed.
-parallel: Error: false failed.
 parallel: Error: false failed.
   parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $?
 1
@@ -236,6 +234,58 @@ ls: cannot access '/OK-if-missing-file': No such file or directory
 echo 'bug #41613: --compress --line-buffer no newline';   perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
 bug #41613: --compress --line-buffer no newline
 It worked
+echo 'bug #48658: --linebuffer --files';   doit() { parallel --files --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; };   export -f doit;   parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
+bug #48658: --linebuffer --files
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+  doit() { parallel --results /tmp/par48658$1 --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; rm -rf "/tmp/par48658$1"; };   export -f doit;   parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+  doit() { parallel --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; };   export -f doit;   parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
+100000
 echo '**'
 **
 echo "### Test -I";   seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'

testsuite/wanted-results/parallel-tutorial

@@ -350,6 +350,31 @@ pre- A-post
 pre-A -post
   parallel --trim lr echo pre-{}-post ::: ' A '
 pre-A-post
+  parallel echo \={} ::: zsh bash ls
+=zsh
+=bash
+=ls
+  parallel 'set a="{}"; if( { test -d "$a" } ) echo "$a is a dir"' ::: *
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="abc-file"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="abc0-file"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="abc_-file"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="def-file"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="num1000000"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="num128"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="num30000"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="num8"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="num_%header"; if( { test -d "$a" } ) echo "$a is a dir"'
+/bin/bash: -c: line 0: syntax error near unexpected token `)'
+/bin/bash: -c: line 0: `set a="tsv-file.tsv"; if( { test -d "$a" } ) echo "$a is a dir"'
   parallel --tag echo foo-{} ::: A B C
 A	foo-A
 B	foo-B