From 74e35933f4224fccdad4ae05ab60ed8818ecb2d6 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 23 Sep 2016 00:57:30 +0200 Subject: [PATCH] parcat: Small fixup. testsuite: Fixups for --line-buffer/--files. --- doc/release_new_version | 27 +++++++--- src/parallel.pod | 4 +- src/parcat | 18 ++----- testsuite/Makefile | 3 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 2 +- testsuite/tests-to-run/parallel-local-100s.sh | 12 ++--- testsuite/tests-to-run/parallel-local-1s.sh | 14 +++++ testsuite/wanted-results/parallel-install | 48 +++++++++++++++-- testsuite/wanted-results/parallel-local-0.3s | 4 +- testsuite/wanted-results/parallel-local-1s | 54 ++++++++++++++++++- testsuite/wanted-results/parallel-tutorial | 25 +++++++++ 11 files changed, 173 insertions(+), 38 deletions(-) mode change 100644 => 100755 src/parcat diff --git a/doc/release_new_version b/doc/release_new_version index a79e3e76..b57ecdc1 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -219,18 +219,15 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20160922 ('Umbria') released <<[stable]>> +Subject: GNU Parallel 20160922 ('Christiania') released <<[stable]>> -GNU Parallel 20160922 ('Umbria') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20160922 ('Christiania') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> Haiku of the month: - Want to monitor - many job outputs at once: - --tmuxpane - --ole-tange + -- Ole Tange New in this release: @@ -266,8 +263,26 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> +* parcat - cat files or fifos in parallel + +* GNU Parallel was cited in: Velocity: A Netflow Based Optimized Geo-IP Lookup Tool https://mospace.umsystem.edu/xmlui/bitstream/handle/10355/50842/Thesis_2016_Pakhare.pdf?sequence=1&isAllowed=y + +* GNU Parallel was cited in: Viral deep sequencing needs an adaptive approach: IRMA, the iterative refinement meta-assembler http://www.ncbi.nlm.nih.gov/pmc/articles/PMC5011931/ + +* GNU Parallel is used in Gene Set Omic Analysis: https://bitbucket.org/srp33/gsoa + +* Using GNU Parallel and ImageMagick for Conversions https://blog.fractalcomic.com/tutorials/using-gnu-parallel-and-imagemagick-for-conversions/ + +* Training in GNU Parallel http://www.cism.ucl.ac.be/Services/Formations/parcomp.pdf + * Chown Ceph OSD data directory using GNU Parallel http://blog.widodh.nl/2016/08/chown-ceph-osd-data-directory-using-gnu-parallel/ +* FastGBS uses GNU Parallel https://bitbucket.org/jerlar73/fastgbs + +* dDocent: A guide to ddRAD sequencing https://ddocent.wordpress.com/ddocent-pipeline-user-guide/ + +* SOFTWARE GNU Parallel http://msi-riss.readthedocs.io/en/latest/software/gnu-parallel.html + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel.pod b/src/parallel.pod index 051c9419..c852334b 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -264,11 +264,11 @@ shell quote a string number of jobs in total -=item Z<> B<$job->slot()> +=item Z<> B<$job->>B slot number of job -=item Z<> B<$job->seq()> +=item Z<> B<$job->>B sequence number of job diff --git a/src/parcat b/src/parcat old mode 100644 new mode 100755 index 5d9c943a..6a819e21 --- a/src/parcat +++ b/src/parcat @@ -32,15 +32,8 @@ GNU B is part of GNU B. Report bugs to . =head1 AUTHOR -When using GNU B for a publication please cite: - -O. Tange (2011): GNU SQL - A Command Line Tool for Accessing Different -Databases Using DBURLs, ;login: The USENIX Magazine, April 2011:29-32. - -Copyright (C) 2008,2009,2010 Ole Tange http://ole.tange.dk - -Copyright (C) 2010,2011 Ole Tange, http://ole.tange.dk and Free -Software Foundation, Inc. +Copyright (C) 2016 Ole Tange, http://ole.tange.dk and Free Software +Foundation, Inc. =head1 LICENSE @@ -203,14 +196,12 @@ sub producer { } my $s = IO::Select->new(); -my %fhr; my %buffer; sub add_file { my $fd = shift; open(my $fh, "<&=", $fd) || die; $s->add($fh); - $fhr{$fh}++; # Tell the producer now opened here and can be released $okq->enqueue($fd); # Initialize the buffer @@ -236,9 +227,9 @@ my (@ready,$file,$rv,$buf); do { # Wait until at least one file is opened add_files_block(); - while($q->pending or keys %fhr) { + while($q->pending or keys %buffer) { add_files(); - while(keys %fhr) { + while(keys %buffer) { @ready = $s->can_read(0.01); if(not @ready) { add_files(); @@ -252,7 +243,6 @@ do { } else { # This file is done $s->remove($file); - delete $fhr{$file}; print @{$buffer{$file}}; delete $buffer{$file}; # Closing the $file causes it to block diff --git a/testsuite/Makefile b/testsuite/Makefile index 51edd3b3..29de6f58 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -58,7 +58,7 @@ startvm: # If they are already running: Don't fail VBoxManage startvm CentOS3-root:centos3 || VBoxManage controlvm CentOS3-root:centos3 resume || true VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true - parallel ping -c 3 ::: redhat9.tange.dk centos3.tange.dk oracle11.tange.dk + parallel 'ping {} | read' ::: redhat9.tange.dk centos3.tange.dk stopvm: # || true - because this should not fail if the VM is not running @@ -73,6 +73,7 @@ startdb: #echo shutdown abort | sudo su - oracle -c "sqlplus / as sysdba" sudo parallel /etc/init.d/{} status '||' /etc/init.d/{} restart ::: postgresql mysql # oracle-xe VBoxManage startvm OracleXE || true + parallel 'ping {} | read' ::: oracle11.tange.dk clean: rm -rf input-files/random_dirs_no_newline diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index b1ef1c9d..9806953e 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -643,7 +643,7 @@ echo '**' echo '### bug #48658: --linebuffer --files' - parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l + stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l echo '**' diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index 061b8c11..2bdefb6a 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -33,23 +33,21 @@ measure() { # Input: # $1 = iterations # $2 = sleep 1 sec for every $2 - seq $1 | - stdout time -v parallel -u sleep '{= $_=$_%'$2'?0:1 =}' | - grep Maximum | field 6; + seq $1 | ramusage parallel -u sleep '{= $_=$_%'$2'?0:1 =}' } export -f measure no_mem_leak() { # Return false if leaking - max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 | + max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 1000 1000 1000 1000 | sort -n | tail -n 1) - min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 | + min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 | sort -n | head -n 1) if [ $max1000 -gt $min30000 ] ; then # Make sure there are a few sleeps - max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 | + max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 1000 1000 1000 1000 | sort -n | tail -n 1) - min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 | + min30000=$(parallel measure {} 100 ::: 30000 30000 30000 | sort -n | head -n 1) if [ $max1000 -gt $min30000 ] ; then echo $max1000 -gt $min30000 = no leak diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 06c46098..ad14ac1e 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -82,6 +82,20 @@ echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo +echo 'bug #48658: --linebuffer --files'; + + doit() { parallel --files --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; + export -f doit; + parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz + + doit() { parallel --results /tmp/par48658$1 --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; rm -rf "/tmp/par48658$1"; }; + export -f doit; + parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz + + doit() { parallel --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; + export -f doit; + parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz + echo '**' echo "### Test -I"; diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index 912de29a..df3475fd 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -119,18 +119,32 @@ Warning: pod2html not found. Using old parallel_tutorial.html rm -f ./pod2htm* pod2texi --output=./parallel.texi ./parallel.pod \ || echo "Warning: pod2texi not found. Using old parallel.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parallel.texi pod2texi --output=./env_parallel.texi ./env_parallel.pod \ || echo "Warning: pod2texi not found. Using old env_parallel.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old env_parallel.texi pod2texi --output=./sem.texi ./sem.pod \ || echo "Warning: pod2texi not found. Using old sem.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old sem.texi pod2texi --output=./sql.texi ./sql \ || echo "Warning: pod2texi not found. Using old sql.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old sql.texi pod2texi --output=./niceload.texi ./niceload.pod \ || echo "Warning: pod2texi not found. Using old niceload.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old niceload.texi pod2texi --output=./parallel_tutorial.texi ./parallel_tutorial.pod \ || echo "Warning: pod2texi not found. Using old parallel_tutorial.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parallel_tutorial.texi pod2texi --output=./parallel_design.texi ./parallel_design.pod \ || echo "Warning: pod2texi not found. Using old parallel_design.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parallel_design.texi pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \ || echo "Warning: pod2pdf not found. Using old parallel.pdf" /bin/bash: pod2pdf: command not found @@ -161,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P Warning: pod2pdf not found. Using old parallel_design.pdf make[0]: Entering directory '/tmp/parallel-00000000/src' /bin/mkdir -p '/usr/local/bin' - /usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin' + /usr/bin/install -c parallel sql niceload parcat env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin' make install-exec-hook make[0]: Entering directory '/tmp/parallel-00000000/src' rm /usr/local/bin/sem || true @@ -209,6 +223,34 @@ pod2html --title "GNU Parallel tutorial" ./parallel_tutorial.pod > ./parallel_tu /bin/bash: pod2html: command not found Warning: pod2html not found. Using old parallel_tutorial.html rm -f ./pod2htm* +pod2texi --output=./parallel.texi ./parallel.pod \ +|| echo "Warning: pod2texi not found. Using old parallel.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parallel.texi +pod2texi --output=./env_parallel.texi ./env_parallel.pod \ +|| echo "Warning: pod2texi not found. Using old env_parallel.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old env_parallel.texi +pod2texi --output=./sem.texi ./sem.pod \ +|| echo "Warning: pod2texi not found. Using old sem.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old sem.texi +pod2texi --output=./sql.texi ./sql \ +|| echo "Warning: pod2texi not found. Using old sql.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old sql.texi +pod2texi --output=./niceload.texi ./niceload.pod \ +|| echo "Warning: pod2texi not found. Using old niceload.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old niceload.texi +pod2texi --output=./parallel_tutorial.texi ./parallel_tutorial.pod \ +|| echo "Warning: pod2texi not found. Using old parallel_tutorial.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parallel_tutorial.texi +pod2texi --output=./parallel_design.texi ./parallel_design.pod \ +|| echo "Warning: pod2texi not found. Using old parallel_design.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parallel_design.texi pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \ || echo "Warning: pod2pdf not found. Using old parallel.pdf" /bin/bash: pod2pdf: command not found @@ -238,7 +280,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P /bin/bash: pod2pdf: command not found Warning: pod2pdf not found. Using old parallel_design.pdf /bin/mkdir -p '/usr/local/share/doc/parallel' - /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf '/usr/local/share/doc/parallel' + /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parcat.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parcat.pdf '/usr/local/share/doc/parallel' pod2man --release='00000000' --center='parallel' \ --section=1 ./parallel.pod > ./parallel.1n \ && mv ./parallel.1n ./parallel.1 \ @@ -282,7 +324,7 @@ pod2man --release='00000000' --center='parallel' \ /bin/bash: pod2man: command not found Warning: pod2man not found. Using old parallel_design.7 /bin/mkdir -p '/usr/local/share/man/man1' - /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 '/usr/local/share/man/man1' + /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 '/usr/local/share/man/man1' /bin/mkdir -p '/usr/local/share/man/man7' /usr/bin/install -c -m 644 parallel_tutorial.7 parallel_design.7 '/usr/local/share/man/man7' make[0]: Leaving directory '/tmp/parallel-00000000/src' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index c68ee7a9..06b4df02 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1761,9 +1761,9 @@ echo '**' ** echo '### bug #48658: --linebuffer --files' ### bug #48658: --linebuffer --files - parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l + stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l 10 echo '**' ** ### 1 .par file from --files expected -0 +10 diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 42f9bc34..a1032044 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -74,8 +74,6 @@ parallel: Error: false failed. (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1 1 parallel: Error: false failed. -parallel: Error: false failed. -parallel: Error: false failed. parallel: Error: false failed. parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $? 1 @@ -236,6 +234,58 @@ ls: cannot access '/OK-if-missing-file': No such file or directory echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo bug #41613: --compress --line-buffer no newline It worked +echo 'bug #48658: --linebuffer --files'; doit() { parallel --files --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; export -f doit; parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz +bug #48658: --linebuffer --files +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 + doit() { parallel --results /tmp/par48658$1 --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; rm -rf "/tmp/par48658$1"; }; export -f doit; parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 + doit() { parallel --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; export -f doit; parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 echo '**' ** echo "### Test -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 82ef88fd..f24f2c7c 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -350,6 +350,31 @@ pre- A-post pre-A -post parallel --trim lr echo pre-{}-post ::: ' A ' pre-A-post + parallel echo \={} ::: zsh bash ls +=zsh +=bash +=ls + parallel 'set a="{}"; if( { test -d "$a" } ) echo "$a is a dir"' ::: * +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="abc-file"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="abc0-file"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="abc_-file"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="def-file"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="num1000000"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="num128"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="num30000"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="num8"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="num_%header"; if( { test -d "$a" } ) echo "$a is a dir"' +/bin/bash: -c: line 0: syntax error near unexpected token `)' +/bin/bash: -c: line 0: `set a="tsv-file.tsv"; if( { test -d "$a" } ) echo "$a is a dir"' parallel --tag echo foo-{} ::: A B C A foo-A B foo-B