tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parcat: Small fixup.

Browse source 
testsuite: Fixups for --line-buffer/--files.
This commit is contained in:
Ole Tange 2016-09-23 00:57:30 +02:00
parent a4ddcbf8bc
commit 74e35933f4
11 changed files with 173 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
doc/release_new_version
View file

@ -219,18 +219,15 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20160922 ('Umbria') released <<[stable]>>
Subject: GNU Parallel 20160922 ('Christiania') released <<[stable]>>
GNU Parallel 20160922 ('Umbria') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20160922 ('Christiania') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
Haiku of the month:
Want to monitor
many job outputs at once:
--tmuxpane
--ole-tange
-- Ole Tange
New in this release:
@ -266,8 +263,26 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
* <<Possible: http://link.springer.com/article/10.1007/s12021-015-9290-5 http://link.springer.com/protocol/10.1007/978-1-4939-3578-9_14>>
* parcat - cat files or fifos in parallel
* GNU Parallel was cited in: Velocity: A Netflow Based Optimized Geo-IP Lookup Tool https://mospace.umsystem.edu/xmlui/bitstream/handle/10355/50842/Thesis_2016_Pakhare.pdf?sequence=1&isAllowed=y
* GNU Parallel was cited in: Viral deep sequencing needs an adaptive approach: IRMA, the iterative refinement meta-assembler http://www.ncbi.nlm.nih.gov/pmc/articles/PMC5011931/
* GNU Parallel is used in Gene Set Omic Analysis: https://bitbucket.org/srp33/gsoa
* Using GNU Parallel and ImageMagick for Conversions https://blog.fractalcomic.com/tutorials/using-gnu-parallel-and-imagemagick-for-conversions/
* Training in GNU Parallel http://www.cism.ucl.ac.be/Services/Formations/parcomp.pdf
* Chown Ceph OSD data directory using GNU Parallel http://blog.widodh.nl/2016/08/chown-ceph-osd-data-directory-using-gnu-parallel/
* FastGBS uses GNU Parallel https://bitbucket.org/jerlar73/fastgbs
* dDocent: A guide to ddRAD sequencing https://ddocent.wordpress.com/ddocent-pipeline-user-guide/
* SOFTWARE GNU Parallel http://msi-riss.readthedocs.io/en/latest/software/gnu-parallel.html
* Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane.

4
src/parallel.pod
View file

@ -264,11 +264,11 @@ shell quote a string
number of jobs in total
=item Z<> B<$job->slot()>
=item Z<> B<$job->>B<slot()>
slot number of job
=item Z<> B<$job->seq()>
=item Z<> B<$job->>B<seq()>
sequence number of job

18
src/parcat Normal file → Executable file
View file

@ -32,15 +32,8 @@ GNU B<parcat> is part of GNU B<parallel>. Report bugs to <bug-parallel@gnu.org>.
=head1 AUTHOR
When using GNU B<sql> for a publication please cite:
O. Tange (2011): GNU SQL - A Command Line Tool for Accessing Different
Databases Using DBURLs, ;login: The USENIX Magazine, April 2011:29-32.
Copyright (C) 2008,2009,2010 Ole Tange http://ole.tange.dk
Copyright (C) 2010,2011 Ole Tange, http://ole.tange.dk and Free
Software Foundation, Inc.
Copyright (C) 2016 Ole Tange, http://ole.tange.dk and Free Software
Foundation, Inc.
=head1 LICENSE
@ -203,14 +196,12 @@ sub producer {
}
my $s = IO::Select->new();
my %fhr;
my %buffer;
sub add_file {
my $fd = shift;
open(my $fh, "<&=", $fd) || die;
$s->add($fh);
$fhr{$fh}++;
# Tell the producer now opened here and can be released
$okq->enqueue($fd);
# Initialize the buffer
@ -236,9 +227,9 @@ my (@ready,$file,$rv,$buf);
do {
# Wait until at least one file is opened
add_files_block();
while($q->pending or keys %fhr) {
while($q->pending or keys %buffer) {
add_files();
while(keys %fhr) {
while(keys %buffer) {
@ready = $s->can_read(0.01);
if(not @ready) {
add_files();
@ -252,7 +243,6 @@ do {
} else {
# This file is done
$s->remove($file);
delete $fhr{$file};
print @{$buffer{$file}};
delete $buffer{$file};
# Closing the $file causes it to block

3
testsuite/Makefile
View file

@ -58,7 +58,7 @@ startvm:
# If they are already running: Don't fail
VBoxManage startvm CentOS3-root:centos3 || VBoxManage controlvm CentOS3-root:centos3 resume || true
VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true
parallel ping -c 3 ::: redhat9.tange.dk centos3.tange.dk oracle11.tange.dk
parallel 'ping {} | read' ::: redhat9.tange.dk centos3.tange.dk
stopvm:
# || true - because this should not fail if the VM is not running
@ -73,6 +73,7 @@ startdb:
#echo shutdown abort | sudo su - oracle -c "sqlplus / as sysdba"
sudo parallel /etc/init.d/{} status '||' /etc/init.d/{} restart ::: postgresql mysql # oracle-xe
VBoxManage startvm OracleXE || true
parallel 'ping {} | read' ::: oracle11.tange.dk
clean:
rm -rf input-files/random_dirs_no_newline

2
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -643,7 +643,7 @@ echo '**'
echo '### bug #48658: --linebuffer --files'
parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
echo '**'

12
testsuite/tests-to-run/parallel-local-100s.sh
View file

@ -33,23 +33,21 @@ measure() {
# Input:
# $1 = iterations
# $2 = sleep 1 sec for every $2
seq $1 |
stdout time -v parallel -u sleep '{= $_=$_%'$2'?0:1 =}' |
grep Maximum | field 6;
seq $1 | ramusage parallel -u sleep '{= $_=$_%'$2'?0:1 =}'
}
export -f measure
no_mem_leak() {
# Return false if leaking
max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 |
max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 1000 1000 1000 1000 |
sort -n | tail -n 1)
min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 |
min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 |
sort -n | head -n 1)
if [ $max1000 -gt $min30000 ] ; then
# Make sure there are a few sleeps
max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 |
max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 1000 1000 1000 1000 |
sort -n | tail -n 1)
min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 |
min30000=$(parallel measure {} 100 ::: 30000 30000 30000 |
sort -n | head -n 1)
if [ $max1000 -gt $min30000 ] ; then
echo $max1000 -gt $min30000 = no leak

14
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -82,6 +82,20 @@ echo 'bug #41613: --compress --line-buffer no newline';
perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
echo 'bug #48658: --linebuffer --files';
doit() { parallel --files --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; };
export -f doit;
parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
doit() { parallel --results /tmp/par48658$1 --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; rm -rf "/tmp/par48658$1"; };
export -f doit;
parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
doit() { parallel --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; };
export -f doit;
parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
echo '**'
echo "### Test -I";

48
testsuite/wanted-results/parallel-install
View file

@ -119,18 +119,32 @@ Warning: pod2html not found. Using old parallel_tutorial.html
rm -f ./pod2htm*
pod2texi --output=./parallel.texi ./parallel.pod \
|| echo "Warning: pod2texi not found. Using old parallel.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel.texi
pod2texi --output=./env_parallel.texi ./env_parallel.pod \
|| echo "Warning: pod2texi not found. Using old env_parallel.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old env_parallel.texi
pod2texi --output=./sem.texi ./sem.pod \
|| echo "Warning: pod2texi not found. Using old sem.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old sem.texi
pod2texi --output=./sql.texi ./sql \
|| echo "Warning: pod2texi not found. Using old sql.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old sql.texi
pod2texi --output=./niceload.texi ./niceload.pod \
|| echo "Warning: pod2texi not found. Using old niceload.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old niceload.texi
pod2texi --output=./parallel_tutorial.texi ./parallel_tutorial.pod \
|| echo "Warning: pod2texi not found. Using old parallel_tutorial.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel_tutorial.texi
pod2texi --output=./parallel_design.texi ./parallel_design.pod \
|| echo "Warning: pod2texi not found. Using old parallel_design.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel_design.texi
pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
|| echo "Warning: pod2pdf not found. Using old parallel.pdf"
/bin/bash: pod2pdf: command not found
@ -161,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
Warning: pod2pdf not found. Using old parallel_design.pdf
make[0]: Entering directory '/tmp/parallel-00000000/src'
/bin/mkdir -p '/usr/local/bin'
/usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin'
/usr/bin/install -c parallel sql niceload parcat env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin'
make install-exec-hook
make[0]: Entering directory '/tmp/parallel-00000000/src'
rm /usr/local/bin/sem || true
@ -209,6 +223,34 @@ pod2html --title "GNU Parallel tutorial" ./parallel_tutorial.pod > ./parallel_tu
/bin/bash: pod2html: command not found
Warning: pod2html not found. Using old parallel_tutorial.html
rm -f ./pod2htm*
pod2texi --output=./parallel.texi ./parallel.pod \
|| echo "Warning: pod2texi not found. Using old parallel.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel.texi
pod2texi --output=./env_parallel.texi ./env_parallel.pod \
|| echo "Warning: pod2texi not found. Using old env_parallel.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old env_parallel.texi
pod2texi --output=./sem.texi ./sem.pod \
|| echo "Warning: pod2texi not found. Using old sem.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old sem.texi
pod2texi --output=./sql.texi ./sql \
|| echo "Warning: pod2texi not found. Using old sql.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old sql.texi
pod2texi --output=./niceload.texi ./niceload.pod \
|| echo "Warning: pod2texi not found. Using old niceload.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old niceload.texi
pod2texi --output=./parallel_tutorial.texi ./parallel_tutorial.pod \
|| echo "Warning: pod2texi not found. Using old parallel_tutorial.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel_tutorial.texi
pod2texi --output=./parallel_design.texi ./parallel_design.pod \
|| echo "Warning: pod2texi not found. Using old parallel_design.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel_design.texi
pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
|| echo "Warning: pod2pdf not found. Using old parallel.pdf"
/bin/bash: pod2pdf: command not found
@ -238,7 +280,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
/bin/bash: pod2pdf: command not found
Warning: pod2pdf not found. Using old parallel_design.pdf
/bin/mkdir -p '/usr/local/share/doc/parallel'
/usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf '/usr/local/share/doc/parallel'
/usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parcat.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parcat.pdf '/usr/local/share/doc/parallel'
pod2man --release='00000000' --center='parallel' \
--section=1 ./parallel.pod > ./parallel.1n \
&& mv ./parallel.1n ./parallel.1 \
@ -282,7 +324,7 @@ pod2man --release='00000000' --center='parallel' \
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parallel_design.7
/bin/mkdir -p '/usr/local/share/man/man1'
/usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 '/usr/local/share/man/man1'
/usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 '/usr/local/share/man/man1'
/bin/mkdir -p '/usr/local/share/man/man7'
/usr/bin/install -c -m 644 parallel_tutorial.7 parallel_design.7 '/usr/local/share/man/man7'
make[0]: Leaving directory '/tmp/parallel-00000000/src'

4
testsuite/wanted-results/parallel-local-0.3s
View file

@ -1761,9 +1761,9 @@ echo '**'
**
echo '### bug #48658: --linebuffer --files'
### bug #48658: --linebuffer --files
parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
10
echo '**'
**
### 1 .par file from --files expected
0
10

54
testsuite/wanted-results/parallel-local-1s
View file

@ -74,8 +74,6 @@ parallel: Error: false failed.
(parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1
1
parallel: Error: false failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $?
1
@ -236,6 +234,58 @@ ls: cannot access '/OK-if-missing-file': No such file or directory
echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
bug #41613: --compress --line-buffer no newline
It worked
echo 'bug #48658: --linebuffer --files'; doit() { parallel --files --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; export -f doit; parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
bug #48658: --linebuffer --files
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
doit() { parallel --results /tmp/par48658$1 --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; rm -rf "/tmp/par48658$1"; }; export -f doit; parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
doit() { parallel --linebuffer --compress-program $1 seq ::: 100000 | wc -l ; }; export -f doit; parallel -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
100000
echo '**'
**
echo "### Test -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'

25
testsuite/wanted-results/parallel-tutorial
View file

@ -350,6 +350,31 @@ pre- A-post
pre-A -post
parallel --trim lr echo pre-{}-post ::: ' A '
pre-A-post
parallel echo \={} ::: zsh bash ls
=zsh
=bash
=ls
parallel 'set a="{}"; if( { test -d "$a" } ) echo "$a is a dir"' ::: *
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="abc-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="abc0-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="abc_-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="def-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num1000000"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num128"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num30000"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num8"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num_%header"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="tsv-file.tsv"; if( { test -d "$a" } ) echo "$a is a dir"'
parallel --tag echo foo-{} ::: A B C
A foo-A
B foo-B