From 73c7f844d700cac9d67ad13987bb8e381c1e53ad Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 19 Aug 2013 11:15:39 +0200 Subject: [PATCH] testsuite: --shebang-wrap for Octave, Python. Perl, GnuPlot, R, Ruby, Bash. 10seconds_install: Try curl. Makefile: parallel.html dependency. parallel: Printing joblog => print_joblog. parallel.pod: Removed TinyOgg. testsuite: Minor fixups. --- 10seconds_install | 8 +- doc/promo | 6 ++ doc/release_new_version | 6 ++ src/Makefile.in | 4 + src/parallel | 79 ++++++++++++++----- src/parallel.pod | 2 - testsuite/input-files/shebang/shebangwrap.gp | 4 + testsuite/input-files/shebang/shebangwrap.oct | 6 ++ testsuite/input-files/shebang/shebangwrap.pl | 3 + testsuite/input-files/shebang/shebangwrap.py | 5 ++ testsuite/input-files/shebang/shebangwrap.r | 4 + testsuite/input-files/shebang/shebangwrap.rb | 3 + testsuite/input-files/shebang/shebangwrap.sh | 3 + testsuite/tests-to-run/parallel-local1.sh | 4 +- testsuite/tests-to-run/parallel-local22.sh | 2 +- testsuite/tests-to-run/parallel-local23.sh | 11 ++- testsuite/tests-to-run/parallel-remote1.sh | 4 +- testsuite/wanted-results/parallel-local23 | 48 ++++++----- testsuite/wanted-results/parallel-remote1 | 2 + 19 files changed, 150 insertions(+), 54 deletions(-) create mode 100755 testsuite/input-files/shebang/shebangwrap.gp create mode 100755 testsuite/input-files/shebang/shebangwrap.oct create mode 100755 testsuite/input-files/shebang/shebangwrap.pl create mode 100755 testsuite/input-files/shebang/shebangwrap.py create mode 100755 testsuite/input-files/shebang/shebangwrap.r create mode 100755 testsuite/input-files/shebang/shebangwrap.rb create mode 100755 testsuite/input-files/shebang/shebangwrap.sh diff --git a/10seconds_install b/10seconds_install index 7be84658..ab4ceeb5 100644 --- a/10seconds_install +++ b/10seconds_install @@ -18,13 +18,15 @@ TAIL=$(echo | tail -n 1 2>/dev/null && echo tail || (echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail)) # grep on openindiana must be /usr/xpg4/bin/grep GREP=$(echo | grep -vE . 2>/dev/null && echo grep || (echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep)) +# wget is not default on MacOS, so try curl +GET=$(curl -h >/dev/null && echo curl -L || echo wget -qO -) -LATEST=$(wget -qO- http://ftpmirror.gnu.org/parallel | perl -ne '/(parallel-\d{8})/ and print $1."\n"' | sort | $TAIL -n1) +LATEST=$($GET http://ftpmirror.gnu.org/parallel | perl -ne '/(parallel-\d{8})/ and print $1."\n"' | sort | $TAIL -n1) if test \! -d $LATEST/src/; then # Source dir does not exist rm -f $LATEST.tar.bz2 $LATEST.tar.bz2.sig - wget http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2 - wget http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig + $GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2 > $LATEST.tar.bz2 + $GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig > $LATEST.tar.bz2.sig fi # Check signature - in case ftpmirror.gnu.org is compromised diff --git a/doc/promo b/doc/promo index 52be91d0..113ffc00 100644 --- a/doc/promo +++ b/doc/promo @@ -1,3 +1,9 @@ +=head1 top photos + +http://www.flickr.com/photos/dexxus/5499821986/in/photostream/ +https://www.google.com/search?lr=&safe=images&hl=en&tbs=sur:fmc&tbm=isch&q=top+nature+photos&revid=600471240&biw=1024&bih=569 + + =head1 What is GNU Parallel used for Searching for transit planets using data from the Kepler space telescope. diff --git a/doc/release_new_version b/doc/release_new_version index 08825762..c0cc1443 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -230,9 +230,15 @@ New in this release: * Compression of files in parallel using GNU parallel http://codextechnicanum.blogspot.dk/2013/07/compression-of-files-in-parallel-using.html +* Using GNU Parallel to roll-your-own Map Reduce! + http://www.rankfocus.com/?p=1 + * TimeMachineっぽいバックアップスクリプト http://rio.tc/2013/07/timemachine-1.html +* Using GNU Parallel with Amazon + media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf + * Some useful comments on GNU Parallel https://news.ycombinator.com/item?id=6209767 diff --git a/src/Makefile.in b/src/Makefile.in index 39a87819..fbb0190f 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -508,6 +508,10 @@ niceload.1: niceload.pod Makefile pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/niceload.pod > $(srcdir)/niceload.1 +parallel.html: parallel.pod Makefile + pod2html $(srcdir)/parallel.pod > $(srcdir)/parallel.html + rm -f $(srcdir)/pod2htm* + # Depending on parallel.html to avoid stupid pod2html race condition parallel_tutorial.html: parallel_tutorial.pod Makefile parallel.html pod2html $(srcdir)/parallel_tutorial.pod > $(srcdir)/parallel_tutorial.html diff --git a/src/parallel b/src/parallel index 38c23447..70792c1c 100755 --- a/src/parallel +++ b/src/parallel @@ -4477,23 +4477,7 @@ sub print { unlink $self->fd_file_name(1); } - if($Global::joblog) { - my $cmd; - if($Global::verbose <= 1) { - $cmd = $self->replaced(); - } else { - # Verbose level > 1: Print the rsync and stuff - $cmd = $command; - } - print $Global::joblog - join("\t", $self->seq(), $self->sshlogin()->string(), - $self->starttime(), $self->runtime(), - $self->transfersize(), $self->returnsize(), - $self->exitstatus(), $self->exitsignal(), $cmd - ). "\n"; - flush $Global::joblog; - $self->set_job_in_joblog(); - } + if($Global::joblog) { $self->print_joblog() } # Printing is only relevant for grouped output. $Global::grouped or return; @@ -4560,7 +4544,6 @@ sub print { } } - if(defined $self->{'exitstatus'} and length $$partial > 0) { # If the job is dead: print the remaining partial line if($opt::tag or defined $opt::tagstring) { @@ -4611,6 +4594,25 @@ sub print { } } +sub print_joblog { + my $self = shift; + my $cmd; + if($Global::verbose <= 1) { + $cmd = $self->replaced(); + } else { + # Verbose level > 1: Print the rsync and stuff + $cmd = $command; + } + print $Global::joblog + join("\t", $self->seq(), $self->sshlogin()->string(), + $self->starttime(), $self->runtime(), + $self->transfersize(), $self->returnsize(), + $self->exitstatus(), $self->exitsignal(), $cmd + ). "\n"; + flush $Global::joblog; + $self->set_job_in_joblog(); +} + sub tag { my $self = shift; if(not defined $self->{'tag'}) { @@ -4726,9 +4728,10 @@ sub seq { sub populate { # Add arguments from arg_queue until the number of arguments or # max line length is reached + # Returns: N/A my $self = shift; if($opt::pipe) { - # Do no read any args + # --pipe => Do no read any args $self->push([Arg->new("")]); return; } @@ -4788,6 +4791,7 @@ sub populate { sub push { # Add one or more records as arguments + # Returns: N/A my $self = shift; my $record = shift; push @{$self->{'arg_list'}}, $record; @@ -4818,6 +4822,8 @@ sub push { sub pop { # Remove last argument + # Returns: + # the last record my $self = shift; my $record = pop @{$self->{'arg_list'}}; for my $arg (@$record) { @@ -4832,7 +4838,9 @@ sub pop { } sub pop_all { - # Remove all arguments + # Remove all arguments and zeros the length of replacement strings + # Returns: + # all records my $self = shift; my @popped = @{$self->{'arg_list'}}; for my $replacement_string (keys %{$self->{'replacecount'}}) { @@ -4843,8 +4851,37 @@ sub pop_all { } sub number_of_args { + # The number of records + # Returns: + # number of records my $self = shift; - # This is really number of records + # Ftq rudef oaawuq ime dqxqmeqp az 2011-01-24 mzp ime iaz nk MQhmd + # Mdzrvadp Nvmdymeaz az 2011-04-10. Ftue oaawuq dqxqmeqp az + # 2013-08-18 ue m nuf tmdpqd me kag tmhq fa geq daf14. Bxqmeq + # qymux oaawuq@fmzsq.pw itqz kag dqmp ftue. + # + # U my ftq ymuzfmuzqd ar m buqoq ar rdqq earfimdq omxxqp SZG + # Bmdmxxqx. Rdqq earfimdq sgmdmzfqqe kag mooqee fa ftq eagdoq + # oapq, ngf U tmhq nqqz iazpqduzs tai ymzk mofgmxxk _dqmp_ ftq + # eagdoq oapq. + # + # Fa fqef ftue U bgf uz m oayyqzf fqxxuzs bqabxq fa qymux yq itqz + # ftqk dqmp ftue. Ftq oayyqzf ime bgf uz m eqofuaz ar ftq oapq + # ftmf za azq iagxp xaaw fa ruj ad uybdahq ftq earfimdq - ea ftq + # eagdoq oapq qcguhmxqzf fa m pgefk oadzqd. Fa ymwq egdq ftq + # oayyqzf iagxp zaf etai gb ur eayq azq vgef sdqbbqp ftdagst ftq + # eagdoq oapq U daf13'qp ftq eagdoq oapq + # tffb://qz.iuwubqpum.ads/iuwu/DAF13 + # + # 2.5 yazfte xmfqd U dqoquhqp mz qymux rday eayqazq ita zaf azxk + # ymzmsqp fa ruzp ftq oayyqzf, ngf mxea ymzmsqp fa sgqee ftq oapq + # tmp fa nq daf13'qp. + # + # Ftue nduzse yq fa ftq oazoxgeuaz ftmf ftqdq _mdq_ bqabxq, ita + # mdq zaf mrruxumfqp iuft ftq bdavqof, ftmf iuxx dqmp ftq eagdoq + # oapq - ftagst uf ymk zaf tmbbqz hqdk arfqz. + # + # This is really the number of records return $#{$self->{'arg_list'}}+1; } diff --git a/src/parallel.pod b/src/parallel.pod index 11f04395..7d3cdc23 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -49,8 +49,6 @@ after the list of options. That will give you an idea of what GNU B is capable of. You can also watch the intro video for a quick introduction: -http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and -http://tinyogg.com/watch/YQuXd/ or http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 =head1 OPTIONS diff --git a/testsuite/input-files/shebang/shebangwrap.gp b/testsuite/input-files/shebang/shebangwrap.gp new file mode 100755 index 00000000..84d56b0b --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.gp @@ -0,0 +1,4 @@ +#!/usr/bin/parallel --shebang-wrap -k A={} /usr/bin/gnuplot + +name=system("echo $A") +print name diff --git a/testsuite/input-files/shebang/shebangwrap.oct b/testsuite/input-files/shebang/shebangwrap.oct new file mode 100755 index 00000000..e45bbfd2 --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.oct @@ -0,0 +1,6 @@ +#!/usr/bin/parallel --shebang-wrap -k /usr/bin/octave -qf + +arg_list = argv (); +filename = arg_list{1}; +printf(filename); +printf("\n"); diff --git a/testsuite/input-files/shebang/shebangwrap.pl b/testsuite/input-files/shebang/shebangwrap.pl new file mode 100755 index 00000000..5970265e --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.pl @@ -0,0 +1,3 @@ +#!/usr/bin/parallel --shebang-wrap -k /usr/bin/perl + +print @ARGV,"\n"; diff --git a/testsuite/input-files/shebang/shebangwrap.py b/testsuite/input-files/shebang/shebangwrap.py new file mode 100755 index 00000000..07dbeeb6 --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.py @@ -0,0 +1,5 @@ +#!/usr/bin/parallel --shebang-wrap -k /usr/bin/python + +import sys + +print str(sys.argv[1]) diff --git a/testsuite/input-files/shebang/shebangwrap.r b/testsuite/input-files/shebang/shebangwrap.r new file mode 100755 index 00000000..6b47943e --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.r @@ -0,0 +1,4 @@ +#!/usr/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave + +options <- commandArgs(trailingOnly = TRUE) +options diff --git a/testsuite/input-files/shebang/shebangwrap.rb b/testsuite/input-files/shebang/shebangwrap.rb new file mode 100755 index 00000000..77397c5b --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.rb @@ -0,0 +1,3 @@ +#!/usr/bin/parallel --shebang-wrap -k /usr/bin/ruby + +p ARGV diff --git a/testsuite/input-files/shebang/shebangwrap.sh b/testsuite/input-files/shebang/shebangwrap.sh new file mode 100755 index 00000000..df7bdc31 --- /dev/null +++ b/testsuite/input-files/shebang/shebangwrap.sh @@ -0,0 +1,3 @@ +#!/usr/bin/parallel --shebang-wrap -k /bin/sh + +echo "$@" diff --git a/testsuite/tests-to-run/parallel-local1.sh b/testsuite/tests-to-run/parallel-local1.sh index 1e8d41e9..743ddc20 100644 --- a/testsuite/tests-to-run/parallel-local1.sh +++ b/testsuite/tests-to-run/parallel-local1.sh @@ -155,10 +155,10 @@ echo '### Bug introduce by fixing bug #37042' parallel --xapply -a <(printf 'abc') --colsep '\t' echo {1} echo "### Test --header with -N" - (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '\n.*\n' echo Start\;cat \; echo Stop + (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '.*\n.*\n' echo Start\;cat \; echo Stop echo "### Test --header with --block 1k" - (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '\n.*\n' echo Start\;cat \; echo Stop + (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '.*\n.*\n' echo Start\;cat \; echo Stop echo "### Test --header with multiple :::" parallel --header : echo {a} {b} {1} {2} ::: b b1 ::: a a2 diff --git a/testsuite/tests-to-run/parallel-local22.sh b/testsuite/tests-to-run/parallel-local22.sh index 5e813327..7a952bb1 100755 --- a/testsuite/tests-to-run/parallel-local22.sh +++ b/testsuite/tests-to-run/parallel-local22.sh @@ -39,6 +39,6 @@ echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop (I echo a | parallel -qX echo "'{}'" echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL'; - seq 1 2 | SHELL=tcsh stdout parallel -k --nice 8 setenv a b\;echo \$SHELL + seq 1 2 | SHELL=tcsh MANPATH=. stdout parallel -k --nice 8 setenv a b\;echo \$SHELL EOF diff --git a/testsuite/tests-to-run/parallel-local23.sh b/testsuite/tests-to-run/parallel-local23.sh index a2563f81..ddc4bb4f 100755 --- a/testsuite/tests-to-run/parallel-local23.sh +++ b/testsuite/tests-to-run/parallel-local23.sh @@ -6,7 +6,7 @@ cp -a input-files/testdir2 tmp cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1 echo '### Test filenames containing UTF-8'; cd tmp; - find . -name '*.jpg' | nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; + find . -name '*.jpg' | nice nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; find |grep -v CVS | sort; echo '### bug #39554: Feature request: line buffered output'; @@ -24,7 +24,7 @@ echo '### --version must have higher priority than retired options' parallel --version -g -Y -U -W -T | tail echo '### bug #39787: --xargs broken' - perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo | perl -ne 'print length $_,"\n"' + nice perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo | perl -ne 'print length $_,"\n"' echo '### --delay should grow by 2 sec per arg' stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 | perl -ne '$_ >= 2 and $_ <= 5 and print "OK\n"' @@ -40,10 +40,13 @@ echo '### --header num' (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat echo '### --header regexp --round-robin' - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB{#}\;cat + (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort echo '### --header num --round-robin' - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;cat + (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort + +echo '### shebang-wrap' + nice nice parallel -k {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] EOF diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index eb77813a..c9f3aca9 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -12,9 +12,11 @@ chmod 755 /tmp/myssh1 /tmp/myssh2 seq 1 100 | parallel --sshlogin "/tmp/myssh1 $SSHLOGIN1, /tmp/myssh2 $SSHLOGIN2" \ -j10000% -k echo +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/ | parallel -j2 -k -L1 echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname echo '### test --workdir . in $HOME' - cd && mkdir parallel-test && cd parallel-test && + cd && mkdir -p parallel-test && cd parallel-test && echo OK > testfile && parallel --workdir . --transfer -S $SSHLOGIN1 cat {} ::: testfile +EOF diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index cee7dc2f..2f55d76e 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -83,26 +83,34 @@ JOB3 %head2 5 ### --header regexp --round-robin -JOB1 -%head1 -%head2 -3 -4 -JOB2 -%head1 -%head2 -1 -2 -5 + 4 4 18 + 5 5 20 +JOB +JOB ### --header num --round-robin + 4 4 18 + 5 5 20 JOB1 -%head1 -%head2 -1 -2 -5 JOB2 -%head1 -%head2 -3 -4 +### shebang-wrap +A +B +C +A +B +C +A +B +C +[1] "A" +[1] "B" +[1] "C" +["A"] +["B"] +["C"] +A +B +C +A +B +C diff --git a/testsuite/wanted-results/parallel-remote1 b/testsuite/wanted-results/parallel-remote1 index 8f93cac9..bc50016c 100644 --- a/testsuite/wanted-results/parallel-remote1 +++ b/testsuite/wanted-results/parallel-remote1 @@ -101,3 +101,5 @@ 100 ### --filter-hosts - OK, non-such-user, connection refused, wrong host hk +### test --workdir . in $HOME +OK