diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 71de09ed..5bd1c7bb 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -242,7 +242,7 @@ parallel convert -resize {3} -quality {2} {1} {1.}_{3}_q{2}.jpg :::: - ::: 25 50 # Thank you for watching # # If you like GNU Parallel: -# * Post this video on forums/blogs/Twitter/Facebook/Linkedin +# * Post this video on forums/blogs/Google+/Twitter/Facebook/Linkedin # * Join the mailing list http://lists.gnu.org/mailman/listinfo/parallel # * Request or write a review for your favourite blog or magazine # * Request or build a package for your favourite distribution diff --git a/doc/release_new_version b/doc/release_new_version index b3116fa1..996e8e6f 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -193,10 +193,26 @@ me too'. Debian people: This is the (2 years old) bug to push: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=518696 +Several have suggested splitting moreutils into moreutils and +moreutils-parallel just like FreeBSD and RawHide have already done: +http://www.freshports.org/sysutils/moreutils-parallel/ +http://rpm.pbone.net/index.php3/stat/4/idpl/15740186/dir/rawhide/com/moreutils-parallel-0.44-1.fc16.x86_64.rpm.html + +To me that sounds like a good solution: Most people will never have +any problems and the few that install both GNU Parallel and packages +that depend on moreutils-parallel will be aware of it. + New in this release: -* Neat example of a parallel webcrawler in the man page. +(( timeout )) + +* CPU detection improved for Mac OSX. Thanks to Wayne E. Seguin. + +* Example of a parallel webcrawler in the man page. + +* Backup up PostgreSQL with GNU Parallel. Thanks to Stephane Wirtel. + http://wirtel.be/2011/07/15/rsync_parallel/ * Blog post in Japanese. http://dminor11th.blogspot.com/2011/08/gnu-parallel.html diff --git a/src/parallel b/src/parallel index ac8745a9..5cb2d1c2 100755 --- a/src/parallel +++ b/src/parallel @@ -79,6 +79,7 @@ if($::opt_nonall or $::opt_onall) { close $outfh; push @argfiles, $name; } + if(@::opt_basefile) { setup_basefile(); } # for each sshlogin do: # parallel -S $sshlogin $command :::: @argfiles # @@ -104,6 +105,7 @@ if($::opt_nonall or $::opt_onall) { shell_quote_scalar($command)." :::: @argfiles\n"; } close PARALLEL; + if(@::opt_basefile) { cleanup_basefile(); } unlink(@argfiles); wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); } @@ -480,7 +482,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20110802; + $Global::version = 20110819; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -820,9 +822,7 @@ sub read_args_from_command_line { sub cleanup { # Returns: N/A - if(@::opt_basefile) { - cleanup_basefile(); - } + if(@::opt_basefile) { cleanup_basefile(); } } sub __QUOTING_ARGUMENTS_FOR_SHELL__ {} @@ -934,9 +934,7 @@ sub init_run_jobs { $Global::tty_taken = 0; $SIG{USR1} = \&list_running_jobs; $SIG{USR2} = \&toggle_progress; - if(@::opt_basefile) { - setup_basefile(); - } + if(@::opt_basefile) { setup_basefile(); } } sub start_more_jobs { diff --git a/src/parallel.pod b/src/parallel.pod index f23a4d29..20aa8028 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1531,7 +1531,7 @@ download images for the past 30 days: B -B<$(date -d "today -{1} days" +%Y%m%d)> with give the dates in +B<$(date -d "today -{1} days" +%Y%m%d)> will give the dates in YYYYMMDD with {1} days subtracted. diff --git a/testsuite/tests-to-run/test43.sh b/testsuite/tests-to-run/test43.sh index e58b4530..7688702a 100644 --- a/testsuite/tests-to-run/test43.sh +++ b/testsuite/tests-to-run/test43.sh @@ -5,21 +5,21 @@ mkdir -p /dev/shm/parallel echo '### Test $TMPDIR' TMPDIR=/dev/shm/parallel stdout timeout -k 1 6 parallel head -c 2000m '<'{} >/dev/null ::: /dev/zero & -seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 1" \ +seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 0.1" \ | stdout timeout -k 1 10 perl -ne 'BEGIN{$a=<>} $b=<>; if ($a-1000 > $b) { die "More than 1 MB gone. Good!" }' wait -sleep 1 +sleep 0.1 echo '### Test --tmpdir' stdout timeout -k 1 6 parallel --tmpdir /dev/shm/parallel head -c 2000m '<'{} >/dev/null ::: /dev/zero & -seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 1" \ +seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 0.1" \ | stdout timeout -k 1 10 perl -ne 'BEGIN{$a=<>} $b=<>; if ($a-1000 > $b) { die "More than 1 MB gone. Good!" }' wait -sleep 1 +sleep 0.1 echo '### Test $TMPDIR and --tmpdir' TMPDIR=/tmp stdout timeout -k 1 6 parallel --tmpdir /dev/shm/parallel head -c 2000m '<'{} >/dev/null ::: /dev/zero & -seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 1" \ +seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 0.1" \ | stdout timeout -k 1 10 perl -ne 'BEGIN{$a=<>} $b=<>; if ($a-1000 > $b) { die "More than 1 MB gone. Good!" }' wait -sleep 1 +sleep 0.1 diff --git a/testsuite/tests-to-run/test51.sh b/testsuite/tests-to-run/test51.sh index 41c88513..1a4cb945 100644 --- a/testsuite/tests-to-run/test51.sh +++ b/testsuite/tests-to-run/test51.sh @@ -5,7 +5,21 @@ echo '### Test --pipe' seq 1 1000000 >/tmp/parallel-seq shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j2 -k -L1 +echo '### Test 200M records with too small block'; + ( + echo start; + seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; + echo end; + echo start; + seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; + echo end; + echo start; + seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; + echo end; + ) | stdout parallel -k --block 200M -j2 --pipe --recend 'end\n' wc -c | + egrep -v '^0$' + echo '### Test -N with multiple jobslots and multiple args' seq 1 1 | parallel -j2 -k -N 3 --pipe 'cat;echo a' | uniq seq 1 2 | parallel -j2 -k -N 3 --pipe 'cat;echo bb' | uniq @@ -30,7 +44,7 @@ echo -n 01a02a0a0a12a34a45a6a | echo -n 01a02a0a0a12a34a45a6a | parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' -echo '### Test 100M records with too big block'; +echo '### Test 10M records with too big block'; ( echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; @@ -44,20 +58,6 @@ echo '### Test 100M records with too big block'; ) | stdout parallel -k --block 10M -j2 --pipe --recstart 'start\n' wc -c | egrep -v '^0$' -echo '### Test 300M records with too small block'; - ( - echo start; - seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; - echo end; - echo start; - seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; - echo end; - echo start; - seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; - echo end; - ) | stdout parallel -k --block 200M -j2 --pipe --recend 'end\n' wc -c | - egrep -v '^0$' - echo '### Test --rrs -N1 --recend single'; echo 12a34a45a6 | parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' diff --git a/testsuite/tests-to-run/test60.sh b/testsuite/tests-to-run/test60.sh index 94036f2a..89e10cbe 100644 --- a/testsuite/tests-to-run/test60.sh +++ b/testsuite/tests-to-run/test60.sh @@ -23,4 +23,12 @@ echo '### Test --nonall -u'; echo '### Test read sshloginfile from STDIN'; echo nlv.pi.dk | parallel -S - --nonall hostname; echo nlv.pi.dk | parallel --sshloginfile - --nonall hostname + +echo '### Test --nonall --basefile'; + touch /tmp/nonall--basefile; + parallel --nonall --basefile /tmp/nonall--basefile -S parallel@$SERVER2,$SERVER1 ls /tmp/nonall--basefile + +echo '### Test --onall --basefile'; + touch /tmp/onall--basefile; + parallel --onall --basefile /tmp/onall--basefile -S parallel@$SERVER2,$SERVER1 ls ::: /tmp/onall--basefile EOF diff --git a/testsuite/wanted-results/test38 b/testsuite/wanted-results/test38 index 5430ed10..7db53ceb 100644 --- a/testsuite/wanted-results/test38 +++ b/testsuite/wanted-results/test38 @@ -2,4 +2,3 @@ ### See if we get compile error perl ### See if we read modules outside perllib -read(3, "#!/usr/bin/perl -w\n\n# open3 used"..., 4096) = 4096 diff --git a/testsuite/wanted-results/test51 b/testsuite/wanted-results/test51 index 51ab90c8..fee40012 100644 --- a/testsuite/wanted-results/test51 +++ b/testsuite/wanted-results/test51 @@ -1,4 +1,8 @@ ### Test --pipe +### Test 200M records with too small block +303111434 +303111434 +303111434 ### Test -N with multiple jobslots and multiple args 1 a @@ -117,14 +121,10 @@ d 1>01a02a0a 2>0a12a34a 3>45a6a -### Test 100M records with too big block +### Test 10M records with too big block 6888906 6888906 6888906 -### Test 300M records with too small block -303111434 -303111434 -303111434 ### Test --rrs -N1 --recend single 1>12 2>34 diff --git a/testsuite/wanted-results/test60 b/testsuite/wanted-results/test60 index b4361745..c375a938 100644 --- a/testsuite/wanted-results/test60 +++ b/testsuite/wanted-results/test60 @@ -60,3 +60,9 @@ nlv.pi.dk ### Test read sshloginfile from STDIN nlv.pi.dk nlv.pi.dk +### Test --nonall --basefile +/tmp/nonall--basefile +/tmp/nonall--basefile +### Test --onall --basefile +/tmp/onall--basefile +/tmp/onall--basefile