diff --git a/doc/release_new_version b/doc/release_new_version index b7240da9..08500022 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -199,7 +199,7 @@ to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20180422 ('Trèbes') released <<[stable]>> +Subject: GNU Parallel 20180422 ('Trèbes/Tiangong-1/Winnie Mandela/ASIM/Algeria') released <<[stable]>> GNU Parallel 20180422 ('Trèbes') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ @@ -212,6 +212,13 @@ Quote of the month: New in this release: +Launch at DIKU + +https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently + +http://fliplinux.com/gnu-parallel-6.html + +https://www.youtube.com/watch?v=sHpTywpb4_4 <> diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 93ee9c6c..f6228edd 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -51,11 +51,11 @@ env_parallel() { } _names_of_FUNCTIONS() { # myfunc is a function - type `_names_of_maybe_FUNCTIONS` | + LANG=C type `_names_of_maybe_FUNCTIONS` | perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' } _bodies_of_FUNCTIONS() { - type "$@" | perl -ne '/^(\S+) is a function$/ or print' + LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print' } _names_of_VARIABLES() { # This may screw up if variables contain \n and = @@ -131,7 +131,7 @@ env_parallel() { # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # parallel is a tracked alias for /usr/local/bin/parallel (ksh) # Return 0 if found, 1 otherwise - type "$@" | + LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || s/ is aliased to .*// || s/ is a function// || diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 46f6f536..d67bfd94 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -152,7 +152,7 @@ env_parallel() { # which is hashed (/usr/bin/which) # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # Return 0 if found, 1 otherwise - type "$@" | + LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || s/ is aliased to .*// || s/ is a function// || diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 45f2a03e..baefb34e 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -51,11 +51,11 @@ env_parallel() { } _names_of_FUNCTIONS() { # myfunc is a function - type `_names_of_maybe_FUNCTIONS` | + LANG=C type `_names_of_maybe_FUNCTIONS` | perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' } _bodies_of_FUNCTIONS() { - type "$@" | perl -ne '/^(\S+) is a function$/ or print' + LANC=C type "$@" | perl -ne '/^(\S+) is a function$/ or print' } _names_of_VARIABLES() { # This may screw up if variables contain \n and = @@ -131,7 +131,7 @@ env_parallel() { # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # parallel is a tracked alias for /usr/local/bin/parallel (ksh) # Return 0 if found, 1 otherwise - type "$@" | + LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || s/ is aliased to .*// || s/ is a function// || diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 84ce6772..533cf460 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -111,7 +111,7 @@ env_parallel() { # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # parallel is a tracked alias for /usr/local/bin/parallel (ksh) # Return 0 if found, 1 otherwise - type "$@" | + LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || s/ is aliased to .*// || s/ is a function// || diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 30a55a4f..3101a2bb 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -51,11 +51,11 @@ env_parallel() { } _names_of_FUNCTIONS() { # myfunc is a function - type `_names_of_maybe_FUNCTIONS` | + LANG=C type `_names_of_maybe_FUNCTIONS` | perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' } _bodies_of_FUNCTIONS() { - type "$@" | perl -ne '/^(\S+) is a function$/ or print' + LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print' } _names_of_VARIABLES() { # This may screw up if variables contain \n and = @@ -131,7 +131,7 @@ env_parallel() { # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # parallel is a tracked alias for /usr/local/bin/parallel (ksh) # Return 0 if found, 1 otherwise - type "$@" | + LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || s/ is aliased to .*// || s/ is a function// || diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 6de78057..63c13ad3 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -105,7 +105,7 @@ env_parallel() { # which is hashed (/usr/bin/which) # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # Return 0 if found, 1 otherwise - type "$@" | + LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || s/ is aliased to .*// || s/ is a function// || diff --git a/src/parallel b/src/parallel index 4eba3f67..37b0931d 100755 --- a/src/parallel +++ b/src/parallel @@ -48,7 +48,6 @@ if($Global::max_number_of_args) { } my @command = @ARGV; - my @input_source_fh; if($opt::pipepart) { if($opt::tee) { @@ -226,10 +225,11 @@ sub pipepart_setup { # $Global::JobQueue if($opt::tee) { # Prepend each command with - # cat file - my $cat_string = "cat ".::shell_quote_scalar($opt::a[0]); + # < file + my $cat_string = "< ".::shell_quote_scalar($opt::a[0]); for(1..$Global::JobQueue->total_jobs()) { - push @Global::cat_prepends, $cat_string; + push @Global::cat_appends, $cat_string; + push @Global::cat_prepends, ""; } } else { if(not $opt::blocksize) { @@ -286,15 +286,13 @@ sub pipe_tee_setup { open STDOUT, ">","/dev/null"; exec "tee",@fifos; } - # Make cats for each fifo - # cat fifo1 | grep 1 - # cat fifo2 | grep 2 - # cat fifo3 | grep 3 - # cat fifo4 | grep 4 - # cat fifo5 | grep 5 + # For each fifo + # (rm fifo1; grep 1) < fifo1 + # (rm fifo2; grep 2) < fifo2 + # (rm fifo3; grep 3) < fifo3 # Remove the tmpfifo as soon as it is open - # 'exec' saves a process - @Global::cat_prepends = map { "(rm $_; exec cat) < $_" } @fifos; + @Global::cat_prepends = map { "(rm $_;" } @fifos; + @Global::cat_appends = map { ") < $_" } @fifos; } sub pipe_part_files { @@ -407,7 +405,10 @@ sub find_split_positions { } } } - push(@pos,$size); + if($pos[$#pos] != $size) { + # Last splitpoint was not at end of the file: add it + push(@pos,$size); + } close $fh; return @pos; } @@ -440,7 +441,7 @@ sub cat_partial { } }); return "<". shell_quote_scalar($file) . - " perl -e '$script' @start_len"; + " perl -e '$script' @start_len |"; } sub spreadstdin { @@ -4097,8 +4098,8 @@ sub usage { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", - " ;login: The USENIX Magazine, February 2011:42-47.", + " O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,", + " DOI https://doi.org/10.5281/zenodo.1146014", "", "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", @@ -4125,12 +4126,15 @@ sub citation_notice { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", - " ;login: The USENIX Magazine, February 2011:42-47.", + " O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,", + " DOI https://doi.org/10.5281/zenodo.1146014", "", "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "", + "More about funding GNU Parallel and the citation notice:", + "https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice", + "", "To silence this citation notice: run 'parallel --citation' once.", "" ); @@ -4216,8 +4220,9 @@ sub citation { " author = {Tange, Ole},", " title = {GNU Parallel 2018},", " publisher = {Ole Tange},", + " month = Apr,", " year = 2018,", - " month = apr,", + " ISBN = {9781387509881},", " doi = {10.5281/zenodo.1146014},", " url = {https://doi.org/10.5281/zenodo.1146014}", "}", @@ -4227,6 +4232,9 @@ sub citation { "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "", + "More about funding GNU Parallel and the citation notice:", + "https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice", + "", "If you send a copy of your published article to tange\@gnu.org, it will be", "mentioned in the release notes of next version of GNU Parallel.", "" @@ -7765,11 +7773,12 @@ sub wrapped { # }' 0 0 0 11 | # # --pipepart --tee: prepend: - # cat dash-a-file | + # < dash-a-file # - # --pipe --tee: prepend: - # cat fifo | - $command = (shift @Global::cat_prepends). " | ($command)"; + # --pipe --tee: wrap: + # (rm fifo; ... ) < fifo + $command = (shift @Global::cat_prepends). "($command)". + (shift @Global::cat_appends); } elsif($opt::pipe) { # Wrap with EOF-detector to avoid starting $command if EOF. $command = empty_input_wrapper($command); diff --git a/src/parallel.pod b/src/parallel.pod index 56d4a0d7..7a7ac9e2 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2485,7 +2485,7 @@ a bc " -> "a bc". This is the default if B<--colsep> is used. =item B<-u> -Ungroup output. Output is printed as soon as possible and by passes +Ungroup output. Output is printed as soon as possible and bypasses GNU B internal processing. This may cause output from different commands to be mixed thus should only be used if you do not care about the output. Compare these: @@ -2917,7 +2917,35 @@ for Blue Marble world map and create a 10240x20480 map. convert -append line{0..19}.jpg world.jpg -=head1 EXAMPLE: Copy files as last modified date (ISO8601) with added random digits +=head1 EXAMPLE: Download Apollo-11 images from NASA using jq + +Search NASA using their API to get JSON for images related to 'apollo +11' and has 'moon landing' in the description. + +The search query returns JSON containing URLs to JSON containing +collections of pictures. One of the pictures in each of these +collection is I. + +B is used to get the JSON for the search query. B is then +used to extract the URLs of the collections. B then calls +B to get each collection, which is passed to B to extract +the URLs of all images. B filters out the I images, and +B finally uses B to fetch the images. + + base="https://images-api.nasa.gov/search" + q="q=apollo 11" + description="description=moon landing" + media_type="media_type=image" + wget -O - "$base?$q&$description&$media_type" | + jq -r .collection.items[].href | + parallel wget -O - | + jq -r .[] | + grep large | + parallel wget + + +=head1 EXAMPLE: Copy files as last modified date (ISO8601) with added +random digits find . | parallel cp {} '../destdir/{= $a=int(10000*rand); $_=pQ($_); $_=`date -r "$_" +%FT%T"$a"`; chomp; =}' @@ -3205,7 +3233,8 @@ significant. Compare the speed of these: If your program cannot take multiple arguments, then you can use GNU B to spawn multiple GNU Bs: - seq -w 0 999999 | parallel -j10 --pipe parallel -j0 touch pict{}.jpg + seq -w 0 9999999 | + parallel -j10 -q -I,, --pipe parallel -j0 touch pict{}.jpg If B<-j0> normally spawns 252 jobs, then the above will try to spawn 2520 jobs. On a normal GNU/Linux system you can spawn 32000 jobs using diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index a18e7d77..0036626b 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -1294,8 +1294,9 @@ While you I use multi-lined commands in GNU B, to improve readibilty GNU B discourages the use of multi-line commands. In most cases it can be written as a function: - seq 1 3 | parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \ - echo finish {}' + seq 1 3 | + parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \ + echo finish {}' Could be written as: @@ -1311,7 +1312,7 @@ The failed commands can be resumed with: seq 1 3 | parallel --resume-failed --joblog my.log 'sleep {}; echo {};\ - echo finish {}' + echo finish {}' =back diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 4dedb5b0..102ad527 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -5,11 +5,20 @@ This tutorial shows off much of GNU B's functionality. The tutorial is meant to learn the options in and syntax of GNU B. The tutorial is B to show realistic examples from the -real world. For realistic examples see B in the EXAMPLE -section. +real world. + +=head2 Reader's guide + +Start by watching the intro videos for a quick introduction: +http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 + +Then look at the Bs after the list of B in B (Use B). That will give you +an idea of what GNU B is capable of. + +Then spend a couple of hours walking through this tutorial (B). Your command line will love you for it. -Spend an hour walking through the tutorial. Your command line will -love you for it. =head1 Prerequisites diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 4de13654..1d9b153f 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -315,8 +315,8 @@ par_tcsh_man Academic tradition requires you to cite works you base your article par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a par_tcsh_man scientific publication, please cite: par_tcsh_man -par_tcsh_man O. Tange (2011): GNU Parallel - The Command-Line Power Tool, -par_tcsh_man ;login: The USENIX Magazine, February 2011:42-47. +par_tcsh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, +par_tcsh_man DOI https://doi.org/10.5281/zenodo.1146014 par_tcsh_man par_tcsh_man This helps funding further development; AND IT WON'T COST YOU A CENT. par_tcsh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing. @@ -1019,8 +1019,8 @@ par_csh_man par_csh_man par_csh_man par_csh_man {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} -par_csh_man ;login: The USENIX Magazine, February 2011:42-47. -par_csh_man O. Tange (2011): GNU Parallel - The Command-Line Power Tool, +par_csh_man DOI https://doi.org/10.5281/zenodo.1146014 +par_csh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, par_csh_man ### From man env_parallel par_csh_man --colsep regexp Split input on regexp for positional replacements par_csh_man --nonall Run the given command with no arguments on all sshlogins diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index d977cec0..2a55cc3d 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -4,12 +4,15 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2011): GNU Parallel - The Command-Line Power Tool, - ;login: The USENIX Magazine, February 2011:42-47. + O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, + DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. +More about funding GNU Parallel and the citation notice: +https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + To silence this citation notice: run 'parallel --citation' once. echo a @@ -24,12 +27,15 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2011): GNU Parallel - The Command-Line Power Tool, - ;login: The USENIX Magazine, February 2011:42-47. + O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, + DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. +More about funding GNU Parallel and the citation notice: +https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + To silence this citation notice: run 'parallel --citation' once. 10 files to edit @@ -41,8 +47,9 @@ scientific publication, please cite: author = {Tange, Ole}, title = {GNU Parallel 2018}, publisher = {Ole Tange}, + month = Apr, year = 2018, - month = apr, + ISBN = {9781387509881}, doi = {10.5281/zenodo.1146014}, url = {https://doi.org/10.5281/zenodo.1146014} } @@ -52,6 +59,9 @@ scientific publication, please cite: This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. +More about funding GNU Parallel and the citation notice: +https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + If you send a copy of your published article to tange@gnu.org, it will be mentioned in the release notes of next version of GNU Parallel. diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 10abb751..4d6980f7 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -75,11 +75,11 @@ parallel: Error: --pipepart is incompatible with --max-replace-args, --max-lines echo '### bug #42893: --block should not cause decimals in cat_partial' ### bug #42893: --block should not cause decimals in cat_partial seq 100000 >/tmp/parallel-decimal; parallel --dry-run -kvv --pipepart --block 0.12345M -a /tmp/parallel-decimal true; rm /tmp/parallel-decimal -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 | (true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 | (true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 | (true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 | (true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 | (true) +1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true) +1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true) +1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true) +1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true) +1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true) echo '### bug #42902: profiles containing arguments with space' ### bug #42902: profiles containing arguments with space echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH; parallel -JFULLPATH echo FULLPATH ::: $0 diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 42881895..4d691de7 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -171,8 +171,8 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2011): GNU Parallel - The Command-Line Power Tool, - ;login: The USENIX Magazine, February 2011:42-47. + O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, + DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing.