env_parallel.*sh: Parsing 'type' depends on $LANG. Set LANG=C.

--tee: No longer spawns cat, but uses redirection instead.
This commit is contained in:
Ole Tange 2018-04-20 23:29:44 +02:00
parent b60a203bd8
commit 8b050b68d4
15 changed files with 127 additions and 62 deletions

View file

@ -199,7 +199,7 @@ to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20180422 ('Trèbes') released <<[stable]>>
Subject: GNU Parallel 20180422 ('Trèbes/Tiangong-1/Winnie Mandela/ASIM/Algeria') released <<[stable]>>
GNU Parallel 20180422 ('Trèbes') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
@ -212,6 +212,13 @@ Quote of the month:
New in this release:
Launch at DIKU
https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently
http://fliplinux.com/gnu-parallel-6.html
https://www.youtube.com/watch?v=sHpTywpb4_4
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>

View file

@ -51,11 +51,11 @@ env_parallel() {
}
_names_of_FUNCTIONS() {
# myfunc is a function
type `_names_of_maybe_FUNCTIONS` |
LANG=C type `_names_of_maybe_FUNCTIONS` |
perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
}
_bodies_of_FUNCTIONS() {
type "$@" | perl -ne '/^(\S+) is a function$/ or print'
LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
}
_names_of_VARIABLES() {
# This may screw up if variables contain \n and =
@ -131,7 +131,7 @@ env_parallel() {
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# parallel is a tracked alias for /usr/local/bin/parallel (ksh)
# Return 0 if found, 1 otherwise
type "$@" |
LANG=C type "$@" |
perl -pe '$exit += (s/ is an alias for .*// ||
s/ is aliased to .*// ||
s/ is a function// ||

View file

@ -152,7 +152,7 @@ env_parallel() {
# which is hashed (/usr/bin/which)
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# Return 0 if found, 1 otherwise
type "$@" |
LANG=C type "$@" |
perl -pe '$exit += (s/ is an alias for .*// ||
s/ is aliased to .*// ||
s/ is a function// ||

View file

@ -51,11 +51,11 @@ env_parallel() {
}
_names_of_FUNCTIONS() {
# myfunc is a function
type `_names_of_maybe_FUNCTIONS` |
LANG=C type `_names_of_maybe_FUNCTIONS` |
perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
}
_bodies_of_FUNCTIONS() {
type "$@" | perl -ne '/^(\S+) is a function$/ or print'
LANC=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
}
_names_of_VARIABLES() {
# This may screw up if variables contain \n and =
@ -131,7 +131,7 @@ env_parallel() {
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# parallel is a tracked alias for /usr/local/bin/parallel (ksh)
# Return 0 if found, 1 otherwise
type "$@" |
LANG=C type "$@" |
perl -pe '$exit += (s/ is an alias for .*// ||
s/ is aliased to .*// ||
s/ is a function// ||

View file

@ -111,7 +111,7 @@ env_parallel() {
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# parallel is a tracked alias for /usr/local/bin/parallel (ksh)
# Return 0 if found, 1 otherwise
type "$@" |
LANG=C type "$@" |
perl -pe '$exit += (s/ is an alias for .*// ||
s/ is aliased to .*// ||
s/ is a function// ||

View file

@ -51,11 +51,11 @@ env_parallel() {
}
_names_of_FUNCTIONS() {
# myfunc is a function
type `_names_of_maybe_FUNCTIONS` |
LANG=C type `_names_of_maybe_FUNCTIONS` |
perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
}
_bodies_of_FUNCTIONS() {
type "$@" | perl -ne '/^(\S+) is a function$/ or print'
LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
}
_names_of_VARIABLES() {
# This may screw up if variables contain \n and =
@ -131,7 +131,7 @@ env_parallel() {
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# parallel is a tracked alias for /usr/local/bin/parallel (ksh)
# Return 0 if found, 1 otherwise
type "$@" |
LANG=C type "$@" |
perl -pe '$exit += (s/ is an alias for .*// ||
s/ is aliased to .*// ||
s/ is a function// ||

View file

@ -105,7 +105,7 @@ env_parallel() {
# which is hashed (/usr/bin/which)
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# Return 0 if found, 1 otherwise
type "$@" |
LANG=C type "$@" |
perl -pe '$exit += (s/ is an alias for .*// ||
s/ is aliased to .*// ||
s/ is a function// ||

View file

@ -48,7 +48,6 @@ if($Global::max_number_of_args) {
}
my @command = @ARGV;
my @input_source_fh;
if($opt::pipepart) {
if($opt::tee) {
@ -226,10 +225,11 @@ sub pipepart_setup {
# $Global::JobQueue
if($opt::tee) {
# Prepend each command with
# cat file
my $cat_string = "cat ".::shell_quote_scalar($opt::a[0]);
# < file
my $cat_string = "< ".::shell_quote_scalar($opt::a[0]);
for(1..$Global::JobQueue->total_jobs()) {
push @Global::cat_prepends, $cat_string;
push @Global::cat_appends, $cat_string;
push @Global::cat_prepends, "";
}
} else {
if(not $opt::blocksize) {
@ -286,15 +286,13 @@ sub pipe_tee_setup {
open STDOUT, ">","/dev/null";
exec "tee",@fifos;
}
# Make cats for each fifo
# cat fifo1 | grep 1
# cat fifo2 | grep 2
# cat fifo3 | grep 3
# cat fifo4 | grep 4
# cat fifo5 | grep 5
# For each fifo
# (rm fifo1; grep 1) < fifo1
# (rm fifo2; grep 2) < fifo2
# (rm fifo3; grep 3) < fifo3
# Remove the tmpfifo as soon as it is open
# 'exec' saves a process
@Global::cat_prepends = map { "(rm $_; exec cat) < $_" } @fifos;
@Global::cat_prepends = map { "(rm $_;" } @fifos;
@Global::cat_appends = map { ") < $_" } @fifos;
}
sub pipe_part_files {
@ -407,7 +405,10 @@ sub find_split_positions {
}
}
}
push(@pos,$size);
if($pos[$#pos] != $size) {
# Last splitpoint was not at end of the file: add it
push(@pos,$size);
}
close $fh;
return @pos;
}
@ -440,7 +441,7 @@ sub cat_partial {
}
});
return "<". shell_quote_scalar($file) .
" perl -e '$script' @start_len";
" perl -e '$script' @start_len |";
}
sub spreadstdin {
@ -4097,8 +4098,8 @@ sub usage {
"If you use programs that use GNU Parallel to process data for an article in a",
"scientific publication, please cite:",
"",
" O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
" ;login: The USENIX Magazine, February 2011:42-47.",
" O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,",
" DOI https://doi.org/10.5281/zenodo.1146014",
"",
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
@ -4125,12 +4126,15 @@ sub citation_notice {
"If you use programs that use GNU Parallel to process data for an article in a",
"scientific publication, please cite:",
"",
" O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
" ;login: The USENIX Magazine, February 2011:42-47.",
" O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,",
" DOI https://doi.org/10.5281/zenodo.1146014",
"",
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
"",
"More about funding GNU Parallel and the citation notice:",
"https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice",
"",
"To silence this citation notice: run 'parallel --citation' once.",
""
);
@ -4216,8 +4220,9 @@ sub citation {
" author = {Tange, Ole},",
" title = {GNU Parallel 2018},",
" publisher = {Ole Tange},",
" month = Apr,",
" year = 2018,",
" month = apr,",
" ISBN = {9781387509881},",
" doi = {10.5281/zenodo.1146014},",
" url = {https://doi.org/10.5281/zenodo.1146014}",
"}",
@ -4227,6 +4232,9 @@ sub citation {
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
"",
"More about funding GNU Parallel and the citation notice:",
"https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice",
"",
"If you send a copy of your published article to tange\@gnu.org, it will be",
"mentioned in the release notes of next version of GNU Parallel.",
""
@ -7765,11 +7773,12 @@ sub wrapped {
# }' 0 0 0 11 |
#
# --pipepart --tee: prepend:
# cat dash-a-file |
# < dash-a-file
#
# --pipe --tee: prepend:
# cat fifo |
$command = (shift @Global::cat_prepends). " | ($command)";
# --pipe --tee: wrap:
# (rm fifo; ... ) < fifo
$command = (shift @Global::cat_prepends). "($command)".
(shift @Global::cat_appends);
} elsif($opt::pipe) {
# Wrap with EOF-detector to avoid starting $command if EOF.
$command = empty_input_wrapper($command);

View file

@ -2485,7 +2485,7 @@ a bc " -> "a bc". This is the default if B<--colsep> is used.
=item B<-u>
Ungroup output. Output is printed as soon as possible and by passes
Ungroup output. Output is printed as soon as possible and bypasses
GNU B<parallel> internal processing. This may cause output from
different commands to be mixed thus should only be used if you do not
care about the output. Compare these:
@ -2917,7 +2917,35 @@ for Blue Marble world map and create a 10240x20480 map.
convert -append line{0..19}.jpg world.jpg
=head1 EXAMPLE: Copy files as last modified date (ISO8601) with added random digits
=head1 EXAMPLE: Download Apollo-11 images from NASA using jq
Search NASA using their API to get JSON for images related to 'apollo
11' and has 'moon landing' in the description.
The search query returns JSON containing URLs to JSON containing
collections of pictures. One of the pictures in each of these
collection is I<large>.
B<wget> is used to get the JSON for the search query. B<jq> is then
used to extract the URLs of the collections. B<parallel> then calls
B<wget> to get each collection, which is passed to B<jq> to extract
the URLs of all images. B<grep> filters out the I<large> images, and
B<parallel> finally uses B<wget> to fetch the images.
base="https://images-api.nasa.gov/search"
q="q=apollo 11"
description="description=moon landing"
media_type="media_type=image"
wget -O - "$base?$q&$description&$media_type" |
jq -r .collection.items[].href |
parallel wget -O - |
jq -r .[] |
grep large |
parallel wget
=head1 EXAMPLE: Copy files as last modified date (ISO8601) with added
random digits
find . | parallel cp {} '../destdir/{= $a=int(10000*rand); $_=pQ($_);
$_=`date -r "$_" +%FT%T"$a"`; chomp; =}'
@ -3205,7 +3233,8 @@ significant. Compare the speed of these:
If your program cannot take multiple arguments, then you can use GNU
B<parallel> to spawn multiple GNU B<parallel>s:
seq -w 0 999999 | parallel -j10 --pipe parallel -j0 touch pict{}.jpg
seq -w 0 9999999 |
parallel -j10 -q -I,, --pipe parallel -j0 touch pict{}.jpg
If B<-j0> normally spawns 252 jobs, then the above will try to spawn
2520 jobs. On a normal GNU/Linux system you can spawn 32000 jobs using

View file

@ -1294,8 +1294,9 @@ While you I<can> use multi-lined commands in GNU B<parallel>, to
improve readibilty GNU B<parallel> discourages the use of multi-line
commands. In most cases it can be written as a function:
seq 1 3 | parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \
echo finish {}'
seq 1 3 |
parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \
echo finish {}'
Could be written as:
@ -1311,7 +1312,7 @@ The failed commands can be resumed with:
seq 1 3 |
parallel --resume-failed --joblog my.log 'sleep {}; echo {};\
echo finish {}'
echo finish {}'
=back

View file

@ -5,11 +5,20 @@
This tutorial shows off much of GNU B<parallel>'s functionality. The
tutorial is meant to learn the options in and syntax of GNU
B<parallel>. The tutorial is B<not> to show realistic examples from the
real world. For realistic examples see B<man parallel> in the EXAMPLE
section.
real world.
=head2 Reader's guide
Start by watching the intro videos for a quick introduction:
http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1
Then look at the B<EXAMPLE>s after the list of B<OPTIONS> in B<man
parallel> (Use B<LESS=+/EXAMPLE\: man parallel>). That will give you
an idea of what GNU B<parallel> is capable of.
Then spend a couple of hours walking through this tutorial (B<man
parallel_tutorial>). Your command line will love you for it.
Spend an hour walking through the tutorial. Your command line will
love you for it.
=head1 Prerequisites

View file

@ -315,8 +315,8 @@ par_tcsh_man Academic tradition requires you to cite works you base your article
par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a
par_tcsh_man scientific publication, please cite:
par_tcsh_man
par_tcsh_man O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
par_tcsh_man ;login: The USENIX Magazine, February 2011:42-47.
par_tcsh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
par_tcsh_man DOI https://doi.org/10.5281/zenodo.1146014
par_tcsh_man
par_tcsh_man This helps funding further development; AND IT WON'T COST YOU A CENT.
par_tcsh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
@ -1019,8 +1019,8 @@ par_csh_man
par_csh_man
par_csh_man
par_csh_man {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
par_csh_man ;login: The USENIX Magazine, February 2011:42-47.
par_csh_man O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
par_csh_man DOI https://doi.org/10.5281/zenodo.1146014
par_csh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
par_csh_man ### From man env_parallel
par_csh_man --colsep regexp Split input on regexp for positional replacements
par_csh_man --nonall Run the given command with no arguments on all sshlogins

View file

@ -4,12 +4,15 @@ Academic tradition requires you to cite works you base your article on.
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
DOI https://doi.org/10.5281/zenodo.1146014
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
More about funding GNU Parallel and the citation notice:
https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
To silence this citation notice: run 'parallel --citation' once.
echo a
@ -24,12 +27,15 @@ Academic tradition requires you to cite works you base your article on.
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
DOI https://doi.org/10.5281/zenodo.1146014
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
More about funding GNU Parallel and the citation notice:
https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
To silence this citation notice: run 'parallel --citation' once.
10 files to edit
@ -41,8 +47,9 @@ scientific publication, please cite:
author = {Tange, Ole},
title = {GNU Parallel 2018},
publisher = {Ole Tange},
month = Apr,
year = 2018,
month = apr,
ISBN = {9781387509881},
doi = {10.5281/zenodo.1146014},
url = {https://doi.org/10.5281/zenodo.1146014}
}
@ -52,6 +59,9 @@ scientific publication, please cite:
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
More about funding GNU Parallel and the citation notice:
https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
If you send a copy of your published article to tange@gnu.org, it will be
mentioned in the release notes of next version of GNU Parallel.

View file

@ -75,11 +75,11 @@ parallel: Error: --pipepart is incompatible with --max-replace-args, --max-lines
echo '### bug #42893: --block should not cause decimals in cat_partial'
### bug #42893: --block should not cause decimals in cat_partial
seq 100000 >/tmp/parallel-decimal; parallel --dry-run -kvv --pipepart --block 0.12345M -a /tmp/parallel-decimal true; rm /tmp/parallel-decimal
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 | (true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 | (true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 | (true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 | (true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 | (true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true)
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true)
echo '### bug #42902: profiles containing arguments with space'
### bug #42902: profiles containing arguments with space
echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH; parallel -JFULLPATH echo FULLPATH ::: $0

View file

@ -171,8 +171,8 @@ Academic tradition requires you to cite works you base your article on.
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
DOI https://doi.org/10.5281/zenodo.1146014
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.