mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
Fixed bug #52740: Bash redirection with process substitution blocks.
This commit is contained in:
Ole Tange
parent
d9bac1136f
commit
d1aa26c1c9
|
|
@ -210,20 +210,6 @@ Haiku of the month:
|
|||
|
||||
New in this release:
|
||||
|
||||
* env_parset for ash, dash, ksh, sh, zsh
|
||||
|
||||
* Automatically create hostgroups if argument ends in @sshlogin
|
||||
|
||||
* Tips for running jobs on your VM https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel
|
||||
|
||||
* Node-local scheduling https://mogonwiki.zdv.uni-mainz.de/dokuwiki/node_local_scheduling
|
||||
|
||||
* GNU Parallel https://docs.computecanada.ca/wiki/GNU_Parallel
|
||||
|
||||
* GNU Parallel のエラー処理 https://qiita.com/inouet/items/bfc208668c86caf8ff74
|
||||
|
||||
* Using GNU Parallel https://wiki.ncsa.illinois.edu/display/ROGER/Using+GNU+Parallel
|
||||
|
||||
|
||||
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20171222;
|
||||
$Global::version = 20171224;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
27
src/parallel
27
src/parallel
|
|
@ -1100,8 +1100,13 @@ sub get_options_from_array {
|
|||
sub parse_options {
|
||||
# Returns: N/A
|
||||
init_globals();
|
||||
my @argv_before = @ARGV;
|
||||
@ARGV = read_options();
|
||||
|
||||
if(defined $opt::citation) {
|
||||
citation(\@argv_before,\@ARGV);
|
||||
wait_and_exit(0);
|
||||
}
|
||||
# no-* overrides *
|
||||
if($opt::nokeeporder) { $opt::keeporder = undef; }
|
||||
|
||||
|
|
@ -1162,7 +1167,6 @@ sub parse_options {
|
|||
print Limits::Command::real_max_length(),"\n"; wait_and_exit(0);
|
||||
}
|
||||
if(defined $opt::version) { version(); wait_and_exit(0); }
|
||||
if(defined $opt::citation) { citation(); wait_and_exit(0); }
|
||||
if(defined $opt::record_env) { record_env(); wait_and_exit(0); }
|
||||
if(defined $opt::show_limits) { show_limits(); }
|
||||
if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
|
||||
|
|
@ -1387,7 +1391,7 @@ sub check_invalid_option_combinations {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20171222;
|
||||
$Global::version = 20171224;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -2750,8 +2754,8 @@ sub drain_job_queue {
|
|||
if($Global::start_sqlworker) {
|
||||
# Start an SQL worker as we are now sure there is work to do
|
||||
$Global::start_sqlworker = 0;
|
||||
if(fork()) {
|
||||
# skip
|
||||
if(my $pid = fork()) {
|
||||
$Global::unkilled_sqlworker = $pid;
|
||||
} else {
|
||||
# Replace --sql/--sqlandworker with --sqlworker
|
||||
my @ARGV = map { s/^--sql(andworker)?$/--sqlworker/; $_ } @Global::options_in_argv;
|
||||
|
|
@ -4010,11 +4014,13 @@ sub wait_and_exit {
|
|||
}
|
||||
for (keys %Global::unkilled_children) {
|
||||
# Kill any (non-jobs) children
|
||||
kill 9, $_;
|
||||
kill 9, $_;
|
||||
waitpid($_,0);
|
||||
delete $Global::unkilled_children{$_};
|
||||
}
|
||||
wait();
|
||||
if($Global::unkilled_sqlworker) {
|
||||
waitpid($Global::unkilled_sqlworker,0);
|
||||
}
|
||||
exit($error);
|
||||
}
|
||||
|
||||
|
|
@ -4160,6 +4166,15 @@ sub version {
|
|||
|
||||
sub citation {
|
||||
# Returns: N/A
|
||||
my ($all_argv_ref,$argv_options_removed_ref) = @_;
|
||||
my $all_argv = "@$all_argv_ref";
|
||||
my $no_opts = "@$argv_options_removed_ref";
|
||||
$all_argv=~s/--citation//;
|
||||
if($all_argv ne $no_opts) {
|
||||
::warning("--citation ignores all other options and arguments.");
|
||||
::status("");
|
||||
}
|
||||
|
||||
::status(
|
||||
"Academic tradition requires you to cite works you base your article on.",
|
||||
"If you use programs that use GNU Parallel to process data for an article in a",
|
||||
|
|
|
|||
|
|
@ -470,8 +470,8 @@ Implies B<--semaphore>.
|
|||
|
||||
=item B<--citation>
|
||||
|
||||
Print the BibTeX entry for GNU B<parallel> and silence citation
|
||||
notice.
|
||||
Print the BibTeX entry for GNU B<parallel>, silence citation
|
||||
notice and exit. It will not run any commands.
|
||||
|
||||
If it is impossible for you to run B<--bibtex> you can use
|
||||
B<--will-cite>.
|
||||
|
|
|
|||
|
|
@ -1327,6 +1327,47 @@ It can be emulated with GNU B<parallel> using this Bash function:
|
|||
|
||||
This works execpt for the B<--exclude> option.
|
||||
|
||||
=head2 DIFFERENCES BETWEEN pyargs AND GNU Parallel
|
||||
|
||||
B<pyargs> deals badly with input containing spaces. It buffers stdout,
|
||||
but not stderr. It buffers in RAM. {} does not work as replacement
|
||||
string. It does not support running functions.
|
||||
|
||||
B<pyargs> does not support composed commands if run with B<--lines>,
|
||||
and fails on B<pyargs traceroute pi.dk fi.dk>.
|
||||
|
||||
=head3 Examples
|
||||
|
||||
seq 5 | pyargs -P50 -L seq
|
||||
seq 5 | parallel -P50 --lb seq
|
||||
|
||||
seq 5 | pyargs -P50 --mark -L seq
|
||||
seq 5 | parallel -P50 --lb \
|
||||
--tagstring OUTPUT'[{= $_=$job->replaced()=}]' seq
|
||||
# Similar, but not precisely the same
|
||||
seq 5 | parallel -P50 --lb --tag seq
|
||||
|
||||
seq 5 | pyargs -P50 --mark command
|
||||
# Somewhat longer with GNU Parallel due to the special
|
||||
# --mark formatting
|
||||
cmd="$(echo "command" | parallel --shellquote)"
|
||||
wrap_cmd() {
|
||||
echo "MARK $cmd $@================================" >&3
|
||||
echo "OUTPUT START[$cmd $@]:"
|
||||
eval $cmd "$@"
|
||||
echo "OUTPUT END[$cmd $@]"
|
||||
}
|
||||
(seq 5 | env_parallel -P2 wrap_cmd) 3>&1
|
||||
# Similar, but not exactly the same
|
||||
seq 5 | parallel -t --tag command
|
||||
|
||||
(echo '1 2 3';echo 4 5 6) | pyargs --stream seq
|
||||
(echo '1 2 3';echo 4 5 6) | perl -pe 's/\n/ /' |
|
||||
parallel -r -d' ' seq
|
||||
# Similar, but not exactly the same
|
||||
parallel seq ::: 1 2 3 4 5 6
|
||||
|
||||
|
||||
=head2 Todo
|
||||
|
||||
machma. Requires Go >= 1.7.
|
||||
|
|
@ -1344,6 +1385,8 @@ https://github.com/Julian/Verge
|
|||
|
||||
https://github.com/amattn/paral
|
||||
|
||||
pyargs
|
||||
|
||||
|
||||
=head1 TESTING OTHER TOOLS
|
||||
|
||||
|
|
@ -1363,18 +1406,22 @@ will most likely mix.
|
|||
cat <<-EOF > mycommand
|
||||
#!/bin/bash
|
||||
|
||||
# If 'a', 'b' and 'c' mix: Very bad
|
||||
perl -e 'print "a"x3000_000," "'
|
||||
perl -e 'print "b"x3000_000," "'
|
||||
perl -e 'print "c"x3000_000," "'
|
||||
# If a, b, c, d, e, and f mix: Very bad
|
||||
perl -e 'print STDOUT "a"x3000_000," "'
|
||||
perl -e 'print STDERR "b"x3000_000," "'
|
||||
perl -e 'print STDOUT "c"x3000_000," "'
|
||||
perl -e 'print STDERR "d"x3000_000," "'
|
||||
perl -e 'print STDOUT "e"x3000_000," "'
|
||||
perl -e 'print STDERR "f"x3000_000," "'
|
||||
echo
|
||||
echo >&2
|
||||
EOF
|
||||
chmod +x mycommand
|
||||
|
||||
# Run 30 jobs in parallel
|
||||
seq 30 | $paralleltool -j0 ./mycommand | tr -s abc
|
||||
seq 30 | $paralleltool -j0 ./mycommand > >(tr -s abcdef) 2> >(tr -s abcdef >&2)
|
||||
|
||||
# 'a b c' should always stay together
|
||||
# 'a c e' and 'b d f' should always stay together
|
||||
# and there should only be a single line per job
|
||||
|
||||
=head2 B: Output limited by RAM
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20171222;
|
||||
$Global::version = 20171224;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ portable:
|
|||
time bash Portable.sh
|
||||
|
||||
timings: tests-to-run/* ../src/parallel
|
||||
ls tests-to-run/*.sh | parallel echo /usr/bin/time -f %e bash >/tmp/timing.script
|
||||
ls tests-to-run/*.sh | parallel echo /usr/bin/time -f "'%e\ %P'" bash >/tmp/timing.script
|
||||
stdout bash -x /tmp/timing.script | tee /tmp/timing.out
|
||||
echo + .usr.bin.time_END >>/tmp/timing.out
|
||||
perl -ne '/\+ .usr.bin.time/ and do { print $$last.$$h; $$h=$$_ }; chomp; s/.*\0//;$$last = $$_' /tmp/timing.out |sort -n >timings
|
||||
|
|
|
|||
|
|
@ -804,6 +804,17 @@ par_halt_one_job() {
|
|||
parallel -j1 --halt soon,fail=1 'echo {#};exit {}' ::: 1 0 1
|
||||
}
|
||||
|
||||
par_blocking_redir() {
|
||||
(
|
||||
echo 'bug #52740: Bash redirection with process substitution blocks'
|
||||
echo Test stdout
|
||||
echo 3 | parallel seq > >(echo stdout;wc) 2> >(echo stderr >&2; wc >&2)
|
||||
echo Test stderr
|
||||
echo nOfilE | parallel ls > >(echo stdout;wc) 2> >(echo stderr >&2; wc >&2)
|
||||
) 2>&1 | sort
|
||||
}
|
||||
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
compgen -A function | grep par_ | sort |
|
||||
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'
|
||||
|
|
|
|||
|
|
@ -1344,6 +1344,17 @@ par_basic_halt true
|
|||
par_basic_halt parallel: Error: --halt must have 'never', 'soon', or 'now'.
|
||||
par_basic_halt parallel: Error: --halt soon must be followed by ,success or ,fail.
|
||||
par_basic_halt parallel: Error: --halt now must be followed by ,success or ,fail.
|
||||
par_blocking_redir 0 0 0
|
||||
par_blocking_redir 0 0 0
|
||||
par_blocking_redir 1 9 54
|
||||
par_blocking_redir 3 3 6
|
||||
par_blocking_redir Test stderr
|
||||
par_blocking_redir Test stdout
|
||||
par_blocking_redir bug #52740: Bash redirection with process substitution blocks
|
||||
par_blocking_redir stderr
|
||||
par_blocking_redir stderr
|
||||
par_blocking_redir stdout
|
||||
par_blocking_redir stdout
|
||||
par_colsep_0 bug --colsep 0
|
||||
par_colsep_0 OK
|
||||
par_colsep_0 OK
|
||||
|
|
|
|||
Loading…
Reference in a new issue