parallel: --latest-line implemented.

This commit is contained in:
Ole Tange 2022-05-14 01:27:17 +02:00
parent 5fa7e57b6b
commit 5ecca1ccf4
13 changed files with 478 additions and 384 deletions

View file

@ -254,7 +254,7 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20220522 ('Emmanuel Jean-Michel Frédéric<<>>') released [stable]
Subject: GNU Parallel 20220522 ('Слава Україні Emmanuel Jean-Michel Frédéric<<>>') released [stable]
GNU Parallel 20220522 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
@ -272,6 +272,10 @@ New in this release:
News about GNU Parallel:
* Building a fault-tolerant work queue for command-line executions with GNU Parallel https://www.jvt.me/posts/2022/04/28/shell-queue/
https://www.blopig.com/blog/2022/05/make-your-code-do-more-with-less/
<<>>
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

View file

@ -128,6 +128,16 @@ To install in all shells run:
env_parallel --install
In a script you need to run this before using env_parallel:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man env_parallel
_EOS

View file

@ -1716,6 +1716,9 @@ sub options_completion_hash() {
("ungroup|u".
"[Output is printed as soon as possible and bypasses GNU parallel internal processing]"
=> \$opt::ungroup),
("last-line-buffer|last-line-buffered|lastlinebuffer|lastlinebuffered|llb".
"[Print latest line of each job]"
=> \$opt::lastlinebuffer),
("line-buffer|line-buffered|linebuffer|linebuffered|lb".
"[Buffer output on line basis]"
=> \$opt::linebuffer),
@ -2155,6 +2158,9 @@ sub parse_options(@) {
or defined $opt::color) {
$Global::color = 1;
}
if($opt::linebuffer or $opt::lastlinebuffer) {
$Global::linebuffer = 1;
}
if(defined $opt::tag and not defined $opt::tagstring) {
# Default = {}
$opt::tagstring = $Global::parensleft.$Global::parensright;
@ -2163,7 +2169,7 @@ sub parse_options(@) {
$opt::tagstring = unquote_printf($opt::tagstring);
if($opt::tagstring =~ /\Q$Global::parensleft\E.*\Q$Global::parensright\E/
and
$opt::linebuffer) {
$Global::linebuffer) {
# --tagstring contains {= =} and --linebuffer =>
# recompute replacement string for each use (do not cache)
$Global::cache_replacement_eval = 0;
@ -2606,7 +2612,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20220429;
$Global::version = 20220501;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -4965,33 +4971,31 @@ sub onall($@) {
# --jobs = number of hosts to run on simultaneously.
# For each host a parallel command with the args will be running.
# Uses:
# $Global::debug
# $Global::exitstatus
# $Global::joblog
# $Global::quoting
# @opt::basefile
# $opt::jobs
# $opt::linebuffer
# $opt::ungroup
# $opt::group
# $opt::keeporder
# $opt::D
# $opt::plain
# $opt::max_chars
# $opt::files
# $opt::arg_file_sep
# $opt::arg_sep
# $opt::colsep
# $opt::timeout
# $opt::files
# $opt::group
# $opt::joblog
# $opt::jobs
# $opt::keeporder
# $opt::linebuffer
# $opt::max_chars
# $opt::plain
# $opt::retries
# $opt::max_chars
# $opt::arg_sep
# $opt::arg_file_sep
# @opt::v
# @opt::env
# %Global::host
# $Global::exitstatus
# $Global::debug
# $Global::joblog
# $opt::joblog
# $opt::tag
# $opt::tee
# $opt::timeout
# $opt::ungroup
# %Global::host
# @opt::basefile
# @opt::env
# @opt::v
# Input:
# @command = command to run on all hosts
# Returns: N/A
@ -6474,7 +6478,7 @@ sub reap_usleep() {
kill_youngest_if_over_limit();
}
exit_if_disk_full();
if($opt::linebuffer) {
if($Global::linebuffer) {
my $something_printed = 0;
if($opt::keeporder) {
for my $job (values %Global::running) {
@ -8903,7 +8907,7 @@ sub openoutputfiles($) {
my $self = shift;
my ($outfhw, $errfhw, $outname, $errname);
if($opt::linebuffer and not
if($Global::linebuffer and not
($opt::keeporder or $opt::files or $opt::results or
$opt::compress or $opt::compress_program or
$opt::decompress_program)) {
@ -8999,7 +9003,7 @@ sub openoutputfiles($) {
} elsif(not $opt::ungroup) {
$self->grouped();
}
if($opt::linebuffer) {
if($Global::linebuffer) {
# Make it possible to read non-blocking from
# the buffer files
# Used for --linebuffer with -k, --files, --res, --compress*
@ -10808,7 +10812,7 @@ sub print($) {
next;
}
::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n");
if($opt::linebuffer) {
if($Global::linebuffer) {
# Line buffered print out
$self->print_linebuffer($fdno,$in_fh,$out_fh);
} elsif($opt::files) {
@ -11001,7 +11005,7 @@ sub combine_ref($) {
push @out, \$sep, $column;
}
}
# Pop off a $sep
# Remove the first $sep: ,val,"val" => val,"val"
shift @out;
return @out;
}
@ -11047,146 +11051,194 @@ sub print_files($) {
}
}
sub print_linebuffer($) {
my $self = shift;
my ($fdno,$in_fh,$out_fh) = @_;
if(defined $self->{'exitstatus'}) {
# If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::compress_program." failed.");
$self->set_exitstatus(255);
}
if($opt::compress) {
# Blocked reading in final round
for my $fdno (1,2) {
::set_fh_blocking($self->fh($fdno,'r'));
}
}
}
if(not $self->virgin()) {
if($opt::files or ($opt::results and not $Global::csvsep)) {
# Print filename
if($fdno == 1 and not $self->fh($fdno,"printed")) {
print $out_fh $self->tag(),$self->fh($fdno,"name"),"\n";
if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, $self->tag(),
$self->fh($fdno,"name"));
}
$self->set_fh($fdno,"printed",1);
}
# No need for reading $in_fh, as it is from "cat >/dev/null"
} else {
# Read halflines and print full lines
my $outputlength = 0;
my $halfline_ref = $self->{'halfline'}{$fdno};
my ($buf,$i,$rv);
# 1310720 gives 1.2 GB/s
# 131072 gives 0.9 GB/s
# The optimal block size differs
# It has been measured on:
# AMD 6376: 60800 (>70k is also reasonable)
# Intel i7-3632QM: 52-59k, 170-175k
# seq 64 | ppar --test $1 --lb 'yes {} `seq 1000`|head -c 10000000' >/dev/null
while($rv = sysread($in_fh, $buf, 60800)) {
$outputlength += $rv;
# TODO --recend
# Treat both \n and \r as line end
# Only test for \r if there is no \n
# Test:
# perl -e '$a="x"x1000000;
# $b="$a\r$a\n$a\r$a\n";
# map { print $b,$_ } 1..10'
$i = ((rindex($buf,"\n")+1) || (rindex($buf,"\r")+1));
if($i) {
# One or more complete lines were found
if($Global::color) {
# To paint a full line, we need to turn on paint
# BEFORE printing \n
# print @$halfline_ref, substr($buf,0,$i);
my $print = join("",@$halfline_ref, substr($buf,0,$i));
chomp($print);
# print color-on \n tag
my $tag = $self->tag();
# TODO fix \r
$print =~ s/([\n\r])(?=.|$)/$tag/gs;
print $out_fh $tag,$print;
} elsif($opt::tag or defined $opt::tagstring) {
# Replace ^ with $tag within the full line
if($Global::cache_replacement_eval) {
# Replace with the same value for tag
my $tag = $self->tag();
unshift @$halfline_ref, $tag;
# TODO --recend that can be partially in @$halfline_ref
substr($buf,0,$i-1) =~
s/(?<=[\n\r])(?=.|$)/$tag/gs;
# The length changed, so find the new ending pos
$i = ::max((rindex($buf,"\n")+1),
(rindex($buf,"\r")+1));
} else {
# Replace with freshly computed value of tag
unshift @$halfline_ref, $self->tag();
substr($buf,0,$i-1) =~
s/(?<=[\n\r])(?=.|$)/$self->tag()/gse;
# The length changed, so find the new ending pos
$i = ::max((rindex($buf,"\n")+1),
(rindex($buf,"\r")+1));
}
# Print the partial line (halfline) and the last half
print $out_fh @$halfline_ref, substr($buf,0,$i);
} else {
# Print the partial line (halfline) and the last half
print $out_fh @$halfline_ref, substr($buf,0,$i);
}
# Buffer in memory for SQL and CSV-output
if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}},
@$halfline_ref, substr($buf,0,$i));
}
# Remove the printed part by keeping the unprinted part
@$halfline_ref = (substr($buf,$i));
} else {
# No newline, so append to the halfline
push @$halfline_ref, $buf;
}
}
$self->add_returnsize($outputlength);
{
my ($up,$eol,$reset_color,$init,$curseq,$maxseq);
sub print_linebuffer($) {
sub print_llb($) {
my $self = shift;
my $out_fh = shift;
my $str = shift;
my $seq = $self->seq();
$maxseq = $seq > $maxseq ? $seq : $maxseq;
print($out_fh
"$up"x($curseq - $seq),
"\n"x($seq - $curseq),
"\r", $self->tag(),
$str, $eol, $reset_color,
"\n"x($maxseq-$seq+1));
$curseq = $maxseq + 1;
}
my $self = shift;
my ($fdno,$in_fh,$out_fh) = @_;
if(defined $self->{'exitstatus'}) {
if($opt::files or ($opt::results and not $Global::csvsep)) {
$self->add_returnsize(-s $self->fh($fdno,"name"));
} else {
# If the job is dead: print the remaining partial line
# read remaining
my $halfline_ref = $self->{'halfline'}{$fdno};
if(grep /./, @$halfline_ref) {
my $returnsize = 0;
for(@{$self->{'halfline'}{$fdno}}) {
$returnsize += length $_;
}
$self->add_returnsize($returnsize);
if($opt::tag or defined $opt::tagstring) {
# Prepend $tag the the remaining half line
unshift @$halfline_ref, $self->tag();
}
# Print the partial line (halfline)
print $out_fh @{$self->{'halfline'}{$fdno}};
# Buffer in memory for SQL and CSV-output
if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, @$halfline_ref);
}
@$halfline_ref = ();
# If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::compress_program." failed.");
$self->set_exitstatus(255);
}
if($opt::compress) {
# Blocked reading in final round
for my $fdno (1,2) {
::set_fh_blocking($self->fh($fdno,'r'));
}
}
if($self->fh($fdno,"rpid") and
CORE::kill 0, $self->fh($fdno,"rpid")) {
# decompress still running
}
if(not $init) {
$init = 1;
if($Global::color or $opt::lastlinebuffer) {
# cursor_up cuu1 = up one line
$up = `tput cuu1 </dev/tty`;
chomp($up);
# clr_eol el = clear to end of line
$eol = `tput el </dev/tty`;
chomp($eol);
# exit_attribute_mode sgr0 = turn off all attributes
$reset_color = `tput sgr0 </dev/tty`;
chomp($reset_color);
$curseq = 1;
$maxseq = 1;
}
}
if(not $self->virgin()) {
if($opt::files or ($opt::results and not $Global::csvsep)) {
# Print filename
if($fdno == 1 and not $self->fh($fdno,"printed")) {
print $out_fh $self->tag(),$self->fh($fdno,"name"),"\n";
if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, $self->tag(),
$self->fh($fdno,"name"));
}
$self->set_fh($fdno,"printed",1);
}
# No need for reading $in_fh, as it is from "cat >/dev/null"
} else {
# decompress done: close fh
close $in_fh;
if($? and $opt::compress) {
::error($opt::decompress_program." failed.");
$self->set_exitstatus(255);
# Read halflines and print full lines
my $outputlength = 0;
my $halfline_ref = $self->{'halfline'}{$fdno};
my ($buf,$i,$rv);
# 1310720 gives 1.2 GB/s
# 131072 gives 0.9 GB/s
# The optimal block size differs
# It has been measured on:
# AMD 6376: 60800 (>70k is also reasonable)
# Intel i7-3632QM: 52-59k, 170-175k
# seq 64 | ppar --_test $1 --lb \
# 'yes {} `seq 1000`|head -c 10000000' >/dev/null
while($rv = sysread($in_fh, $buf, 60800)) {
$outputlength += $rv;
# TODO --recend
# Treat both \n and \r as line end
# Only test for \r if there is no \n
# Test:
# perl -e '$a="x"x1000000;
# $b="$a\r$a\n$a\r$a\n";
# map { print $b,$_ } 1..10'
$i = ((rindex($buf,"\n")+1) || (rindex($buf,"\r")+1));
if($i) {
if($opt::lastlinebuffer) {
# Remove the final \n/\r
my $l = join('', @$halfline_ref,
substr($buf,0,$i-1));
my $j = ((rindex($l,"\n")+1) ||
(rindex($l,"\r")+1));
$self->print_llb($out_fh,substr($l,$j));
# Remove the printed part by keeping the unprinted
@$halfline_ref = (substr($buf,$i));
} else {
# One or more complete lines were found
if($Global::color) {
my $print = join("",@$halfline_ref,
substr($buf,0,$i));
chomp($print);
my $tag = $self->tag();
$print =~ s/([\n\r])(?=.|$)/$eol$reset_color$1$tag/gs;
print $out_fh $tag, $print, $eol,
$reset_color, "\n";
} elsif($opt::tag or defined $opt::tagstring) {
# Replace ^ with $tag within the full line
if($Global::cache_replacement_eval) {
# Replace with the same value for tag
my $tag = $self->tag();
unshift @$halfline_ref, $tag;
# TODO --recend that can be partially in
# @$halfline_ref
substr($buf,0,$i-1) =~
s/(?<=[\n\r])(?=.|$)/$tag/gs;
# The length changed,
# so find the new ending pos
$i = ::max((rindex($buf,"\n")+1),
(rindex($buf,"\r")+1));
} else {
# Replace with freshly computed tag-value
unshift @$halfline_ref, $self->tag();
substr($buf,0,$i-1) =~
s/(?<=[\n\r])(?=.|$)/$self->tag()/gse;
# The length changed,
# so find the new ending pos
$i = ::max((rindex($buf,"\n")+1),
(rindex($buf,"\r")+1));
}
# Print the partial line (halfline)
# and the last half
print $out_fh @$halfline_ref, substr($buf,0,$i);
} else {
# Print the partial line (halfline)
# and the last half
print $out_fh @$halfline_ref, substr($buf,0,$i);
}
# Buffer in memory for SQL and CSV-output
if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}},
@$halfline_ref, substr($buf,0,$i));
}
# Remove the printed part by keeping the unprinted
@$halfline_ref = (substr($buf,$i));
}
} else {
# No newline, so append to the halfline
push @$halfline_ref, $buf;
}
}
$self->add_returnsize($outputlength);
}
if(defined $self->{'exitstatus'}) {
if($opt::files or ($opt::results and not $Global::csvsep)) {
$self->add_returnsize(-s $self->fh($fdno,"name"));
} else {
# If the job is dead: print the remaining partial line
# read remaining
my $halfline_ref = $self->{'halfline'}{$fdno};
if(grep /./, @$halfline_ref) {
my $returnsize = 0;
for(@{$self->{'halfline'}{$fdno}}) {
$returnsize += length $_;
}
$self->add_returnsize($returnsize);
if($opt::tag or defined $opt::tagstring) {
# Prepend $tag the the remaining half line
unshift @$halfline_ref, $self->tag();
}
# Print the partial line (halfline)
print $out_fh @{$self->{'halfline'}{$fdno}};
# Buffer in memory for SQL and CSV-output
if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, @$halfline_ref);
}
@$halfline_ref = ();
}
}
if($self->fh($fdno,"rpid") and
CORE::kill 0, $self->fh($fdno,"rpid")) {
# decompress still running
} else {
# decompress done: close fh
close $in_fh;
if($? and $opt::compress) {
::error($opt::decompress_program." failed.");
$self->set_exitstatus(255);
}
}
}
}
@ -11277,8 +11329,11 @@ sub print_normal($) {
$outputlength += length $_;
# Tag lines with \r, too
$_ =~ s/(?<=[\r])(?=.|$)/$tag/gs;
$Global::color and chomp();
print $out_fh $tag,$_;
if($Global::color) {
print_color($out_fh,$tag,$_);
} else {
print $out_fh $tag,$_;
}
if($Global::membuffer) {
push @{$self->{'output'}{$fdno}}, $tag, $_;
}
@ -11384,34 +11439,60 @@ sub print_joblog($) {
{
my @color;
my $color_on = "";
my ($up,$eol,$reset_color,$init,$curseq,$maxseq);
sub init_color() {
$init = 1;
# color combinations that are readable: black/white text
# on colored background, but not white on yellow
my @color_combinations =
# Force each color code to have the same length in chars
# This will make \t work as expected
((map { [sprintf("%03d",$_),"000"] }
6..7,9..11,13..15,40..51,75..87,113..123,147..159,
171..182,185..231,249..254),
(map { [sprintf("%03d",$_),231] }
1..9,12..13,16..45,52..81,88..114,124..149,
160..178,180,182..184,196..214,232..250));
# reorder list so adjacent colors are dissimilar
# %23 and %7 were found experimentally
@color_combinations = @color_combinations[
sort { ($a%23 <=> $b%23) or ($b%7 <=> $a%7) }
0..$#color_combinations
];
@color = map {
# TODO Can this be done with `tput` codes?
"\033[48;5;".$_->[0].";38;5;".$_->[1]."m"
} @color_combinations;
# cursor_up cuu1 = up one line
$up = `tput cuu1 </dev/tty`;
chomp($up);
# clr_eol el = clear to end of line
$eol = `tput el </dev/tty`;
chomp($eol);
# exit_attribute_mode sgr0 = turn off all attributes
$reset_color = `tput sgr0 </dev/tty`;
chomp($reset_color);
$curseq = 1;
$maxseq = 1;
}
sub print_color() {
# @text is a single line
my ($out_fh,@text) = @_;
chomp(@text);
print $out_fh @text,$reset_color,"\n";
}
sub tag($) {
sub init_color() {
# color combinations that are readable: black/white text
# on colored background, but not white on yellow
@color =
# Force each color code to have the same length in chars
# This will make \t work as expected
((map { [sprintf("%03d",$_),"000"] }
6..7,9..11,13..15,40..51,75..87,113..123,147..159,
171..231,249..254),
(map { [sprintf("%03d",$_),231] }
1..9,12..13,16..45,52..81,88..114,124..149,153,
160..178,180,182..185,187..186,189,196..214,232..250,
253..254));
# reorder list so adjacent colors are dissimilar
# %23 and %7 were found experimentally
@color = @color[
sort { ($a%23 <=> $b%23) or ($b%7 <=> $a%7) } 0..$#color
];
}
my $self = shift;
if(not defined $self->{'tag'} or not $Global::cache_replacement_eval) {
if($Global::color) {
if(not @color) { init_color() }
if(not $init) { init_color() }
# Choose a value based on the seq
my $col = @color[$self->seq() % ($#color+1)];
$color_on = "\033[48;5;".$col->[0].";38;5;".$col->[1]."m\n";
$color_on = $color[$self->seq() % ($#color+1)].$eol;
}
if(defined $opt::tag or defined $opt::tagstring) {
$self->{'tag'} =
@ -14526,14 +14607,8 @@ sub main() {
drain_job_queue(@command);
::debug("init", "Done draining\n");
reapers();
if($Global::color) {
print $Global::original_stderr "\033[00m\n";
print "\033[00m\n";
}
::debug("init", "Done reaping\n");
if($Global::semaphore) {
$sem->release();
}
if($Global::semaphore) { $sem->release(); }
cleanup();
::debug("init", "Halt\n");
halt();

View file

@ -748,15 +748,13 @@ B<--transfer>, B<--transferfile> or B<--return>.
See also: B<--basefile> B<--transfer> B<--transferfile> B<--sshlogin>
B<--return>
=item B<--color>
=item B<--color> (alpha testing)
Colour output.
Colour the output. Each job gets its own color combination
(background+foreground).
See also: B<--ctag> B<--ctagstring>
=item B<--colsep> I<regexp>
@ -1545,6 +1543,26 @@ Similar to B<--memfree>.
See also: B<--memfree> B<--load>
=item B<--last-line-buffer> (alpha testing)
=item B<--llb> (alpha testing)
Print the lastest line of each running job.
This only works if the currently running jobs fit on the screen.
Example:
slow_seq() {
seq "$@" |
perl -ne '$|=1; for(split//){ print; select($a,$a,$a,0.03);}'
}
export -f slow_seq
parallel --shuf -j10 --llb --color slow_seq {} ::: {1..100}
See also: B<--line-buffer>
=item B<--line-buffer>
=item B<--lb>
@ -2720,7 +2738,7 @@ exit.
Only used with B<env_parallel>. Aliases, functions, and variables with
names in B<$PARALLEL_IGNORED_NAMES> will not be copied. So you should
set variables/function you want to use I<after> running B<--session>.
set variables/function you want copied I<after> running B<--session>.
It is similar to B<--record-env>, but only for this session.

View file

@ -3375,7 +3375,7 @@ Summary (see legend above):
B<p> is a tiny shell script. It can color output with some predefined
colors, but is otherwise quite limited.
It maxes out at 116260 jobs (probably due to limitations in Bash).
It maxes out at around 116000 jobs (probably due to limitations in Bash).
=head3 EXAMPLES FROM p

View file

@ -1209,7 +1209,6 @@ B<-->. I have seen B<::> used in programming languanges to separate
classes, and I did not want the user to be confused that the separator
had anything to do with classes.
B<:::> also makes a visual separation, which is good if there are
multiple B<:::>.
@ -1219,6 +1218,10 @@ Linking input sources meant having to decide for some way to indicate
linking of B<:::> and B<::::>. B<:::+> and B<::::+> were chosen, so
that they were similar to B<:::> and B<::::>.
In 2022 I realized that B<///> would have been an even better choice,
because you cannot have an file named B<///> whereas you I<can> have a
file named B<:::>.
=head2 Perl replacement strings, {= =}, and --rpl

View file

@ -79,6 +79,7 @@ done
cat <<'_EOS'
parset only works if it is a function. The function is defined as part
of env_parallel.
@ -122,6 +123,16 @@ To install in all shells run:
parset --install
In a script you need to run this before using parset:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man parset
_EOS

View file

@ -8,6 +8,23 @@
# Each should be taking 3-10s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_llb_color() {
echo 'bug #62386: --color (--ctag but without --tag)'
echo 'bug #62438: See last line from multiple jobslots'
slow_seq() {
sleep 0.$1
seq $1 | pv -qL $1
}
export -f slow_seq
run() {
seq 4 | parallel --color $@ slow_seq
}
export -f run
parallel --delay 0.1 -vkj0 run \
::: --lb --llb '' ::: --color '' ::: '--tagstring {}{}' --tag '' ::: -k '' |
md5sum
}
par_process_slot_var() {
echo '### bug #62310: xargs compatibility: --process-slot-var=name'
seq 0.1 0.1 0.5 |

View file

@ -17,112 +17,123 @@ rm -f /run/shm/parallel.db
mkdir -p /run/shm/csv
p_showsqlresult() {
SERVERURL=$1
TABLE=$2
sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;"
# print results stored in $SERVERURL/$TABLE
SERVERURL=$1
TABLE=$2
sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;"
}
p_wrapper() {
INNER=$1
SERVERURL=$(eval echo $2)
TABLE=TBL$RANDOM
DBURL=$SERVERURL/$TABLE
T1=$(tempfile)
T2=$(tempfile)
eval "$INNER"
echo Exit=$?
wait
echo Exit=$?
$DEBUG && sort -u $T1 $T2;
rm $T1 $T2
p_showsqlresult $SERVERURL $TABLE
$DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
INNER=$1
SERVERURL=$(eval echo $2)
# Use a random table for each test
TABLE=TBL$RANDOM
DBURL=$SERVERURL/$TABLE
T1=$(tempfile)
T2=$(tempfile)
# Run $INNER (all the par_* functions)
eval "$INNER"
echo Exit=$?
# $INNER can start background processes - wait for those
wait
echo Exit=$?
# For debugging show the tempfiles
$DEBUG && sort -u $T1 $T2;
rm $T1 $T2
p_showsqlresult $SERVERURL $TABLE
# Drop the table if not debugging
$DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
}
p_template() {
(sleep 6;
parallel --sqlworker $DBURL "$@" sleep .3\;echo >$T1) &
parallel --sqlandworker $DBURL "$@" sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
# Run the
(
# Make sure there is work to be done
sleep 6;
parallel --sqlworker $DBURL "$@" sleep .3\;echo >$T1
) &
parallel --sqlandworker $DBURL "$@" sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
}
par_sqlandworker() {
p_template
p_template
}
par_sqlandworker_lo() {
p_template -S lo
p_template -S lo
}
par_sqlandworker_results() {
p_template --results /tmp/out--sql
p_template --results /tmp/out--sql
}
par_sqlandworker_linebuffer() {
p_template --linebuffer
p_template --linebuffer
}
par_sqlandworker_tag() {
p_template --tag
p_template --tag
}
par_sqlandworker_linebuffer_tag() {
p_template --linebuffer --tag
p_template --linebuffer --tag
}
par_sqlandworker_compress_linebuffer_tag() {
p_template --compress --linebuffer --tag
p_template --compress --linebuffer --tag
}
par_sqlandworker_unbuffer() {
p_template -u
p_template -u
}
par_sqlandworker_total_jobs() {
p_template echo {#} of '{=1 $_=total_jobs(); =};'
p_template echo {#} of '{=1 $_=total_jobs(); =};'
}
par_append() {
parallel --sqlmaster $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
parallel --sqlmaster +$DBURL sleep .3\;echo ::: {11..15} ::: {A..E} >>$T2;
parallel --sqlworker $DBURL sleep .3\;echo >$T1
parallel --sqlmaster $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
parallel --sqlmaster +$DBURL sleep .3\;echo ::: {11..15} ::: {A..E} >>$T2;
parallel --sqlworker $DBURL sleep .3\;echo >$T1
}
par_shuf() {
MD5=$(echo $SERVERURL | md5sum | perl -pe 's/(...).*/$1/')
T=/tmp/parallel-bug49791-$MD5
[ -e $T ] && rm -rf $T
export PARALLEL="--shuf --result $T"
parallel --sqlandworker $DBURL sleep .3\;echo \
::: {1..5} ::: {a..e} >$T2;
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
unset PARALLEL
wait;
# Did it compute correctly?
cat $T/1/*/*/*/stdout
# Did it shuffle (Compare job table to non-shuffled)
SHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
export PARALLEL="--result $T"
parallel --sqlandworker $DBURL sleep .3\;echo \
::: {1..5} ::: {a..e} >$T2;
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
unset PARALLEL
wait;
NOSHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
DIFFSIZE=$(diff <(echo "$SHUF") <(echo "$NOSHUF") | wc -c)
if [ $DIFFSIZE -gt 2500 ]; then
echo OK: Diff bigger than 2500 char
fi
[ -e $T ] && rm -rf $T
touch $T1
MD5=$(echo $SERVERURL | md5sum | perl -pe 's/(...).*/$1/')
T=/tmp/parallel-bug49791-$MD5
[ -e $T ] && rm -rf $T
export PARALLEL="--shuf --result $T"
parallel --sqlandworker $DBURL sleep .3\;echo \
::: {1..5} ::: {a..e} >$T2;
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
unset PARALLEL
wait;
# Did it compute correctly?
cat $T/1/*/*/*/stdout
# Did it shuffle (Compare job table to non-shuffled)
SHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
export PARALLEL="--result $T"
parallel --sqlandworker $DBURL sleep .3\;echo \
::: {1..5} ::: {a..e} >$T2;
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
unset PARALLEL
wait;
NOSHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
DIFFSIZE=$(diff <(echo "$SHUF") <(echo "$NOSHUF") | wc -c)
if [ $DIFFSIZE -gt 2500 ]; then
echo OK: Diff bigger than 2500 char
fi
[ -e $T ] && rm -rf $T
touch $T1
}
par_empty() {
echo Do nothing: TBL99999 does not exist because it is not created
true;
}
@ -131,12 +142,12 @@ export -f $(compgen -A function | egrep 'p_|par_')
# Tested that -j0 in parallel is fastest (up to 15 jobs)
# -j5: SQLite complains about locked database.
compgen -A function | grep par_ | sort |
stdout parallel -vj4 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \
:::: - ::: \$MYSQL \$PG \$SQLITE \$CSV |
perl -pe 's/tbl\d+/TBL99999/gi;' |
perl -pe 's/(from TBL99999 order) .*/$1/g' |
perl -pe 's/ *\b'"$hostname"'\b */hostname/g' |
grep -v -- --------------- |
perl -pe 's/ *\bhost\b */host/g' |
perl -pe 's/ +/ /g'
stdout parallel -vj4 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \
:::: - ::: \$MYSQL \$PG \$SQLITE \$CSV |
perl -pe 's/tbl\d+/TBL99999/gi;' |
perl -pe 's/(from TBL99999 order) .*/$1/g' |
perl -pe 's/ *\b'"$hostname"'\b */hostname/g' |
grep -v -- --------------- |
perl -pe 's/ *\bhost\b */host/g' |
perl -pe 's/ +/ /g'

View file

@ -110,11 +110,9 @@ par_csv_pipe 9000"
par_csv_pipe 11000"
par_ctagstring ### --ctag --ctagstring should be different from --tag --tagstring
par_ctagstring 8
par_ctagstring 
par_ctagstring 34
par_ctagstring 37
par_ctagstring 10
par_ctagstring 
par_ctagstring 36
par_ctagstring 39
par_delimiter ### Test --delimiter and -d: Delimiter instead of newline
par_delimiter # Yes there is supposed to be an extra newline for -d N
par_delimiter This is line 1
@ -955,10 +953,10 @@ par_sem_quote ### sem --quote should not add empty argument
par_sem_quote echo
par_sem_quote
par_shellcompletion ### --shellcompletion
par_shellcompletion 9c26e11f6435802cd18a8fd78f65f71d -
par_shellcompletion 9c26e11f6435802cd18a8fd78f65f71d -
par_shellcompletion 2b8f8856cd7b632edec5526f8d3637c9 -
par_shellcompletion 2b8f8856cd7b632edec5526f8d3637c9 -
par_shellcompletion a040c070a388e8799d899dce6e4d478b -
par_shellcompletion a040c070a388e8799d899dce6e4d478b -
par_shellcompletion 7bd30aa4542083e298ef0d1a60dc1322 -
par_shellcompletion 7bd30aa4542083e298ef0d1a60dc1322 -
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
par_slow_pipe_regexp 0 1 1

View file

@ -135,6 +135,9 @@ par_kill_term bash---pstree
par_lb_mem_usage 1
par_lb_mem_usage 1
par_lb_mem_usage 2
par_llb_color bug #62386: --color (--ctag but without --tag)
par_llb_color bug #62438: See last line from multiple jobslots
par_llb_color 74c668db859b3c8acbecbddf5880fd20 -
par_maxargs ### Test -n and --max-args: Max number of args per line (only with -X and -m)
par_maxargs line 1
par_maxargs line 2

View file

@ -270,120 +270,24 @@ par_append $CSV csv:///%2Frun%2Fshm is not a valid DBURL
par_append $CSV
par_append $CSV sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
par_empty $MYSQL p_wrapper par_empty '$MYSQL'
par_empty $MYSQL Do nothing: TBL99999 does not exist because it is not created
par_empty $MYSQL Exit=0
par_empty $MYSQL Exit=0
par_empty $MYSQL ERROR 1146 (42S02) at line 1: Table 'tange.TBL99999' doesn't exist
par_empty $PG p_wrapper par_empty '$PG'
par_empty $PG Do nothing: TBL99999 does not exist because it is not created
par_empty $PG Exit=0
par_empty $PG Exit=0
par_empty $PG host| command | v1 | v2 | stdout | stderr
par_empty $PG hostname| sleep .3;echo 1 a | 1 | a | 1 a +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 1 b | 1 | b | 1 b +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 1 c | 1 | c | 1 c +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 1 d | 1 | d | 1 d +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 1 e | 1 | e | 1 e +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 2 a | 2 | a | 2 a +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 2 b | 2 | b | 2 b +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 2 c | 2 | c | 2 c +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 2 d | 2 | d | 2 d +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 2 e | 2 | e | 2 e +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 3 a | 3 | a | 3 a +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 3 b | 3 | b | 3 b +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 3 c | 3 | c | 3 c +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 3 d | 3 | d | 3 d +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 3 e | 3 | e | 3 e +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 4 a | 4 | a | 4 a +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 4 b | 4 | b | 4 b +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 4 c | 4 | c | 4 c +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 4 d | 4 | d | 4 d +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 4 e | 4 | e | 4 e +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 5 a | 5 | a | 5 a +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 5 b | 5 | b | 5 b +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 5 c | 5 | c | 5 c +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 5 d | 5 | d | 5 d +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 5 e | 5 | e | 5 e +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 11 A | 11 | A | 11 A +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 11 B | 11 | B | 11 B +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 11 C | 11 | C | 11 C +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 11 D | 11 | D | 11 D +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 11 E | 11 | E | 11 E +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 12 A | 12 | A | 12 A +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 12 B | 12 | B | 12 B +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 12 C | 12 | C | 12 C +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 12 D | 12 | D | 12 D +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 12 E | 12 | E | 12 E +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 13 A | 13 | A | 13 A +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 13 B | 13 | B | 13 B +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 13 C | 13 | C | 13 C +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 13 D | 13 | D | 13 D +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 13 E | 13 | E | 13 E +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 14 A | 14 | A | 14 A +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 14 B | 14 | B | 14 B +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 14 C | 14 | C | 14 C +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 14 D | 14 | D | 14 D +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 14 E | 14 | E | 14 E +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 15 A | 15 | A | 15 A +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 15 B | 15 | B | 15 B +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 15 C | 15 | C | 15 C +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 15 D | 15 | D | 15 D +|
par_empty $PG | | | | |
par_empty $PG hostname| sleep .3;echo 15 E | 15 | E | 15 E +|
par_empty $PG | | | | |
par_empty $PG (50 rows)
par_empty $PG
par_empty $PG ERROR: relation "TBL99999" does not exist
par_empty $PG LINE 1: select Host,Command,V1,V2,Stdout,Stderr from TBL99999 order
par_empty $PG ^
par_empty $SQLITE p_wrapper par_empty '$SQLITE'
par_empty $SQLITE Do nothing: TBL99999 does not exist because it is not created
par_empty $SQLITE Exit=0
par_empty $SQLITE Exit=0
par_empty $SQLITE Error: near line 1: no such table: TBL99999
par_empty $CSV p_wrapper par_empty '$CSV'
par_empty $CSV Do nothing: TBL99999 does not exist because it is not created
par_empty $CSV Exit=0
par_empty $CSV Exit=0
par_empty $CSV Error:

View file

@ -761,6 +761,14 @@ tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
Supports: variables, aliases, arrays with no special chars
To install in all shells run:
env_parallel --install
In a script you need to run this before using env_parallel:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man env_parallel
MYVAR='foo bar'
export MYVAR
@ -837,6 +845,14 @@ tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
Supports: variables, aliases, arrays with no special chars
To install in all shells run:
env_parallel --install
In a script you need to run this before using env_parallel:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man env_parallel
parallel -vv --pipepart --block 1M wc :::: num30000
<num30000 perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>32767?32767:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 168894 |(wc)
@ -885,6 +901,14 @@ csh: Unsupported
tcsh: Unsupported
To install in all shells run:
parset --install
In a script you need to run this before using parset:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man parset
parset myarray seq {} 5 ::: 1 2 3
echo "${myarray[1]}"
@ -925,6 +949,14 @@ csh: Unsupported
tcsh: Unsupported
To install in all shells run:
parset --install
In a script you need to run this before using parset:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man parset
cmd=("echo '<<joe \"double space\" cartoon>>'" "pwd")
parset data ::: "${cmd[@]}"
@ -963,6 +995,14 @@ csh: Unsupported
tcsh: Unsupported
To install in all shells run:
parset --install
In a script you need to run this before using parset:
bash: . `which env_parallel.bash`
ksh: source `which env_parallel.ksh`
mksh: source `which env_parallel.mksh`
pdksh: source `which env_parallel.pdksh`
zsh: . `which env_parallel.zsh`
ash: . `which env_parallel.ash`
dash: . `which env_parallel.dash`
For details: see man parset
parallel --sqlandworker csv:///%2Ftmp/log.csv \
seq ::: 10 ::: 12 13 14
@ -1286,4 +1326,4 @@ mentioned in the release notes of next version of GNU Parallel.
echo A
echo B
echo C
6
8