mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: --latest-line implemented.
This commit is contained in:
Ole Tange
parent
5fa7e57b6b
commit
5ecca1ccf4
|
|
@ -254,7 +254,7 @@ from:tange@gnu.org
|
|||
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
Subject: GNU Parallel 20220522 ('Emmanuel Jean-Michel Frédéric<<>>') released [stable]
|
||||
Subject: GNU Parallel 20220522 ('Слава Україні Emmanuel Jean-Michel Frédéric<<>>') released [stable]
|
||||
|
||||
GNU Parallel 20220522 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
|
||||
|
||||
|
|
@ -272,6 +272,10 @@ New in this release:
|
|||
|
||||
News about GNU Parallel:
|
||||
|
||||
* Building a fault-tolerant work queue for command-line executions with GNU Parallel https://www.jvt.me/posts/2022/04/28/shell-queue/
|
||||
|
||||
https://www.blopig.com/blog/2022/05/make-your-code-do-more-with-less/
|
||||
|
||||
<<>>
|
||||
|
||||
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
|
||||
|
|
|
|||
|
|
@ -128,6 +128,16 @@ To install in all shells run:
|
|||
|
||||
env_parallel --install
|
||||
|
||||
In a script you need to run this before using env_parallel:
|
||||
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
|
||||
For details: see man env_parallel
|
||||
|
||||
_EOS
|
||||
|
|
|
|||
199
src/parallel
199
src/parallel
|
|
@ -1716,6 +1716,9 @@ sub options_completion_hash() {
|
|||
("ungroup|u".
|
||||
"[Output is printed as soon as possible and bypasses GNU parallel internal processing]"
|
||||
=> \$opt::ungroup),
|
||||
("last-line-buffer|last-line-buffered|lastlinebuffer|lastlinebuffered|llb".
|
||||
"[Print latest line of each job]"
|
||||
=> \$opt::lastlinebuffer),
|
||||
("line-buffer|line-buffered|linebuffer|linebuffered|lb".
|
||||
"[Buffer output on line basis]"
|
||||
=> \$opt::linebuffer),
|
||||
|
|
@ -2155,6 +2158,9 @@ sub parse_options(@) {
|
|||
or defined $opt::color) {
|
||||
$Global::color = 1;
|
||||
}
|
||||
if($opt::linebuffer or $opt::lastlinebuffer) {
|
||||
$Global::linebuffer = 1;
|
||||
}
|
||||
if(defined $opt::tag and not defined $opt::tagstring) {
|
||||
# Default = {}
|
||||
$opt::tagstring = $Global::parensleft.$Global::parensright;
|
||||
|
|
@ -2163,7 +2169,7 @@ sub parse_options(@) {
|
|||
$opt::tagstring = unquote_printf($opt::tagstring);
|
||||
if($opt::tagstring =~ /\Q$Global::parensleft\E.*\Q$Global::parensright\E/
|
||||
and
|
||||
$opt::linebuffer) {
|
||||
$Global::linebuffer) {
|
||||
# --tagstring contains {= =} and --linebuffer =>
|
||||
# recompute replacement string for each use (do not cache)
|
||||
$Global::cache_replacement_eval = 0;
|
||||
|
|
@ -2606,7 +2612,7 @@ sub check_invalid_option_combinations() {
|
|||
|
||||
sub init_globals() {
|
||||
# Defaults:
|
||||
$Global::version = 20220429;
|
||||
$Global::version = 20220501;
|
||||
$Global::progname = 'parallel';
|
||||
$::name = "GNU Parallel";
|
||||
$Global::infinity = 2**31;
|
||||
|
|
@ -4965,33 +4971,31 @@ sub onall($@) {
|
|||
# --jobs = number of hosts to run on simultaneously.
|
||||
# For each host a parallel command with the args will be running.
|
||||
# Uses:
|
||||
# $Global::debug
|
||||
# $Global::exitstatus
|
||||
# $Global::joblog
|
||||
# $Global::quoting
|
||||
# @opt::basefile
|
||||
# $opt::jobs
|
||||
# $opt::linebuffer
|
||||
# $opt::ungroup
|
||||
# $opt::group
|
||||
# $opt::keeporder
|
||||
# $opt::D
|
||||
# $opt::plain
|
||||
# $opt::max_chars
|
||||
# $opt::files
|
||||
# $opt::arg_file_sep
|
||||
# $opt::arg_sep
|
||||
# $opt::colsep
|
||||
# $opt::timeout
|
||||
# $opt::files
|
||||
# $opt::group
|
||||
# $opt::joblog
|
||||
# $opt::jobs
|
||||
# $opt::keeporder
|
||||
# $opt::linebuffer
|
||||
# $opt::max_chars
|
||||
# $opt::plain
|
||||
# $opt::retries
|
||||
# $opt::max_chars
|
||||
# $opt::arg_sep
|
||||
# $opt::arg_file_sep
|
||||
# @opt::v
|
||||
# @opt::env
|
||||
# %Global::host
|
||||
# $Global::exitstatus
|
||||
# $Global::debug
|
||||
# $Global::joblog
|
||||
# $opt::joblog
|
||||
# $opt::tag
|
||||
# $opt::tee
|
||||
# $opt::timeout
|
||||
# $opt::ungroup
|
||||
# %Global::host
|
||||
# @opt::basefile
|
||||
# @opt::env
|
||||
# @opt::v
|
||||
# Input:
|
||||
# @command = command to run on all hosts
|
||||
# Returns: N/A
|
||||
|
|
@ -6474,7 +6478,7 @@ sub reap_usleep() {
|
|||
kill_youngest_if_over_limit();
|
||||
}
|
||||
exit_if_disk_full();
|
||||
if($opt::linebuffer) {
|
||||
if($Global::linebuffer) {
|
||||
my $something_printed = 0;
|
||||
if($opt::keeporder) {
|
||||
for my $job (values %Global::running) {
|
||||
|
|
@ -8903,7 +8907,7 @@ sub openoutputfiles($) {
|
|||
my $self = shift;
|
||||
my ($outfhw, $errfhw, $outname, $errname);
|
||||
|
||||
if($opt::linebuffer and not
|
||||
if($Global::linebuffer and not
|
||||
($opt::keeporder or $opt::files or $opt::results or
|
||||
$opt::compress or $opt::compress_program or
|
||||
$opt::decompress_program)) {
|
||||
|
|
@ -8999,7 +9003,7 @@ sub openoutputfiles($) {
|
|||
} elsif(not $opt::ungroup) {
|
||||
$self->grouped();
|
||||
}
|
||||
if($opt::linebuffer) {
|
||||
if($Global::linebuffer) {
|
||||
# Make it possible to read non-blocking from
|
||||
# the buffer files
|
||||
# Used for --linebuffer with -k, --files, --res, --compress*
|
||||
|
|
@ -10808,7 +10812,7 @@ sub print($) {
|
|||
next;
|
||||
}
|
||||
::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n");
|
||||
if($opt::linebuffer) {
|
||||
if($Global::linebuffer) {
|
||||
# Line buffered print out
|
||||
$self->print_linebuffer($fdno,$in_fh,$out_fh);
|
||||
} elsif($opt::files) {
|
||||
|
|
@ -11001,7 +11005,7 @@ sub combine_ref($) {
|
|||
push @out, \$sep, $column;
|
||||
}
|
||||
}
|
||||
# Pop off a $sep
|
||||
# Remove the first $sep: ,val,"val" => val,"val"
|
||||
shift @out;
|
||||
return @out;
|
||||
}
|
||||
|
|
@ -11047,7 +11051,24 @@ sub print_files($) {
|
|||
}
|
||||
}
|
||||
|
||||
{
|
||||
my ($up,$eol,$reset_color,$init,$curseq,$maxseq);
|
||||
|
||||
sub print_linebuffer($) {
|
||||
sub print_llb($) {
|
||||
my $self = shift;
|
||||
my $out_fh = shift;
|
||||
my $str = shift;
|
||||
my $seq = $self->seq();
|
||||
$maxseq = $seq > $maxseq ? $seq : $maxseq;
|
||||
print($out_fh
|
||||
"$up"x($curseq - $seq),
|
||||
"\n"x($seq - $curseq),
|
||||
"\r", $self->tag(),
|
||||
$str, $eol, $reset_color,
|
||||
"\n"x($maxseq-$seq+1));
|
||||
$curseq = $maxseq + 1;
|
||||
}
|
||||
my $self = shift;
|
||||
my ($fdno,$in_fh,$out_fh) = @_;
|
||||
if(defined $self->{'exitstatus'}) {
|
||||
|
|
@ -11064,6 +11085,22 @@ sub print_linebuffer($) {
|
|||
}
|
||||
}
|
||||
}
|
||||
if(not $init) {
|
||||
$init = 1;
|
||||
if($Global::color or $opt::lastlinebuffer) {
|
||||
# cursor_up cuu1 = up one line
|
||||
$up = `tput cuu1 </dev/tty`;
|
||||
chomp($up);
|
||||
# clr_eol el = clear to end of line
|
||||
$eol = `tput el </dev/tty`;
|
||||
chomp($eol);
|
||||
# exit_attribute_mode sgr0 = turn off all attributes
|
||||
$reset_color = `tput sgr0 </dev/tty`;
|
||||
chomp($reset_color);
|
||||
$curseq = 1;
|
||||
$maxseq = 1;
|
||||
}
|
||||
}
|
||||
if(not $self->virgin()) {
|
||||
if($opt::files or ($opt::results and not $Global::csvsep)) {
|
||||
# Print filename
|
||||
|
|
@ -11087,7 +11124,8 @@ sub print_linebuffer($) {
|
|||
# It has been measured on:
|
||||
# AMD 6376: 60800 (>70k is also reasonable)
|
||||
# Intel i7-3632QM: 52-59k, 170-175k
|
||||
# seq 64 | ppar --test $1 --lb 'yes {} `seq 1000`|head -c 10000000' >/dev/null
|
||||
# seq 64 | ppar --_test $1 --lb \
|
||||
# 'yes {} `seq 1000`|head -c 10000000' >/dev/null
|
||||
while($rv = sysread($in_fh, $buf, 60800)) {
|
||||
$outputlength += $rv;
|
||||
# TODO --recend
|
||||
|
|
@ -11099,43 +11137,55 @@ sub print_linebuffer($) {
|
|||
# map { print $b,$_ } 1..10'
|
||||
$i = ((rindex($buf,"\n")+1) || (rindex($buf,"\r")+1));
|
||||
if($i) {
|
||||
if($opt::lastlinebuffer) {
|
||||
# Remove the final \n/\r
|
||||
my $l = join('', @$halfline_ref,
|
||||
substr($buf,0,$i-1));
|
||||
my $j = ((rindex($l,"\n")+1) ||
|
||||
(rindex($l,"\r")+1));
|
||||
$self->print_llb($out_fh,substr($l,$j));
|
||||
# Remove the printed part by keeping the unprinted
|
||||
@$halfline_ref = (substr($buf,$i));
|
||||
} else {
|
||||
# One or more complete lines were found
|
||||
if($Global::color) {
|
||||
# To paint a full line, we need to turn on paint
|
||||
# BEFORE printing \n
|
||||
# print @$halfline_ref, substr($buf,0,$i);
|
||||
my $print = join("",@$halfline_ref, substr($buf,0,$i));
|
||||
my $print = join("",@$halfline_ref,
|
||||
substr($buf,0,$i));
|
||||
chomp($print);
|
||||
# print color-on \n tag
|
||||
my $tag = $self->tag();
|
||||
# TODO fix \r
|
||||
$print =~ s/([\n\r])(?=.|$)/$tag/gs;
|
||||
print $out_fh $tag,$print;
|
||||
$print =~ s/([\n\r])(?=.|$)/$eol$reset_color$1$tag/gs;
|
||||
print $out_fh $tag, $print, $eol,
|
||||
$reset_color, "\n";
|
||||
} elsif($opt::tag or defined $opt::tagstring) {
|
||||
# Replace ^ with $tag within the full line
|
||||
if($Global::cache_replacement_eval) {
|
||||
# Replace with the same value for tag
|
||||
my $tag = $self->tag();
|
||||
unshift @$halfline_ref, $tag;
|
||||
# TODO --recend that can be partially in @$halfline_ref
|
||||
# TODO --recend that can be partially in
|
||||
# @$halfline_ref
|
||||
substr($buf,0,$i-1) =~
|
||||
s/(?<=[\n\r])(?=.|$)/$tag/gs;
|
||||
# The length changed, so find the new ending pos
|
||||
# The length changed,
|
||||
# so find the new ending pos
|
||||
$i = ::max((rindex($buf,"\n")+1),
|
||||
(rindex($buf,"\r")+1));
|
||||
} else {
|
||||
# Replace with freshly computed value of tag
|
||||
# Replace with freshly computed tag-value
|
||||
unshift @$halfline_ref, $self->tag();
|
||||
substr($buf,0,$i-1) =~
|
||||
s/(?<=[\n\r])(?=.|$)/$self->tag()/gse;
|
||||
# The length changed, so find the new ending pos
|
||||
# The length changed,
|
||||
# so find the new ending pos
|
||||
$i = ::max((rindex($buf,"\n")+1),
|
||||
(rindex($buf,"\r")+1));
|
||||
}
|
||||
# Print the partial line (halfline) and the last half
|
||||
# Print the partial line (halfline)
|
||||
# and the last half
|
||||
print $out_fh @$halfline_ref, substr($buf,0,$i);
|
||||
} else {
|
||||
# Print the partial line (halfline) and the last half
|
||||
# Print the partial line (halfline)
|
||||
# and the last half
|
||||
print $out_fh @$halfline_ref, substr($buf,0,$i);
|
||||
}
|
||||
# Buffer in memory for SQL and CSV-output
|
||||
|
|
@ -11143,8 +11193,9 @@ sub print_linebuffer($) {
|
|||
push(@{$self->{'output'}{$fdno}},
|
||||
@$halfline_ref, substr($buf,0,$i));
|
||||
}
|
||||
# Remove the printed part by keeping the unprinted part
|
||||
# Remove the printed part by keeping the unprinted
|
||||
@$halfline_ref = (substr($buf,$i));
|
||||
}
|
||||
} else {
|
||||
# No newline, so append to the halfline
|
||||
push @$halfline_ref, $buf;
|
||||
|
|
@ -11192,6 +11243,7 @@ sub print_linebuffer($) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub free_ressources() {
|
||||
my $self = shift;
|
||||
|
|
@ -11277,8 +11329,11 @@ sub print_normal($) {
|
|||
$outputlength += length $_;
|
||||
# Tag lines with \r, too
|
||||
$_ =~ s/(?<=[\r])(?=.|$)/$tag/gs;
|
||||
$Global::color and chomp();
|
||||
if($Global::color) {
|
||||
print_color($out_fh,$tag,$_);
|
||||
} else {
|
||||
print $out_fh $tag,$_;
|
||||
}
|
||||
if($Global::membuffer) {
|
||||
push @{$self->{'output'}{$fdno}}, $tag, $_;
|
||||
}
|
||||
|
|
@ -11384,34 +11439,60 @@ sub print_joblog($) {
|
|||
{
|
||||
my @color;
|
||||
my $color_on = "";
|
||||
my ($up,$eol,$reset_color,$init,$curseq,$maxseq);
|
||||
|
||||
sub tag($) {
|
||||
sub init_color() {
|
||||
$init = 1;
|
||||
# color combinations that are readable: black/white text
|
||||
# on colored background, but not white on yellow
|
||||
@color =
|
||||
my @color_combinations =
|
||||
# Force each color code to have the same length in chars
|
||||
# This will make \t work as expected
|
||||
((map { [sprintf("%03d",$_),"000"] }
|
||||
6..7,9..11,13..15,40..51,75..87,113..123,147..159,
|
||||
171..231,249..254),
|
||||
171..182,185..231,249..254),
|
||||
(map { [sprintf("%03d",$_),231] }
|
||||
1..9,12..13,16..45,52..81,88..114,124..149,153,
|
||||
160..178,180,182..185,187..186,189,196..214,232..250,
|
||||
253..254));
|
||||
1..9,12..13,16..45,52..81,88..114,124..149,
|
||||
160..178,180,182..184,196..214,232..250));
|
||||
# reorder list so adjacent colors are dissimilar
|
||||
# %23 and %7 were found experimentally
|
||||
@color = @color[
|
||||
sort { ($a%23 <=> $b%23) or ($b%7 <=> $a%7) } 0..$#color
|
||||
@color_combinations = @color_combinations[
|
||||
sort { ($a%23 <=> $b%23) or ($b%7 <=> $a%7) }
|
||||
0..$#color_combinations
|
||||
];
|
||||
@color = map {
|
||||
# TODO Can this be done with `tput` codes?
|
||||
"\033[48;5;".$_->[0].";38;5;".$_->[1]."m"
|
||||
} @color_combinations;
|
||||
|
||||
# cursor_up cuu1 = up one line
|
||||
$up = `tput cuu1 </dev/tty`;
|
||||
chomp($up);
|
||||
# clr_eol el = clear to end of line
|
||||
$eol = `tput el </dev/tty`;
|
||||
chomp($eol);
|
||||
# exit_attribute_mode sgr0 = turn off all attributes
|
||||
$reset_color = `tput sgr0 </dev/tty`;
|
||||
chomp($reset_color);
|
||||
$curseq = 1;
|
||||
$maxseq = 1;
|
||||
}
|
||||
|
||||
sub print_color() {
|
||||
# @text is a single line
|
||||
my ($out_fh,@text) = @_;
|
||||
chomp(@text);
|
||||
print $out_fh @text,$reset_color,"\n";
|
||||
|
||||
}
|
||||
|
||||
sub tag($) {
|
||||
my $self = shift;
|
||||
if(not defined $self->{'tag'} or not $Global::cache_replacement_eval) {
|
||||
if($Global::color) {
|
||||
if(not @color) { init_color() }
|
||||
if(not $init) { init_color() }
|
||||
# Choose a value based on the seq
|
||||
my $col = @color[$self->seq() % ($#color+1)];
|
||||
$color_on = "\033[48;5;".$col->[0].";38;5;".$col->[1]."m\n";
|
||||
$color_on = $color[$self->seq() % ($#color+1)].$eol;
|
||||
}
|
||||
if(defined $opt::tag or defined $opt::tagstring) {
|
||||
$self->{'tag'} =
|
||||
|
|
@ -14526,14 +14607,8 @@ sub main() {
|
|||
drain_job_queue(@command);
|
||||
::debug("init", "Done draining\n");
|
||||
reapers();
|
||||
if($Global::color) {
|
||||
print $Global::original_stderr "\033[00m\n";
|
||||
print "\033[00m\n";
|
||||
}
|
||||
::debug("init", "Done reaping\n");
|
||||
if($Global::semaphore) {
|
||||
$sem->release();
|
||||
}
|
||||
if($Global::semaphore) { $sem->release(); }
|
||||
cleanup();
|
||||
::debug("init", "Halt\n");
|
||||
halt();
|
||||
|
|
|
|||
|
|
@ -748,15 +748,13 @@ B<--transfer>, B<--transferfile> or B<--return>.
|
|||
See also: B<--basefile> B<--transfer> B<--transferfile> B<--sshlogin>
|
||||
B<--return>
|
||||
|
||||
=item B<--color>
|
||||
=item B<--color> (alpha testing)
|
||||
|
||||
Colour output.
|
||||
|
||||
Colour the output. Each job gets its own color combination
|
||||
(background+foreground).
|
||||
|
||||
See also: B<--ctag> B<--ctagstring>
|
||||
|
||||
|
||||
=item B<--colsep> I<regexp>
|
||||
|
||||
|
|
@ -1545,6 +1543,26 @@ Similar to B<--memfree>.
|
|||
See also: B<--memfree> B<--load>
|
||||
|
||||
|
||||
=item B<--last-line-buffer> (alpha testing)
|
||||
|
||||
=item B<--llb> (alpha testing)
|
||||
|
||||
Print the lastest line of each running job.
|
||||
|
||||
This only works if the currently running jobs fit on the screen.
|
||||
|
||||
Example:
|
||||
|
||||
slow_seq() {
|
||||
seq "$@" |
|
||||
perl -ne '$|=1; for(split//){ print; select($a,$a,$a,0.03);}'
|
||||
}
|
||||
export -f slow_seq
|
||||
parallel --shuf -j10 --llb --color slow_seq {} ::: {1..100}
|
||||
|
||||
See also: B<--line-buffer>
|
||||
|
||||
|
||||
=item B<--line-buffer>
|
||||
|
||||
=item B<--lb>
|
||||
|
|
@ -2720,7 +2738,7 @@ exit.
|
|||
|
||||
Only used with B<env_parallel>. Aliases, functions, and variables with
|
||||
names in B<$PARALLEL_IGNORED_NAMES> will not be copied. So you should
|
||||
set variables/function you want to use I<after> running B<--session>.
|
||||
set variables/function you want copied I<after> running B<--session>.
|
||||
|
||||
It is similar to B<--record-env>, but only for this session.
|
||||
|
||||
|
|
|
|||
|
|
@ -3375,7 +3375,7 @@ Summary (see legend above):
|
|||
B<p> is a tiny shell script. It can color output with some predefined
|
||||
colors, but is otherwise quite limited.
|
||||
|
||||
It maxes out at 116260 jobs (probably due to limitations in Bash).
|
||||
It maxes out at around 116000 jobs (probably due to limitations in Bash).
|
||||
|
||||
=head3 EXAMPLES FROM p
|
||||
|
||||
|
|
|
|||
|
|
@ -1209,7 +1209,6 @@ B<-->. I have seen B<::> used in programming languanges to separate
|
|||
classes, and I did not want the user to be confused that the separator
|
||||
had anything to do with classes.
|
||||
|
||||
|
||||
B<:::> also makes a visual separation, which is good if there are
|
||||
multiple B<:::>.
|
||||
|
||||
|
|
@ -1219,6 +1218,10 @@ Linking input sources meant having to decide for some way to indicate
|
|||
linking of B<:::> and B<::::>. B<:::+> and B<::::+> were chosen, so
|
||||
that they were similar to B<:::> and B<::::>.
|
||||
|
||||
In 2022 I realized that B<///> would have been an even better choice,
|
||||
because you cannot have an file named B<///> whereas you I<can> have a
|
||||
file named B<:::>.
|
||||
|
||||
|
||||
=head2 Perl replacement strings, {= =}, and --rpl
|
||||
|
||||
|
|
|
|||
11
src/parset
11
src/parset
|
|
@ -79,6 +79,7 @@ done
|
|||
|
||||
|
||||
cat <<'_EOS'
|
||||
|
||||
parset only works if it is a function. The function is defined as part
|
||||
of env_parallel.
|
||||
|
||||
|
|
@ -122,6 +123,16 @@ To install in all shells run:
|
|||
|
||||
parset --install
|
||||
|
||||
In a script you need to run this before using parset:
|
||||
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
|
||||
For details: see man parset
|
||||
|
||||
_EOS
|
||||
|
|
|
|||
|
|
@ -8,6 +8,23 @@
|
|||
# Each should be taking 3-10s and be possible to run in parallel
|
||||
# I.e.: No race conditions, no logins
|
||||
|
||||
par_llb_color() {
|
||||
echo 'bug #62386: --color (--ctag but without --tag)'
|
||||
echo 'bug #62438: See last line from multiple jobslots'
|
||||
slow_seq() {
|
||||
sleep 0.$1
|
||||
seq $1 | pv -qL $1
|
||||
}
|
||||
export -f slow_seq
|
||||
run() {
|
||||
seq 4 | parallel --color $@ slow_seq
|
||||
}
|
||||
export -f run
|
||||
parallel --delay 0.1 -vkj0 run \
|
||||
::: --lb --llb '' ::: --color '' ::: '--tagstring {}{}' --tag '' ::: -k '' |
|
||||
md5sum
|
||||
}
|
||||
|
||||
par_process_slot_var() {
|
||||
echo '### bug #62310: xargs compatibility: --process-slot-var=name'
|
||||
seq 0.1 0.1 0.5 |
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ rm -f /run/shm/parallel.db
|
|||
mkdir -p /run/shm/csv
|
||||
|
||||
p_showsqlresult() {
|
||||
# print results stored in $SERVERURL/$TABLE
|
||||
SERVERURL=$1
|
||||
TABLE=$2
|
||||
sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;"
|
||||
|
|
@ -25,23 +26,32 @@ p_showsqlresult() {
|
|||
p_wrapper() {
|
||||
INNER=$1
|
||||
SERVERURL=$(eval echo $2)
|
||||
# Use a random table for each test
|
||||
TABLE=TBL$RANDOM
|
||||
DBURL=$SERVERURL/$TABLE
|
||||
T1=$(tempfile)
|
||||
T2=$(tempfile)
|
||||
# Run $INNER (all the par_* functions)
|
||||
eval "$INNER"
|
||||
echo Exit=$?
|
||||
# $INNER can start background processes - wait for those
|
||||
wait
|
||||
echo Exit=$?
|
||||
# For debugging show the tempfiles
|
||||
$DEBUG && sort -u $T1 $T2;
|
||||
rm $T1 $T2
|
||||
p_showsqlresult $SERVERURL $TABLE
|
||||
# Drop the table if not debugging
|
||||
$DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
|
||||
}
|
||||
|
||||
p_template() {
|
||||
(sleep 6;
|
||||
parallel --sqlworker $DBURL "$@" sleep .3\;echo >$T1) &
|
||||
# Run the
|
||||
(
|
||||
# Make sure there is work to be done
|
||||
sleep 6;
|
||||
parallel --sqlworker $DBURL "$@" sleep .3\;echo >$T1
|
||||
) &
|
||||
parallel --sqlandworker $DBURL "$@" sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
|
||||
}
|
||||
|
||||
|
|
@ -123,6 +133,7 @@ par_shuf() {
|
|||
}
|
||||
|
||||
par_empty() {
|
||||
echo Do nothing: TBL99999 does not exist because it is not created
|
||||
true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -110,11 +110,9 @@ par_csv_pipe 9000"
|
|||
par_csv_pipe 11000"
|
||||
par_ctagstring ### --ctag --ctagstring should be different from --tag --tagstring
|
||||
par_ctagstring 8
|
||||
par_ctagstring [00m
|
||||
par_ctagstring 34
|
||||
par_ctagstring 37
|
||||
par_ctagstring 10
|
||||
par_ctagstring [00m
|
||||
par_ctagstring 36
|
||||
par_ctagstring 39
|
||||
par_delimiter ### Test --delimiter and -d: Delimiter instead of newline
|
||||
par_delimiter # Yes there is supposed to be an extra newline for -d N
|
||||
par_delimiter This is line 1
|
||||
|
|
@ -955,10 +953,10 @@ par_sem_quote ### sem --quote should not add empty argument
|
|||
par_sem_quote echo
|
||||
par_sem_quote
|
||||
par_shellcompletion ### --shellcompletion
|
||||
par_shellcompletion 9c26e11f6435802cd18a8fd78f65f71d -
|
||||
par_shellcompletion 9c26e11f6435802cd18a8fd78f65f71d -
|
||||
par_shellcompletion 2b8f8856cd7b632edec5526f8d3637c9 -
|
||||
par_shellcompletion 2b8f8856cd7b632edec5526f8d3637c9 -
|
||||
par_shellcompletion a040c070a388e8799d899dce6e4d478b -
|
||||
par_shellcompletion a040c070a388e8799d899dce6e4d478b -
|
||||
par_shellcompletion 7bd30aa4542083e298ef0d1a60dc1322 -
|
||||
par_shellcompletion 7bd30aa4542083e298ef0d1a60dc1322 -
|
||||
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
|
||||
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
|
||||
par_slow_pipe_regexp 0 1 1
|
||||
|
|
|
|||
|
|
@ -135,6 +135,9 @@ par_kill_term bash---pstree
|
|||
par_lb_mem_usage 1
|
||||
par_lb_mem_usage 1
|
||||
par_lb_mem_usage 2
|
||||
par_llb_color bug #62386: --color (--ctag but without --tag)
|
||||
par_llb_color bug #62438: See last line from multiple jobslots
|
||||
par_llb_color 74c668db859b3c8acbecbddf5880fd20 -
|
||||
par_maxargs ### Test -n and --max-args: Max number of args per line (only with -X and -m)
|
||||
par_maxargs line 1
|
||||
par_maxargs line 2
|
||||
|
|
|
|||
|
|
@ -270,120 +270,24 @@ par_append $CSV csv:///%2Frun%2Fshm is not a valid DBURL
|
|||
par_append $CSV
|
||||
par_append $CSV sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
|
||||
par_empty $MYSQL p_wrapper par_empty '$MYSQL'
|
||||
par_empty $MYSQL Do nothing: TBL99999 does not exist because it is not created
|
||||
par_empty $MYSQL Exit=0
|
||||
par_empty $MYSQL Exit=0
|
||||
par_empty $MYSQL ERROR 1146 (42S02) at line 1: Table 'tange.TBL99999' doesn't exist
|
||||
par_empty $PG p_wrapper par_empty '$PG'
|
||||
par_empty $PG Do nothing: TBL99999 does not exist because it is not created
|
||||
par_empty $PG Exit=0
|
||||
par_empty $PG Exit=0
|
||||
par_empty $PG host| command | v1 | v2 | stdout | stderr
|
||||
par_empty $PG hostname| sleep .3;echo 1 a | 1 | a | 1 a +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 1 b | 1 | b | 1 b +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 1 c | 1 | c | 1 c +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 1 d | 1 | d | 1 d +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 1 e | 1 | e | 1 e +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 2 a | 2 | a | 2 a +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 2 b | 2 | b | 2 b +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 2 c | 2 | c | 2 c +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 2 d | 2 | d | 2 d +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 2 e | 2 | e | 2 e +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 3 a | 3 | a | 3 a +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 3 b | 3 | b | 3 b +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 3 c | 3 | c | 3 c +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 3 d | 3 | d | 3 d +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 3 e | 3 | e | 3 e +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 4 a | 4 | a | 4 a +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 4 b | 4 | b | 4 b +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 4 c | 4 | c | 4 c +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 4 d | 4 | d | 4 d +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 4 e | 4 | e | 4 e +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 5 a | 5 | a | 5 a +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 5 b | 5 | b | 5 b +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 5 c | 5 | c | 5 c +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 5 d | 5 | d | 5 d +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 5 e | 5 | e | 5 e +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 11 A | 11 | A | 11 A +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 11 B | 11 | B | 11 B +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 11 C | 11 | C | 11 C +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 11 D | 11 | D | 11 D +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 11 E | 11 | E | 11 E +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 12 A | 12 | A | 12 A +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 12 B | 12 | B | 12 B +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 12 C | 12 | C | 12 C +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 12 D | 12 | D | 12 D +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 12 E | 12 | E | 12 E +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 13 A | 13 | A | 13 A +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 13 B | 13 | B | 13 B +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 13 C | 13 | C | 13 C +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 13 D | 13 | D | 13 D +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 13 E | 13 | E | 13 E +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 14 A | 14 | A | 14 A +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 14 B | 14 | B | 14 B +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 14 C | 14 | C | 14 C +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 14 D | 14 | D | 14 D +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 14 E | 14 | E | 14 E +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 15 A | 15 | A | 15 A +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 15 B | 15 | B | 15 B +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 15 C | 15 | C | 15 C +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 15 D | 15 | D | 15 D +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG hostname| sleep .3;echo 15 E | 15 | E | 15 E +|
|
||||
par_empty $PG | | | | |
|
||||
par_empty $PG (50 rows)
|
||||
par_empty $PG
|
||||
par_empty $PG ERROR: relation "TBL99999" does not exist
|
||||
par_empty $PG LINE 1: select Host,Command,V1,V2,Stdout,Stderr from TBL99999 order
|
||||
par_empty $PG ^
|
||||
par_empty $SQLITE p_wrapper par_empty '$SQLITE'
|
||||
par_empty $SQLITE Do nothing: TBL99999 does not exist because it is not created
|
||||
par_empty $SQLITE Exit=0
|
||||
par_empty $SQLITE Exit=0
|
||||
par_empty $SQLITE Error: near line 1: no such table: TBL99999
|
||||
par_empty $CSV p_wrapper par_empty '$CSV'
|
||||
par_empty $CSV Do nothing: TBL99999 does not exist because it is not created
|
||||
par_empty $CSV Exit=0
|
||||
par_empty $CSV Exit=0
|
||||
par_empty $CSV Error:
|
||||
|
|
|
|||
|
|
@ -761,6 +761,14 @@ tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
|
|||
Supports: variables, aliases, arrays with no special chars
|
||||
To install in all shells run:
|
||||
env_parallel --install
|
||||
In a script you need to run this before using env_parallel:
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
For details: see man env_parallel
|
||||
MYVAR='foo bar'
|
||||
export MYVAR
|
||||
|
|
@ -837,6 +845,14 @@ tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
|
|||
Supports: variables, aliases, arrays with no special chars
|
||||
To install in all shells run:
|
||||
env_parallel --install
|
||||
In a script you need to run this before using env_parallel:
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
For details: see man env_parallel
|
||||
parallel -vv --pipepart --block 1M wc :::: num30000
|
||||
<num30000 perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>32767?32767:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 168894 |(wc)
|
||||
|
|
@ -885,6 +901,14 @@ csh: Unsupported
|
|||
tcsh: Unsupported
|
||||
To install in all shells run:
|
||||
parset --install
|
||||
In a script you need to run this before using parset:
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
For details: see man parset
|
||||
parset myarray seq {} 5 ::: 1 2 3
|
||||
echo "${myarray[1]}"
|
||||
|
|
@ -925,6 +949,14 @@ csh: Unsupported
|
|||
tcsh: Unsupported
|
||||
To install in all shells run:
|
||||
parset --install
|
||||
In a script you need to run this before using parset:
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
For details: see man parset
|
||||
cmd=("echo '<<joe \"double space\" cartoon>>'" "pwd")
|
||||
parset data ::: "${cmd[@]}"
|
||||
|
|
@ -963,6 +995,14 @@ csh: Unsupported
|
|||
tcsh: Unsupported
|
||||
To install in all shells run:
|
||||
parset --install
|
||||
In a script you need to run this before using parset:
|
||||
bash: . `which env_parallel.bash`
|
||||
ksh: source `which env_parallel.ksh`
|
||||
mksh: source `which env_parallel.mksh`
|
||||
pdksh: source `which env_parallel.pdksh`
|
||||
zsh: . `which env_parallel.zsh`
|
||||
ash: . `which env_parallel.ash`
|
||||
dash: . `which env_parallel.dash`
|
||||
For details: see man parset
|
||||
parallel --sqlandworker csv:///%2Ftmp/log.csv \
|
||||
seq ::: 10 ::: 12 13 14
|
||||
|
|
@ -1286,4 +1326,4 @@ mentioned in the release notes of next version of GNU Parallel.
|
|||
echo A
|
||||
echo B
|
||||
echo C
|
||||
6
|
||||
8
|
||||
|
|
|
|||
Loading…
Reference in a new issue