parallel: -j evaluates expression: +3*log(55)% = ncpu*1.12.

This commit is contained in:
Ole Tange 2023-03-23 22:05:14 +01:00
parent 46d693ea34
commit 5f2e4c39ab
25 changed files with 209 additions and 201 deletions

View file

@ -121,7 +121,7 @@ contribution to the research is big enough to warrant a citation.
The citation is also needed for reproducibility: Let us assume a bug
in GNU Parallel skews the results. People replicating the research
needs to have the information, so they can replicate the (possibly
need to have the information, so they can replicate the (possibly
wrong) results.

View file

@ -4,6 +4,9 @@
Quote of the month:
parallel might be one of the best utilities out there
-- @ThePrimeagen ThePrimeagen
Recently learned how to use GNU parallel (from ChatGPT, no less!) and I've gone mad with power
-- Mark, Anthropomorphic Anuran @reject_resubmit@twitter

View file

@ -262,29 +262,25 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20230322 ('Arrest Warrant') released
Subject: GNU Parallel 20230422 ('Grand Jury<<>>') released
GNU Parallel 20230322 ('Arrest Warrant') has been released. It is available for download at: lbry://@GnuParallel:4
GNU Parallel 20230422 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
Quote of the month:
GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis
-- Best Catboy Key Grip @alamogordoglass@twitter
<<>>
New in this release:
* Better support for wide characters in --latest-line.
* Support for rsync 3.2.7.
<<>>
* Bug fixes and man page updates.
News about GNU Parallel:
* Analyzing multi-gigabyte JSON files locally https://thenybble.de/posts/json-analysis/
* 5 great Perl scripts to keep in your sysadmin toolbox https://www.redhat.com/sysadmin/perl-scripts
https://bash-prompt.net/guides/parallell-bash/
https://medium.com/linuxstories/bash-parallel-command-execution-d4bd7c7cc1d6
<<>>
GNU Parallel - For people who live life in the parallel lane.

View file

@ -393,7 +393,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -395,7 +395,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -393,7 +393,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -376,7 +376,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -378,7 +378,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -393,7 +393,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -368,7 +368,7 @@ _parset_main() {
fi
if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -26,7 +26,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20230322;
$Global::version = 20230323;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($opt::version) {

View file

@ -1799,10 +1799,6 @@ sub options_completion_hash() {
"[Use the replacement string replace-str instead of {%} for job ".
"slot number]:replace-str"
=> \$opt::slotreplace),
("jobs|j=s".
"[(Add +N to/Subtract -N from/Multiply N%) the number of CPU ".
"threads or read parameter from file]:_files"
=> \$opt::jobs),
("delay=s".
"[Delay starting next job by duration]:duration" => \$opt::delay),
("ssh-delay|sshdelay=f".
@ -1985,7 +1981,7 @@ sub options_completion_hash() {
("term-seq|termseq=s".
"[Termination sequence]:sequence" => \$opt::termseq),
# xargs-compatibility - implemented, man, testsuite
("max-procs|maxprocs|P=s".
("max-procs|maxprocs|P|jobs|j=s".
"[Add N to/Subtract N from/Multiply N% with/ the number of CPU ".
"threads or read parameter from file]:+N/-N/N%/N/procfile:_files"
=> \$opt::jobs),
@ -2766,7 +2762,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20230322;
$Global::version = 20230323;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -8245,28 +8241,7 @@ sub user_requested_processes($) {
my $opt_P = shift;
my $processes;
if(defined $opt_P) {
if($opt_P =~ /^\+(\d+)$/) {
# E.g. -P +2
my $j = $1;
$processes =
$self->ncpus() + $j;
} elsif ($opt_P =~ /^-(\d+)$/) {
# E.g. -P -2
my $j = $1;
$processes =
$self->ncpus() - $j;
} elsif ($opt_P =~ /^(\d+(\.\d+)?)\%$/) {
# E.g. -P 10.5%
my $j = $1;
$processes =
$self->ncpus() * $j / 100;
} elsif ($opt_P =~ /^(\d+)$/) {
$processes = $1;
if($processes == 0) {
# -P 0 = infinity (or at least close)
$processes = $Global::infinity;
}
} elsif (-f $opt_P) {
if (-f $opt_P) {
$Global::max_procs_file = $opt_P;
if(open(my $in_fh, "<", $Global::max_procs_file)) {
my $opt_P_file = join("",<$in_fh>);
@ -8277,8 +8252,15 @@ sub user_requested_processes($) {
::wait_and_exit(255);
}
} else {
::error("Parsing of --jobs/-j/--max-procs/-P failed.");
::die_usage();
# -P +3 and -P -1
$opt_P =~ s/^([-+])/\$self->ncpus()$1/;
# -P 40%
$opt_P =~ s:%$:*\$self->ncpus()/100:;
$processes = eval $opt_P;
if($processes == 0) {
# -P 0 = infinity (or at least close)
$processes = $Global::infinity;
}
}
$processes = ::ceil($processes);
}
@ -11452,11 +11434,11 @@ sub print_files($) {
if(::mbswidth($str) == length($str)) {
$str = substr($str,0,$len);
} else {
# Highly inefficient truncator
$str=~s/(.{$len}).*/$1/;
my $rlen = (::mbswidth($str) - $len)/2;
$str=~s/.{$rlen}$//;
while(::mbswidth($str) > $len) {
do {
chop $str;
} while(::mbswidth($str) < 0);
$str=~s/.$//;
}
}
return $str;
@ -11479,8 +11461,9 @@ sub print_files($) {
}
my ($color,$reset_color) = $self->color();
my $termcol = ::terminal_columns();
my $untabify_tag = $self->untabtag();
my $untabify_str = ::untabify($self->{$out_fh,'latestline'});
my $untabify_tag = ::decode_utf8($self->untabtag());
my $untabify_str =
::untabify(::decode_utf8($self->{$out_fh,'latestline'}));
# -1 to make space for $truncated_str
my $maxtaglen = $termcol - 1;
$untabify_tag = truncate_mbs($untabify_tag,$maxtaglen);
@ -11505,8 +11488,8 @@ sub print_files($) {
"%s" # down
),
"$up"x($currow - $row), "\n"x($row - $currow), "\r", $eol,
::decode_utf8($untabify_tag),$truncated_tag,
$color, ::decode_utf8($untabify_str), $truncated_str, $reset_color,
$untabify_tag,$truncated_tag,
$color, $untabify_str, $truncated_str, $reset_color,
"\n"x($maxrow - $row + 1));
$currow = $maxrow + 1;
}
@ -15128,8 +15111,6 @@ sub main() {
# Parallel check all hosts are up. Remove hosts that are down
filter_hosts();
}
if($opt::sqlmaster and $opt::sqlworker) {
# Start a real --sqlworker in the background later
$Global::start_sqlworker = 1;
@ -15184,7 +15165,6 @@ sub main() {
# Compute $Global::max_jobs_running
# Must be done after ungetting any --pipepart jobs.
max_jobs_running();
init_run_jobs();
my $sem;
if($Global::semaphore) {

View file

@ -1375,19 +1375,27 @@ If the host is long, you can use B<column -t> to pretty print it:
See also: B<--resume> B<--resume-failed>
=item B<--jobs> I<N>
=item B<--jobs> I<num> (alpha testing)
=item B<-j> I<N>
=item B<-j> I<num> (alpha testing)
=item B<--max-procs> I<N>
=item B<--max-procs> I<num> (alpha testing)
=item B<-P> I<N>
=item B<-P> I<num> (alpha testing)
Number of jobslots on each machine.
Run up to N jobs in parallel. 0 means as many as possible (this can
take a while to determine). Default is 100% which will run one job per
CPU thread on each machine.
Run up to I<num> jobs in parallel. Default is 100%.
=over 7
=item I<num>
Run up to I<num> jobs in parallel.
=item Z<>0
Run as many as possible (this can take a while to determine).
Due to a bug B<-j 0> will also evaluate replacement strings twice up
to the number of joblots:
@ -1397,81 +1405,49 @@ to the number of joblots:
# This will count from 1
seq 10000 | parallel -j100 echo '{= $_ = $foo++; =}' | head
If B<--semaphore> is set, the default is 1 thus making a mutex.
=item I<num>%
See also: B<--use-cores-instead-of-threads>
B<--use-sockets-instead-of-threads>
Multiply the number of CPU threads by I<num> percent. E.g. 100% means
one job per CPU thread on each machine.
=item +I<num>
Add I<num> to the number of CPU threads.
=item B<--jobs> I<+N>
=item -I<num>
=item B<-j> I<+N>
Subtract I<num> from the number of CPU threads.
=item B<--max-procs> I<+N>
=item I<expr>
=item B<-P> I<+N>
Evaluate I<expr>. E.g. '12/2' to get 6, '+25%' gives the same as
'125%', or complex expressions like '+3*log(55)%' which means:
multiply 3 by log(55), multiply that by the number of CPU threads and
divide by 100, add this to the number of CPU threads.
Add N to the number of CPU threads.
Run this many jobs in parallel.
See also: B<--number-of-threads> B<--number-of-cores>
B<--number-of-sockets>
=item B<--jobs> I<-N>
=item B<-j> I<-N>
=item B<--max-procs> I<-N>
=item B<-P> I<-N>
Subtract N from the number of CPU threads.
Run this many jobs in parallel. If the evaluated number is less than
1 then 1 will be used.
See also: B<--number-of-threads> B<--number-of-cores>
B<--number-of-sockets>
=item B<--jobs> I<N>%
=item B<-j> I<N>%
=item B<--max-procs> I<N>%
=item B<-P> I<N>%
Multiply N% with the number of CPU threads.
Run this many jobs in parallel.
See also: B<--number-of-threads> B<--number-of-cores>
B<--number-of-sockets>
=item B<--jobs> I<procfile>
=item B<-j> I<procfile>
=item B<--max-procs> I<procfile>
=item B<-P> I<procfile>
=item I<procfile>
Read parameter from file.
Use the content of I<procfile> as parameter for
I<-j>. E.g. I<procfile> could contain the string 100% or +2 or 10. If
I<procfile> is changed when a job completes, I<procfile> is read again
and the new number of jobs is computed. If the number is lower than
before, running jobs will be allowed to finish but new jobs will not
be started until the wanted number of jobs has been reached. This
I<-j>. E.g. I<procfile> could contain the string 100% or +2 or 10.
If I<procfile> is changed when a job completes, I<procfile> is read
again and the new number of jobs is computed. If the number is lower
than before, running jobs will be allowed to finish but new jobs will
not be started until the wanted number of jobs has been reached. This
makes it possible to change the number of simultaneous running jobs
while GNU B<parallel> is running.
=back
If the evaluated number is less than 1 then 1 will be used.
If B<--semaphore> is set, the default is 1 thus making a mutex.
See also: B<--use-cores-instead-of-threads>
B<--use-sockets-instead-of-threads>
=item B<--keep-order>

View file

@ -3917,6 +3917,10 @@ https://github.com/jkool702/forkrun
=head2 Todo
https://www.npmjs.com/package/concurrently
https://github.com/thilinaba/bash-parallel
http://code.google.com/p/push/ (cannot compile)
https://github.com/krashanoff/parallel

View file

@ -161,7 +161,7 @@ line number and the URL.
cat urlfile | parallel "wget {} 2>/dev/null || grep -n {} urlfile"
Create a mirror directory with the same filenames except all files and
Create a mirror directory with the same file names except all files and
symlinks are empty files.
cp -rs /the/source/dir mirror_dir
@ -288,6 +288,21 @@ Put all converted in the same directory:
parallel lame {} -o mydir/{/.}.mp3
=head2 EXAMPLE: Replacing parts of file names
If you deal with paired end reads, you will have files like
barcode1_R1.fq.gz, barcode1_R2.fq.gz, barcode2_R1.fq.gz, and
barcode2_R2.fq.gz.
You want barcodeI<N>_R1 to be processed with barcodeI<N>_R2.
parallel --plus myprocess {} {/_R1.fq.gz/_R2.fq.gz} ::: *_R1.fq.gz
If the barcode does not contain '_R1', you can do:
parallel --plus myprocess {} {/_R1/_R2} ::: *_R1.fq.gz
=head2 EXAMPLE: Removing strings from the argument
If you have directory with tar.gz files and want these extracted in
@ -1719,7 +1734,7 @@ Here B<python>:
print "exclusively";
=head2 EXAMPLE: Start editor with filenames from stdin (standard input)
=head2 EXAMPLE: Start editor with file names from stdin (standard input)
You can use GNU B<parallel> to start interactive programs like emacs or vi:

View file

@ -137,7 +137,7 @@ GetOptions(
"help" => \$opt::dummy,
) || exit(255);
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
$Global::version = 20230322;
$Global::version = 20230323;
if($opt::version) { version(); exit 0; }
# Remove -D and --parallel=N
my @s = (grep { ! /^-D$|^--parallel=\S+$/ }

View file

@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20230322;
$Global::version = 20230323;
$Global::progname = 'sql';
# This must be done first as this may exec myself

View file

@ -39,7 +39,7 @@ linebuffer_matters() {
# Random data because it does not compress well
# forcing the compress tool to spit out compressed blocks
perl -pe 'y/[A-Za-z]//cd; $t++ % 1000 or print "\n"' < /dev/urandom |
head -c 10000000 > $randomfile
head -c 10000000 > "$randomfile"
export randomfile
testfunc() {
@ -49,10 +49,10 @@ linebuffer_matters() {
# generate some incompressible ascii
# with lines starting with the same string
id=$1
shuf $randomfile | perl -pe 's/^/'$id' /'
shuf "$randomfile" | perl -pe 's/^/'$id' /'
# Sleep to give time to linebuffer-print the first part
sleep 10
shuf $randomfile | perl -pe 's/^/'$id' /'
shuf "$randomfile" | perl -pe 's/^/'$id' /'
echo
}
export -f incompressible_ascii
@ -72,15 +72,15 @@ linebuffer_matters() {
}
# These can run in parallel if there are enough ressources
testfunc > $nolbfile
testfunc > $controlfile
testfunc --linebuffer > $lbfile
testfunc > "$nolbfile"
testfunc > "$controlfile"
testfunc --linebuffer > "$lbfile"
wait
nolb="$(cat $nolbfile)"
control="$(cat $controlfile)"
lb="$(cat $lbfile)"
rm $nolbfile $lbfile $controlfile $randomfile
nolb="$(cat "$nolbfile")"
control="$(cat "$controlfile")"
lb="$(cat "$lbfile")"
rm "$nolbfile" "$lbfile" "$controlfile" "$randomfile"
if [ "$nolb" == "$control" ] ; then
if [ "$lb" == "$nolb" ] ; then

View file

@ -18,38 +18,55 @@ par_compare_exit_codes() {
echo '### compare the exit codes'
echo 'directly from shells, shells called from parallel,'
echo 'killed with different signals'
echo
# run:
# * exit $val
# * kill -$1 $$'
# both:
# * under different shells
# * from parallel running under different shells
# record:
# * exit value
# * signal
echo sig=joblog_sig shell=parallel=joblog
# The quoting of \n is wrong for csh, so just use a sane TMPDIR here
TMPDIR=/tmp/par-300s
export TMPDIR
mkdir -p "$TMPDIR"
selfkill=$(mktemp)
export selfkill
echo 'kill -$1 $$' >$selfkill
qselfkill=$(parallel -0 --shellquote ::: "$selfkill")
qqselfkill=$(parallel -0 --shellquote --shellquote ::: "$selfkill")
export selfkill qselfkill qqselfkill
echo 'kill -$1 $$' >"$selfkill"
exit=$(mktemp)
export exit
echo 'exit $1' >$exit
qexit=$(parallel -0 --shellquote ::: "$exit")
qqexit=$(parallel -0 --shellquote --shellquote ::: "$exit")
export exit qexit qqexit
echo 'exit $1' >"$exit"
doit() {
shell=$1
sig=$2
sig128=$(( sig + 128 ))
sh -c "$shell $selfkill $sig" 2>/dev/null
sh -c "$shell $qselfkill $sig" 2>/dev/null
raw=$?
sh -c "$shell $exit $sig128" 2>/dev/null
sh -c "$shell $qexit $sig128" 2>/dev/null
raw128=$?
log=$(mktemp)
$shell -c "parallel --halt now,done=1 --jl $log $shell $selfkill ::: $sig" 2>/dev/null
qlog=$(parallel -0 --shellquote ::: "$log")
qqlog=$(parallel -0 --shellquote --shellquote ::: "$log")
$shell -c "parallel --halt now,done=1 --jl $qlog $shell $qqselfkill ::: $sig" 2>/dev/null
#echo parallel $shell $sig = $?
parallel=$?
joblog_exit=$(field 7 < $log | tail -n1)
joblog_signal=$(field 8 < $log | tail -n1)
$shell -c "parallel --halt now,done=1 --jl $log $shell $exit ::: $sig128" 2>/dev/null
joblog_exit=$(field 7 < "$log" | tail -n1)
joblog_signal=$(field 8 < "$log" | tail -n1)
$shell -c "parallel --halt now,done=1 --jl $qlog $shell $qqexit ::: $sig128" 2>/dev/null
parallel128=$?
joblog_exit128=$(field 7 < $log | tail -n1)
joblog_signal128=$(field 8 < $log | tail -n1)
#echo joblog p $shell $sig $(field 8,7 < $log | tail -n1)
rm $log
joblog_exit128=$(field 7 < "$log" | tail -n1)
joblog_signal128=$(field 8 < "$log" | tail -n1)
rm "$log"
echo $shell sig' ' $sig=$joblog_signal $raw=$parallel=$joblog_exit
echo $shell exit $sig128=$joblog_signal128 $raw128=$parallel128=$joblog_exit128
@ -60,17 +77,17 @@ par_compare_exit_codes() {
OK="ash csh dash fish fizsh posh rc sash sh tcsh"
# These do not give the same exit code prepended with 'true;' or not
BAD="bash ksh93 mksh static-sh yash zsh"
(
# Most block on signals: 19+20+21+22
ulimit -n `ulimit -Hn`
parallel -j1000% -k doit ::: $OK $BAD ::: {1..18} {23..64}
# fdsh blocks on a lot more signals
parallel -j1000% -k doit ::: fdsh ::: 2 {9..12} {14..18} {20..23} 26 27 29 30 {32..64}
) |
# Ignore where the exit codes are the same
) |
# Ignore lines where the exit codes are the same
perl -ne '/(\d+)=\1=\1/ or print'
rm $selfkill
rm "$selfkill" "$exit"
rmdir "$TMPDIR"
}
par_retries_unreachable() {

View file

@ -10,14 +10,25 @@ par_ll_asian() {
echo '### --ll with Asian wide chars mess up display'
echo 'bug #63878: Wide East Asian chars in --latest-line'
p="parallel --ll --color --tag"
COLUMNS=80 $p echo tag fits, line fits a{}b{}c \
::: ヌー平行
COLUMNS=80 $p echo tag fits, line too long a{}b{}c \
::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー
COLUMNS=80 $p echo tag too long a{}b{}c \
echo Oops: the first adds '>' too early
COLUMNS=50 $p echo tag fits, line fits a{}b{}c \
::: ヌー平
COLUMNS=50 $p echo tag fits, line too long a{}b{}c \
::: ヌー平行ヌー平行ヌー平行ヌー平行ヌ
COLUMNS=50 $p echo tag too long a{}b{}c \
::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行a
}
par_mbswidth() {
echo '### characters with screen width > 1'
perl -e '@a=qw(ヌ ー 平 行.);
print map {
(join"",map{ $a[$_% $#a] } (1..$_))."\n".
"a".(join"",map{ $a[$_% $#a] } (1..$_))."\n"
} (1..40)' |
COLUMNS=50 parallel -k --ll --color --tag echo
}
par_ll_tag() {
parallel --tag --ll -q printf "a\n{}\n" ::: should-be-tagged-A
parallel --tag --ll -q printf "a\n\r{}\n" ::: should-be-tagged-B

View file

@ -83,6 +83,9 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' "$testsuit
s/(std(out|err)|seq): Permission denied/$1: No such file or directory/;
# Race condition
s/^4-(middle|end)\n//;
# Race condition
s/^parallel: This job failed:\n//;
s/^echo .; exit .\n//;
# Base 64 string with quotes
s:['"'"'"\\+/a-z.0-9=]{50,}(\s['"'"'"\\+/a-z.0-9=]*)*:BASE64:ig;
# Timings are often off

View file

@ -976,10 +976,10 @@ par_sem_quote ### sem --quote should not add empty argument
par_sem_quote echo
par_sem_quote
par_shellcompletion ### --shellcompletion
par_shellcompletion 0097e98f494e6288b33f3201dda7ee11 -
par_shellcompletion 0097e98f494e6288b33f3201dda7ee11 -
par_shellcompletion 1655cb29ff14d0008866108c6b5262f7 -
par_shellcompletion 1655cb29ff14d0008866108c6b5262f7 -
par_shellcompletion 70960cbdbc411e041161ae228f029d70 -
par_shellcompletion 70960cbdbc411e041161ae228f029d70 -
par_shellcompletion aa125ab894780611a20bad4a52d7a58d -
par_shellcompletion aa125ab894780611a20bad4a52d7a58d -
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
par_slow_pipe_regexp 0 1 1

View file

@ -3,99 +3,115 @@ par_linebuffer_files bug #48658: --linebuffer --files
par_linebuffer_files zstd normal
par_linebuffer_files zstd 100000
par_linebuffer_files zstd --files
par_linebuffer_files zstd 1
par_linebuffer_files zstd 4
par_linebuffer_files zstd --results
par_linebuffer_files zstd 1
par_linebuffer_files zstd parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files pzstd normal
par_linebuffer_files pzstd 100000
par_linebuffer_files pzstd --files
par_linebuffer_files pzstd 1
par_linebuffer_files pzstd 4
par_linebuffer_files pzstd --results
par_linebuffer_files pzstd 1
par_linebuffer_files pzstd parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files clzip normal
par_linebuffer_files clzip 100000
par_linebuffer_files clzip --files
par_linebuffer_files clzip 1
par_linebuffer_files clzip 4
par_linebuffer_files clzip --results
par_linebuffer_files clzip 1
par_linebuffer_files clzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files lz4 normal
par_linebuffer_files lz4 100000
par_linebuffer_files lz4 --files
par_linebuffer_files lz4 1
par_linebuffer_files lz4 4
par_linebuffer_files lz4 --results
par_linebuffer_files lz4 1
par_linebuffer_files lz4 parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files lzop normal
par_linebuffer_files lzop 100000
par_linebuffer_files lzop --files
par_linebuffer_files lzop 1
par_linebuffer_files lzop 4
par_linebuffer_files lzop --results
par_linebuffer_files lzop 1
par_linebuffer_files lzop parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files pigz normal
par_linebuffer_files pigz 100000
par_linebuffer_files pigz --files
par_linebuffer_files pigz 1
par_linebuffer_files pigz 4
par_linebuffer_files pigz --results
par_linebuffer_files pigz 1
par_linebuffer_files pigz parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files pxz normal
par_linebuffer_files pxz 100000
par_linebuffer_files pxz --files
par_linebuffer_files pxz 1
par_linebuffer_files pxz 4
par_linebuffer_files pxz --results
par_linebuffer_files pxz 1
par_linebuffer_files pxz parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files gzip normal
par_linebuffer_files gzip 100000
par_linebuffer_files gzip --files
par_linebuffer_files gzip 1
par_linebuffer_files gzip 4
par_linebuffer_files gzip --results
par_linebuffer_files gzip 1
par_linebuffer_files gzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files plzip normal
par_linebuffer_files plzip 100000
par_linebuffer_files plzip --files
par_linebuffer_files plzip 1
par_linebuffer_files plzip 4
par_linebuffer_files plzip --results
par_linebuffer_files plzip 1
par_linebuffer_files plzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files pbzip2 normal
par_linebuffer_files pbzip2 100000
par_linebuffer_files pbzip2 --files
par_linebuffer_files pbzip2 1
par_linebuffer_files pbzip2 4
par_linebuffer_files pbzip2 --results
par_linebuffer_files pbzip2 1
par_linebuffer_files pbzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files lzma normal
par_linebuffer_files lzma 100000
par_linebuffer_files lzma --files
par_linebuffer_files lzma 1
par_linebuffer_files lzma 4
par_linebuffer_files lzma --results
par_linebuffer_files lzma 1
par_linebuffer_files lzma parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files xz normal
par_linebuffer_files xz 100000
par_linebuffer_files xz --files
par_linebuffer_files xz 1
par_linebuffer_files xz 4
par_linebuffer_files xz --results
par_linebuffer_files xz 1
par_linebuffer_files xz parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files lzip normal
par_linebuffer_files lzip 100000
par_linebuffer_files lzip --files
par_linebuffer_files lzip 1
par_linebuffer_files lzip 4
par_linebuffer_files lzip --results
par_linebuffer_files lzip 1
par_linebuffer_files lzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files bzip2 normal
par_linebuffer_files bzip2 100000
par_linebuffer_files bzip2 --files
par_linebuffer_files bzip2 1
par_linebuffer_files bzip2 4
par_linebuffer_files bzip2 --results
par_linebuffer_files bzip2 1
par_linebuffer_files bzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files lbzip2 normal
par_linebuffer_files lbzip2 100000
par_linebuffer_files lbzip2 --files
par_linebuffer_files lbzip2 1
par_linebuffer_files lbzip2 4
par_linebuffer_files lbzip2 --results
par_linebuffer_files lbzip2 1
par_linebuffer_files lbzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_files lrz normal
par_linebuffer_files lrz 100000
par_linebuffer_files lrz --files
par_linebuffer_files lrz 1
par_linebuffer_files lrz 4
par_linebuffer_files lrz --results
par_linebuffer_files lrz 1
par_linebuffer_files lrz parallel: Warning: Use --files0 when $TMPDIR contains newline.
par_linebuffer_matters_compress ### (--linebuffer) --compress should give different output
par_linebuffer_matters_compress OK: --linebuffer makes a difference
par_linebuffer_matters_compress_tag ### (--linebuffer) --compress --tag should give different output

View file

@ -318,7 +318,7 @@ par_test_build_and_install make distdir-am
par_test_build_and_install make[0]: Entering directory '~/privat/parallel/src'
par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src'
par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src'
par_test_build_and_install test -n "" \
par_test_build_and_install test -n " \
par_test_build_and_install || find "parallel-00000000" -type d ! -perm -755 \
par_test_build_and_install -exec chmod u+rwx,go+rx {} \; -o \
par_test_build_and_install ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \

View file

@ -606,29 +606,16 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
9
parallel: This job failed:
echo X; exit X
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo X; exit X
parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
9
parallel: This job failed:
echo X; exit X
parallel -j2 --halt soon,fail=20% echo {}\; exit {} \
::: 0 1 2 3 4 5 6 7 8 9
9
parallel: This job failed:
echo X; exit X
parallel: This job failed:
echo X; exit X
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo X; exit X
parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
9
parallel: This job succeeded:
echo X; exit X
parallel -k --retries 3 \
'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
cat /tmp/runs