mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-23 22:47:55 +00:00
parallel: -j evaluates expression: +3*log(55)% = ncpu*1.12.
This commit is contained in:
Ole Tange
parent
46d693ea34
commit
5f2e4c39ab
|
|
@ -121,7 +121,7 @@ contribution to the research is big enough to warrant a citation.
|
|||
|
||||
The citation is also needed for reproducibility: Let us assume a bug
|
||||
in GNU Parallel skews the results. People replicating the research
|
||||
needs to have the information, so they can replicate the (possibly
|
||||
need to have the information, so they can replicate the (possibly
|
||||
wrong) results.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@
|
|||
|
||||
Quote of the month:
|
||||
|
||||
parallel might be one of the best utilities out there
|
||||
-- @ThePrimeagen ThePrimeagen
|
||||
|
||||
Recently learned how to use GNU parallel (from ChatGPT, no less!) and I've gone mad with power
|
||||
-- Mark, Anthropomorphic Anuran @reject_resubmit@twitter
|
||||
|
||||
|
|
|
|||
|
|
@ -262,29 +262,25 @@ from:tange@gnu.org
|
|||
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
Subject: GNU Parallel 20230322 ('Arrest Warrant') released
|
||||
Subject: GNU Parallel 20230422 ('Grand Jury<<>>') released
|
||||
|
||||
GNU Parallel 20230322 ('Arrest Warrant') has been released. It is available for download at: lbry://@GnuParallel:4
|
||||
GNU Parallel 20230422 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
|
||||
|
||||
Quote of the month:
|
||||
|
||||
GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis
|
||||
-- Best Catboy Key Grip @alamogordoglass@twitter
|
||||
<<>>
|
||||
|
||||
New in this release:
|
||||
|
||||
* Better support for wide characters in --latest-line.
|
||||
|
||||
* Support for rsync 3.2.7.
|
||||
|
||||
<<>>
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
News about GNU Parallel:
|
||||
|
||||
* Analyzing multi-gigabyte JSON files locally https://thenybble.de/posts/json-analysis/
|
||||
|
||||
* 5 great Perl scripts to keep in your sysadmin toolbox https://www.redhat.com/sysadmin/perl-scripts
|
||||
https://bash-prompt.net/guides/parallell-bash/
|
||||
https://medium.com/linuxstories/bash-parallel-command-execution-d4bd7c7cc1d6
|
||||
|
||||
<<>>
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -395,7 +395,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -378,7 +378,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -368,7 +368,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20230322 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20230323 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20230322;
|
||||
$Global::version = 20230323;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
62
src/parallel
62
src/parallel
|
|
@ -1799,10 +1799,6 @@ sub options_completion_hash() {
|
|||
"[Use the replacement string replace-str instead of {%} for job ".
|
||||
"slot number]:replace-str"
|
||||
=> \$opt::slotreplace),
|
||||
("jobs|j=s".
|
||||
"[(Add +N to/Subtract -N from/Multiply N%) the number of CPU ".
|
||||
"threads or read parameter from file]:_files"
|
||||
=> \$opt::jobs),
|
||||
("delay=s".
|
||||
"[Delay starting next job by duration]:duration" => \$opt::delay),
|
||||
("ssh-delay|sshdelay=f".
|
||||
|
|
@ -1985,7 +1981,7 @@ sub options_completion_hash() {
|
|||
("term-seq|termseq=s".
|
||||
"[Termination sequence]:sequence" => \$opt::termseq),
|
||||
# xargs-compatibility - implemented, man, testsuite
|
||||
("max-procs|maxprocs|P=s".
|
||||
("max-procs|maxprocs|P|jobs|j=s".
|
||||
"[Add N to/Subtract N from/Multiply N% with/ the number of CPU ".
|
||||
"threads or read parameter from file]:+N/-N/N%/N/procfile:_files"
|
||||
=> \$opt::jobs),
|
||||
|
|
@ -2766,7 +2762,7 @@ sub check_invalid_option_combinations() {
|
|||
|
||||
sub init_globals() {
|
||||
# Defaults:
|
||||
$Global::version = 20230322;
|
||||
$Global::version = 20230323;
|
||||
$Global::progname = 'parallel';
|
||||
$::name = "GNU Parallel";
|
||||
$Global::infinity = 2**31;
|
||||
|
|
@ -8245,28 +8241,7 @@ sub user_requested_processes($) {
|
|||
my $opt_P = shift;
|
||||
my $processes;
|
||||
if(defined $opt_P) {
|
||||
if($opt_P =~ /^\+(\d+)$/) {
|
||||
# E.g. -P +2
|
||||
my $j = $1;
|
||||
$processes =
|
||||
$self->ncpus() + $j;
|
||||
} elsif ($opt_P =~ /^-(\d+)$/) {
|
||||
# E.g. -P -2
|
||||
my $j = $1;
|
||||
$processes =
|
||||
$self->ncpus() - $j;
|
||||
} elsif ($opt_P =~ /^(\d+(\.\d+)?)\%$/) {
|
||||
# E.g. -P 10.5%
|
||||
my $j = $1;
|
||||
$processes =
|
||||
$self->ncpus() * $j / 100;
|
||||
} elsif ($opt_P =~ /^(\d+)$/) {
|
||||
$processes = $1;
|
||||
if($processes == 0) {
|
||||
# -P 0 = infinity (or at least close)
|
||||
$processes = $Global::infinity;
|
||||
}
|
||||
} elsif (-f $opt_P) {
|
||||
if (-f $opt_P) {
|
||||
$Global::max_procs_file = $opt_P;
|
||||
if(open(my $in_fh, "<", $Global::max_procs_file)) {
|
||||
my $opt_P_file = join("",<$in_fh>);
|
||||
|
|
@ -8277,8 +8252,15 @@ sub user_requested_processes($) {
|
|||
::wait_and_exit(255);
|
||||
}
|
||||
} else {
|
||||
::error("Parsing of --jobs/-j/--max-procs/-P failed.");
|
||||
::die_usage();
|
||||
# -P +3 and -P -1
|
||||
$opt_P =~ s/^([-+])/\$self->ncpus()$1/;
|
||||
# -P 40%
|
||||
$opt_P =~ s:%$:*\$self->ncpus()/100:;
|
||||
$processes = eval $opt_P;
|
||||
if($processes == 0) {
|
||||
# -P 0 = infinity (or at least close)
|
||||
$processes = $Global::infinity;
|
||||
}
|
||||
}
|
||||
$processes = ::ceil($processes);
|
||||
}
|
||||
|
|
@ -11452,11 +11434,11 @@ sub print_files($) {
|
|||
if(::mbswidth($str) == length($str)) {
|
||||
$str = substr($str,0,$len);
|
||||
} else {
|
||||
# Highly inefficient truncator
|
||||
$str=~s/(.{$len}).*/$1/;
|
||||
my $rlen = (::mbswidth($str) - $len)/2;
|
||||
$str=~s/.{$rlen}$//;
|
||||
while(::mbswidth($str) > $len) {
|
||||
do {
|
||||
chop $str;
|
||||
} while(::mbswidth($str) < 0);
|
||||
$str=~s/.$//;
|
||||
}
|
||||
}
|
||||
return $str;
|
||||
|
|
@ -11479,8 +11461,9 @@ sub print_files($) {
|
|||
}
|
||||
my ($color,$reset_color) = $self->color();
|
||||
my $termcol = ::terminal_columns();
|
||||
my $untabify_tag = $self->untabtag();
|
||||
my $untabify_str = ::untabify($self->{$out_fh,'latestline'});
|
||||
my $untabify_tag = ::decode_utf8($self->untabtag());
|
||||
my $untabify_str =
|
||||
::untabify(::decode_utf8($self->{$out_fh,'latestline'}));
|
||||
# -1 to make space for $truncated_str
|
||||
my $maxtaglen = $termcol - 1;
|
||||
$untabify_tag = truncate_mbs($untabify_tag,$maxtaglen);
|
||||
|
|
@ -11505,8 +11488,8 @@ sub print_files($) {
|
|||
"%s" # down
|
||||
),
|
||||
"$up"x($currow - $row), "\n"x($row - $currow), "\r", $eol,
|
||||
::decode_utf8($untabify_tag),$truncated_tag,
|
||||
$color, ::decode_utf8($untabify_str), $truncated_str, $reset_color,
|
||||
$untabify_tag,$truncated_tag,
|
||||
$color, $untabify_str, $truncated_str, $reset_color,
|
||||
"\n"x($maxrow - $row + 1));
|
||||
$currow = $maxrow + 1;
|
||||
}
|
||||
|
|
@ -15128,8 +15111,6 @@ sub main() {
|
|||
# Parallel check all hosts are up. Remove hosts that are down
|
||||
filter_hosts();
|
||||
}
|
||||
|
||||
|
||||
if($opt::sqlmaster and $opt::sqlworker) {
|
||||
# Start a real --sqlworker in the background later
|
||||
$Global::start_sqlworker = 1;
|
||||
|
|
@ -15184,7 +15165,6 @@ sub main() {
|
|||
# Compute $Global::max_jobs_running
|
||||
# Must be done after ungetting any --pipepart jobs.
|
||||
max_jobs_running();
|
||||
|
||||
init_run_jobs();
|
||||
my $sem;
|
||||
if($Global::semaphore) {
|
||||
|
|
|
|||
110
src/parallel.pod
110
src/parallel.pod
|
|
@ -1375,19 +1375,27 @@ If the host is long, you can use B<column -t> to pretty print it:
|
|||
See also: B<--resume> B<--resume-failed>
|
||||
|
||||
|
||||
=item B<--jobs> I<N>
|
||||
=item B<--jobs> I<num> (alpha testing)
|
||||
|
||||
=item B<-j> I<N>
|
||||
=item B<-j> I<num> (alpha testing)
|
||||
|
||||
=item B<--max-procs> I<N>
|
||||
=item B<--max-procs> I<num> (alpha testing)
|
||||
|
||||
=item B<-P> I<N>
|
||||
=item B<-P> I<num> (alpha testing)
|
||||
|
||||
Number of jobslots on each machine.
|
||||
|
||||
Run up to N jobs in parallel. 0 means as many as possible (this can
|
||||
take a while to determine). Default is 100% which will run one job per
|
||||
CPU thread on each machine.
|
||||
Run up to I<num> jobs in parallel. Default is 100%.
|
||||
|
||||
=over 7
|
||||
|
||||
=item I<num>
|
||||
|
||||
Run up to I<num> jobs in parallel.
|
||||
|
||||
=item Z<>0
|
||||
|
||||
Run as many as possible (this can take a while to determine).
|
||||
|
||||
Due to a bug B<-j 0> will also evaluate replacement strings twice up
|
||||
to the number of joblots:
|
||||
|
|
@ -1397,81 +1405,49 @@ to the number of joblots:
|
|||
# This will count from 1
|
||||
seq 10000 | parallel -j100 echo '{= $_ = $foo++; =}' | head
|
||||
|
||||
If B<--semaphore> is set, the default is 1 thus making a mutex.
|
||||
=item I<num>%
|
||||
|
||||
See also: B<--use-cores-instead-of-threads>
|
||||
B<--use-sockets-instead-of-threads>
|
||||
Multiply the number of CPU threads by I<num> percent. E.g. 100% means
|
||||
one job per CPU thread on each machine.
|
||||
|
||||
=item +I<num>
|
||||
|
||||
Add I<num> to the number of CPU threads.
|
||||
|
||||
=item B<--jobs> I<+N>
|
||||
=item -I<num>
|
||||
|
||||
=item B<-j> I<+N>
|
||||
Subtract I<num> from the number of CPU threads.
|
||||
|
||||
=item B<--max-procs> I<+N>
|
||||
=item I<expr>
|
||||
|
||||
=item B<-P> I<+N>
|
||||
Evaluate I<expr>. E.g. '12/2' to get 6, '+25%' gives the same as
|
||||
'125%', or complex expressions like '+3*log(55)%' which means:
|
||||
multiply 3 by log(55), multiply that by the number of CPU threads and
|
||||
divide by 100, add this to the number of CPU threads.
|
||||
|
||||
Add N to the number of CPU threads.
|
||||
|
||||
Run this many jobs in parallel.
|
||||
|
||||
See also: B<--number-of-threads> B<--number-of-cores>
|
||||
B<--number-of-sockets>
|
||||
|
||||
|
||||
=item B<--jobs> I<-N>
|
||||
|
||||
=item B<-j> I<-N>
|
||||
|
||||
=item B<--max-procs> I<-N>
|
||||
|
||||
=item B<-P> I<-N>
|
||||
|
||||
Subtract N from the number of CPU threads.
|
||||
|
||||
Run this many jobs in parallel. If the evaluated number is less than
|
||||
1 then 1 will be used.
|
||||
|
||||
See also: B<--number-of-threads> B<--number-of-cores>
|
||||
B<--number-of-sockets>
|
||||
|
||||
|
||||
=item B<--jobs> I<N>%
|
||||
|
||||
=item B<-j> I<N>%
|
||||
|
||||
=item B<--max-procs> I<N>%
|
||||
|
||||
=item B<-P> I<N>%
|
||||
|
||||
Multiply N% with the number of CPU threads.
|
||||
|
||||
Run this many jobs in parallel.
|
||||
|
||||
See also: B<--number-of-threads> B<--number-of-cores>
|
||||
B<--number-of-sockets>
|
||||
|
||||
|
||||
=item B<--jobs> I<procfile>
|
||||
|
||||
=item B<-j> I<procfile>
|
||||
|
||||
=item B<--max-procs> I<procfile>
|
||||
|
||||
=item B<-P> I<procfile>
|
||||
=item I<procfile>
|
||||
|
||||
Read parameter from file.
|
||||
|
||||
Use the content of I<procfile> as parameter for
|
||||
I<-j>. E.g. I<procfile> could contain the string 100% or +2 or 10. If
|
||||
I<procfile> is changed when a job completes, I<procfile> is read again
|
||||
and the new number of jobs is computed. If the number is lower than
|
||||
before, running jobs will be allowed to finish but new jobs will not
|
||||
be started until the wanted number of jobs has been reached. This
|
||||
I<-j>. E.g. I<procfile> could contain the string 100% or +2 or 10.
|
||||
|
||||
If I<procfile> is changed when a job completes, I<procfile> is read
|
||||
again and the new number of jobs is computed. If the number is lower
|
||||
than before, running jobs will be allowed to finish but new jobs will
|
||||
not be started until the wanted number of jobs has been reached. This
|
||||
makes it possible to change the number of simultaneous running jobs
|
||||
while GNU B<parallel> is running.
|
||||
|
||||
=back
|
||||
|
||||
If the evaluated number is less than 1 then 1 will be used.
|
||||
|
||||
If B<--semaphore> is set, the default is 1 thus making a mutex.
|
||||
|
||||
See also: B<--use-cores-instead-of-threads>
|
||||
B<--use-sockets-instead-of-threads>
|
||||
|
||||
|
||||
=item B<--keep-order>
|
||||
|
||||
|
|
|
|||
|
|
@ -3917,6 +3917,10 @@ https://github.com/jkool702/forkrun
|
|||
|
||||
=head2 Todo
|
||||
|
||||
https://www.npmjs.com/package/concurrently
|
||||
|
||||
https://github.com/thilinaba/bash-parallel
|
||||
|
||||
http://code.google.com/p/push/ (cannot compile)
|
||||
|
||||
https://github.com/krashanoff/parallel
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ line number and the URL.
|
|||
|
||||
cat urlfile | parallel "wget {} 2>/dev/null || grep -n {} urlfile"
|
||||
|
||||
Create a mirror directory with the same filenames except all files and
|
||||
Create a mirror directory with the same file names except all files and
|
||||
symlinks are empty files.
|
||||
|
||||
cp -rs /the/source/dir mirror_dir
|
||||
|
|
@ -288,6 +288,21 @@ Put all converted in the same directory:
|
|||
parallel lame {} -o mydir/{/.}.mp3
|
||||
|
||||
|
||||
=head2 EXAMPLE: Replacing parts of file names
|
||||
|
||||
If you deal with paired end reads, you will have files like
|
||||
barcode1_R1.fq.gz, barcode1_R2.fq.gz, barcode2_R1.fq.gz, and
|
||||
barcode2_R2.fq.gz.
|
||||
|
||||
You want barcodeI<N>_R1 to be processed with barcodeI<N>_R2.
|
||||
|
||||
parallel --plus myprocess {} {/_R1.fq.gz/_R2.fq.gz} ::: *_R1.fq.gz
|
||||
|
||||
If the barcode does not contain '_R1', you can do:
|
||||
|
||||
parallel --plus myprocess {} {/_R1/_R2} ::: *_R1.fq.gz
|
||||
|
||||
|
||||
=head2 EXAMPLE: Removing strings from the argument
|
||||
|
||||
If you have directory with tar.gz files and want these extracted in
|
||||
|
|
@ -1719,7 +1734,7 @@ Here B<python>:
|
|||
print "exclusively";
|
||||
|
||||
|
||||
=head2 EXAMPLE: Start editor with filenames from stdin (standard input)
|
||||
=head2 EXAMPLE: Start editor with file names from stdin (standard input)
|
||||
|
||||
You can use GNU B<parallel> to start interactive programs like emacs or vi:
|
||||
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ GetOptions(
|
|||
"help" => \$opt::dummy,
|
||||
) || exit(255);
|
||||
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
|
||||
$Global::version = 20230322;
|
||||
$Global::version = 20230323;
|
||||
if($opt::version) { version(); exit 0; }
|
||||
# Remove -D and --parallel=N
|
||||
my @s = (grep { ! /^-D$|^--parallel=\S+$/ }
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20230322;
|
||||
$Global::version = 20230323;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ linebuffer_matters() {
|
|||
# Random data because it does not compress well
|
||||
# forcing the compress tool to spit out compressed blocks
|
||||
perl -pe 'y/[A-Za-z]//cd; $t++ % 1000 or print "\n"' < /dev/urandom |
|
||||
head -c 10000000 > $randomfile
|
||||
head -c 10000000 > "$randomfile"
|
||||
export randomfile
|
||||
|
||||
testfunc() {
|
||||
|
|
@ -49,10 +49,10 @@ linebuffer_matters() {
|
|||
# generate some incompressible ascii
|
||||
# with lines starting with the same string
|
||||
id=$1
|
||||
shuf $randomfile | perl -pe 's/^/'$id' /'
|
||||
shuf "$randomfile" | perl -pe 's/^/'$id' /'
|
||||
# Sleep to give time to linebuffer-print the first part
|
||||
sleep 10
|
||||
shuf $randomfile | perl -pe 's/^/'$id' /'
|
||||
shuf "$randomfile" | perl -pe 's/^/'$id' /'
|
||||
echo
|
||||
}
|
||||
export -f incompressible_ascii
|
||||
|
|
@ -72,15 +72,15 @@ linebuffer_matters() {
|
|||
}
|
||||
|
||||
# These can run in parallel if there are enough ressources
|
||||
testfunc > $nolbfile
|
||||
testfunc > $controlfile
|
||||
testfunc --linebuffer > $lbfile
|
||||
testfunc > "$nolbfile"
|
||||
testfunc > "$controlfile"
|
||||
testfunc --linebuffer > "$lbfile"
|
||||
wait
|
||||
|
||||
nolb="$(cat $nolbfile)"
|
||||
control="$(cat $controlfile)"
|
||||
lb="$(cat $lbfile)"
|
||||
rm $nolbfile $lbfile $controlfile $randomfile
|
||||
nolb="$(cat "$nolbfile")"
|
||||
control="$(cat "$controlfile")"
|
||||
lb="$(cat "$lbfile")"
|
||||
rm "$nolbfile" "$lbfile" "$controlfile" "$randomfile"
|
||||
|
||||
if [ "$nolb" == "$control" ] ; then
|
||||
if [ "$lb" == "$nolb" ] ; then
|
||||
|
|
|
|||
|
|
@ -18,38 +18,55 @@ par_compare_exit_codes() {
|
|||
echo '### compare the exit codes'
|
||||
echo 'directly from shells, shells called from parallel,'
|
||||
echo 'killed with different signals'
|
||||
echo
|
||||
# run:
|
||||
# * exit $val
|
||||
# * kill -$1 $$'
|
||||
# both:
|
||||
# * under different shells
|
||||
# * from parallel running under different shells
|
||||
# record:
|
||||
# * exit value
|
||||
# * signal
|
||||
echo sig=joblog_sig shell=parallel=joblog
|
||||
# The quoting of \n is wrong for csh, so just use a sane TMPDIR here
|
||||
TMPDIR=/tmp/par-300s
|
||||
export TMPDIR
|
||||
mkdir -p "$TMPDIR"
|
||||
|
||||
selfkill=$(mktemp)
|
||||
export selfkill
|
||||
echo 'kill -$1 $$' >$selfkill
|
||||
qselfkill=$(parallel -0 --shellquote ::: "$selfkill")
|
||||
qqselfkill=$(parallel -0 --shellquote --shellquote ::: "$selfkill")
|
||||
export selfkill qselfkill qqselfkill
|
||||
echo 'kill -$1 $$' >"$selfkill"
|
||||
exit=$(mktemp)
|
||||
export exit
|
||||
echo 'exit $1' >$exit
|
||||
qexit=$(parallel -0 --shellquote ::: "$exit")
|
||||
qqexit=$(parallel -0 --shellquote --shellquote ::: "$exit")
|
||||
export exit qexit qqexit
|
||||
echo 'exit $1' >"$exit"
|
||||
|
||||
doit() {
|
||||
shell=$1
|
||||
sig=$2
|
||||
sig128=$(( sig + 128 ))
|
||||
sh -c "$shell $selfkill $sig" 2>/dev/null
|
||||
sh -c "$shell $qselfkill $sig" 2>/dev/null
|
||||
raw=$?
|
||||
sh -c "$shell $exit $sig128" 2>/dev/null
|
||||
sh -c "$shell $qexit $sig128" 2>/dev/null
|
||||
raw128=$?
|
||||
|
||||
log=$(mktemp)
|
||||
$shell -c "parallel --halt now,done=1 --jl $log $shell $selfkill ::: $sig" 2>/dev/null
|
||||
qlog=$(parallel -0 --shellquote ::: "$log")
|
||||
qqlog=$(parallel -0 --shellquote --shellquote ::: "$log")
|
||||
$shell -c "parallel --halt now,done=1 --jl $qlog $shell $qqselfkill ::: $sig" 2>/dev/null
|
||||
#echo parallel $shell $sig = $?
|
||||
parallel=$?
|
||||
joblog_exit=$(field 7 < $log | tail -n1)
|
||||
joblog_signal=$(field 8 < $log | tail -n1)
|
||||
$shell -c "parallel --halt now,done=1 --jl $log $shell $exit ::: $sig128" 2>/dev/null
|
||||
joblog_exit=$(field 7 < "$log" | tail -n1)
|
||||
joblog_signal=$(field 8 < "$log" | tail -n1)
|
||||
$shell -c "parallel --halt now,done=1 --jl $qlog $shell $qqexit ::: $sig128" 2>/dev/null
|
||||
parallel128=$?
|
||||
joblog_exit128=$(field 7 < $log | tail -n1)
|
||||
joblog_signal128=$(field 8 < $log | tail -n1)
|
||||
|
||||
#echo joblog p $shell $sig $(field 8,7 < $log | tail -n1)
|
||||
rm $log
|
||||
joblog_exit128=$(field 7 < "$log" | tail -n1)
|
||||
joblog_signal128=$(field 8 < "$log" | tail -n1)
|
||||
|
||||
rm "$log"
|
||||
|
||||
echo $shell sig' ' $sig=$joblog_signal $raw=$parallel=$joblog_exit
|
||||
echo $shell exit $sig128=$joblog_signal128 $raw128=$parallel128=$joblog_exit128
|
||||
|
|
@ -60,17 +77,17 @@ par_compare_exit_codes() {
|
|||
OK="ash csh dash fish fizsh posh rc sash sh tcsh"
|
||||
# These do not give the same exit code prepended with 'true;' or not
|
||||
BAD="bash ksh93 mksh static-sh yash zsh"
|
||||
|
||||
(
|
||||
# Most block on signals: 19+20+21+22
|
||||
ulimit -n `ulimit -Hn`
|
||||
parallel -j1000% -k doit ::: $OK $BAD ::: {1..18} {23..64}
|
||||
# fdsh blocks on a lot more signals
|
||||
parallel -j1000% -k doit ::: fdsh ::: 2 {9..12} {14..18} {20..23} 26 27 29 30 {32..64}
|
||||
) |
|
||||
# Ignore where the exit codes are the same
|
||||
) |
|
||||
# Ignore lines where the exit codes are the same
|
||||
perl -ne '/(\d+)=\1=\1/ or print'
|
||||
rm $selfkill
|
||||
rm "$selfkill" "$exit"
|
||||
rmdir "$TMPDIR"
|
||||
}
|
||||
|
||||
par_retries_unreachable() {
|
||||
|
|
|
|||
|
|
@ -10,14 +10,25 @@ par_ll_asian() {
|
|||
echo '### --ll with Asian wide chars mess up display'
|
||||
echo 'bug #63878: Wide East Asian chars in --latest-line'
|
||||
p="parallel --ll --color --tag"
|
||||
COLUMNS=80 $p echo tag fits, line fits a{}b{}c \
|
||||
::: ヌー平行
|
||||
COLUMNS=80 $p echo tag fits, line too long a{}b{}c \
|
||||
::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー
|
||||
COLUMNS=80 $p echo tag too long a{}b{}c \
|
||||
echo Oops: the first adds '>' too early
|
||||
COLUMNS=50 $p echo tag fits, line fits a{}b{}c \
|
||||
::: ヌー平
|
||||
COLUMNS=50 $p echo tag fits, line too long a{}b{}c \
|
||||
::: ヌー平行ヌー平行ヌー平行ヌー平行ヌ
|
||||
COLUMNS=50 $p echo tag too long a{}b{}c \
|
||||
::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行a
|
||||
}
|
||||
|
||||
par_mbswidth() {
|
||||
echo '### characters with screen width > 1'
|
||||
perl -e '@a=qw(ヌ ー 平 行.);
|
||||
print map {
|
||||
(join"",map{ $a[$_% $#a] } (1..$_))."\n".
|
||||
"a".(join"",map{ $a[$_% $#a] } (1..$_))."\n"
|
||||
} (1..40)' |
|
||||
COLUMNS=50 parallel -k --ll --color --tag echo
|
||||
}
|
||||
|
||||
par_ll_tag() {
|
||||
parallel --tag --ll -q printf "a\n{}\n" ::: should-be-tagged-A
|
||||
parallel --tag --ll -q printf "a\n\r{}\n" ::: should-be-tagged-B
|
||||
|
|
|
|||
|
|
@ -83,6 +83,9 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' "$testsuit
|
|||
s/(std(out|err)|seq): Permission denied/$1: No such file or directory/;
|
||||
# Race condition
|
||||
s/^4-(middle|end)\n//;
|
||||
# Race condition
|
||||
s/^parallel: This job failed:\n//;
|
||||
s/^echo .; exit .\n//;
|
||||
# Base 64 string with quotes
|
||||
s:['"'"'"\\+/a-z.0-9=]{50,}(\s['"'"'"\\+/a-z.0-9=]*)*:BASE64:ig;
|
||||
# Timings are often off
|
||||
|
|
|
|||
|
|
@ -976,10 +976,10 @@ par_sem_quote ### sem --quote should not add empty argument
|
|||
par_sem_quote echo
|
||||
par_sem_quote
|
||||
par_shellcompletion ### --shellcompletion
|
||||
par_shellcompletion 0097e98f494e6288b33f3201dda7ee11 -
|
||||
par_shellcompletion 0097e98f494e6288b33f3201dda7ee11 -
|
||||
par_shellcompletion 1655cb29ff14d0008866108c6b5262f7 -
|
||||
par_shellcompletion 1655cb29ff14d0008866108c6b5262f7 -
|
||||
par_shellcompletion 70960cbdbc411e041161ae228f029d70 -
|
||||
par_shellcompletion 70960cbdbc411e041161ae228f029d70 -
|
||||
par_shellcompletion aa125ab894780611a20bad4a52d7a58d -
|
||||
par_shellcompletion aa125ab894780611a20bad4a52d7a58d -
|
||||
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
|
||||
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
|
||||
par_slow_pipe_regexp 0 1 1
|
||||
|
|
|
|||
|
|
@ -3,99 +3,115 @@ par_linebuffer_files bug #48658: --linebuffer --files
|
|||
par_linebuffer_files zstd normal
|
||||
par_linebuffer_files zstd 100000
|
||||
par_linebuffer_files zstd --files
|
||||
par_linebuffer_files zstd 1
|
||||
par_linebuffer_files zstd 4
|
||||
par_linebuffer_files zstd --results
|
||||
par_linebuffer_files zstd 1
|
||||
par_linebuffer_files zstd parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files pzstd normal
|
||||
par_linebuffer_files pzstd 100000
|
||||
par_linebuffer_files pzstd --files
|
||||
par_linebuffer_files pzstd 1
|
||||
par_linebuffer_files pzstd 4
|
||||
par_linebuffer_files pzstd --results
|
||||
par_linebuffer_files pzstd 1
|
||||
par_linebuffer_files pzstd parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files clzip normal
|
||||
par_linebuffer_files clzip 100000
|
||||
par_linebuffer_files clzip --files
|
||||
par_linebuffer_files clzip 1
|
||||
par_linebuffer_files clzip 4
|
||||
par_linebuffer_files clzip --results
|
||||
par_linebuffer_files clzip 1
|
||||
par_linebuffer_files clzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files lz4 normal
|
||||
par_linebuffer_files lz4 100000
|
||||
par_linebuffer_files lz4 --files
|
||||
par_linebuffer_files lz4 1
|
||||
par_linebuffer_files lz4 4
|
||||
par_linebuffer_files lz4 --results
|
||||
par_linebuffer_files lz4 1
|
||||
par_linebuffer_files lz4 parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files lzop normal
|
||||
par_linebuffer_files lzop 100000
|
||||
par_linebuffer_files lzop --files
|
||||
par_linebuffer_files lzop 1
|
||||
par_linebuffer_files lzop 4
|
||||
par_linebuffer_files lzop --results
|
||||
par_linebuffer_files lzop 1
|
||||
par_linebuffer_files lzop parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files pigz normal
|
||||
par_linebuffer_files pigz 100000
|
||||
par_linebuffer_files pigz --files
|
||||
par_linebuffer_files pigz 1
|
||||
par_linebuffer_files pigz 4
|
||||
par_linebuffer_files pigz --results
|
||||
par_linebuffer_files pigz 1
|
||||
par_linebuffer_files pigz parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files pxz normal
|
||||
par_linebuffer_files pxz 100000
|
||||
par_linebuffer_files pxz --files
|
||||
par_linebuffer_files pxz 1
|
||||
par_linebuffer_files pxz 4
|
||||
par_linebuffer_files pxz --results
|
||||
par_linebuffer_files pxz 1
|
||||
par_linebuffer_files pxz parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files gzip normal
|
||||
par_linebuffer_files gzip 100000
|
||||
par_linebuffer_files gzip --files
|
||||
par_linebuffer_files gzip 1
|
||||
par_linebuffer_files gzip 4
|
||||
par_linebuffer_files gzip --results
|
||||
par_linebuffer_files gzip 1
|
||||
par_linebuffer_files gzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files plzip normal
|
||||
par_linebuffer_files plzip 100000
|
||||
par_linebuffer_files plzip --files
|
||||
par_linebuffer_files plzip 1
|
||||
par_linebuffer_files plzip 4
|
||||
par_linebuffer_files plzip --results
|
||||
par_linebuffer_files plzip 1
|
||||
par_linebuffer_files plzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files pbzip2 normal
|
||||
par_linebuffer_files pbzip2 100000
|
||||
par_linebuffer_files pbzip2 --files
|
||||
par_linebuffer_files pbzip2 1
|
||||
par_linebuffer_files pbzip2 4
|
||||
par_linebuffer_files pbzip2 --results
|
||||
par_linebuffer_files pbzip2 1
|
||||
par_linebuffer_files pbzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files lzma normal
|
||||
par_linebuffer_files lzma 100000
|
||||
par_linebuffer_files lzma --files
|
||||
par_linebuffer_files lzma 1
|
||||
par_linebuffer_files lzma 4
|
||||
par_linebuffer_files lzma --results
|
||||
par_linebuffer_files lzma 1
|
||||
par_linebuffer_files lzma parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files xz normal
|
||||
par_linebuffer_files xz 100000
|
||||
par_linebuffer_files xz --files
|
||||
par_linebuffer_files xz 1
|
||||
par_linebuffer_files xz 4
|
||||
par_linebuffer_files xz --results
|
||||
par_linebuffer_files xz 1
|
||||
par_linebuffer_files xz parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files lzip normal
|
||||
par_linebuffer_files lzip 100000
|
||||
par_linebuffer_files lzip --files
|
||||
par_linebuffer_files lzip 1
|
||||
par_linebuffer_files lzip 4
|
||||
par_linebuffer_files lzip --results
|
||||
par_linebuffer_files lzip 1
|
||||
par_linebuffer_files lzip parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files bzip2 normal
|
||||
par_linebuffer_files bzip2 100000
|
||||
par_linebuffer_files bzip2 --files
|
||||
par_linebuffer_files bzip2 1
|
||||
par_linebuffer_files bzip2 4
|
||||
par_linebuffer_files bzip2 --results
|
||||
par_linebuffer_files bzip2 1
|
||||
par_linebuffer_files bzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files lbzip2 normal
|
||||
par_linebuffer_files lbzip2 100000
|
||||
par_linebuffer_files lbzip2 --files
|
||||
par_linebuffer_files lbzip2 1
|
||||
par_linebuffer_files lbzip2 4
|
||||
par_linebuffer_files lbzip2 --results
|
||||
par_linebuffer_files lbzip2 1
|
||||
par_linebuffer_files lbzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_files lrz normal
|
||||
par_linebuffer_files lrz 100000
|
||||
par_linebuffer_files lrz --files
|
||||
par_linebuffer_files lrz 1
|
||||
par_linebuffer_files lrz 4
|
||||
par_linebuffer_files lrz --results
|
||||
par_linebuffer_files lrz 1
|
||||
par_linebuffer_files lrz parallel: Warning: Use --files0 when $TMPDIR contains newline.
|
||||
par_linebuffer_matters_compress ### (--linebuffer) --compress should give different output
|
||||
par_linebuffer_matters_compress OK: --linebuffer makes a difference
|
||||
par_linebuffer_matters_compress_tag ### (--linebuffer) --compress --tag should give different output
|
||||
|
|
|
|||
|
|
@ -318,7 +318,7 @@ par_test_build_and_install make distdir-am
|
|||
par_test_build_and_install make[0]: Entering directory '~/privat/parallel/src'
|
||||
par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src'
|
||||
par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src'
|
||||
par_test_build_and_install test -n "" \
|
||||
par_test_build_and_install test -n " \
|
||||
par_test_build_and_install || find "parallel-00000000" -type d ! -perm -755 \
|
||||
par_test_build_and_install -exec chmod u+rwx,go+rx {} \; -o \
|
||||
par_test_build_and_install ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \
|
||||
|
|
|
|||
|
|
@ -606,29 +606,16 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
|
|||
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
||||
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
||||
9
|
||||
parallel: This job failed:
|
||||
echo X; exit X
|
||||
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
|
||||
parallel: This job failed:
|
||||
echo X; exit X
|
||||
parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
|
||||
9
|
||||
parallel: This job failed:
|
||||
echo X; exit X
|
||||
parallel -j2 --halt soon,fail=20% echo {}\; exit {} \
|
||||
::: 0 1 2 3 4 5 6 7 8 9
|
||||
9
|
||||
parallel: This job failed:
|
||||
echo X; exit X
|
||||
parallel: This job failed:
|
||||
echo X; exit X
|
||||
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
|
||||
parallel: This job failed:
|
||||
echo X; exit X
|
||||
parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
|
||||
9
|
||||
parallel: This job succeeded:
|
||||
echo X; exit X
|
||||
parallel -k --retries 3 \
|
||||
'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
|
||||
cat /tmp/runs
|
||||
|
|
|
|||
Loading…
Reference in a new issue