mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 13:37:56 +00:00
parallel: Most basic --fast implemented.
This commit is contained in:
parent
1daaf604c7
commit
3916f78fb2
14
NEWS
14
NEWS
|
@ -2,6 +2,20 @@
|
||||||
|
|
||||||
New in this release:
|
New in this release:
|
||||||
|
|
||||||
|
* --fast disables a lot of functionality to speed up running jobs.
|
||||||
|
|
||||||
|
* Bug fixes and man page updates.
|
||||||
|
|
||||||
|
News about GNU Parallel:
|
||||||
|
|
||||||
|
* Job requiring GNU Parallel knowledge
|
||||||
|
https://www.capgemini.com/ca-en/jobs/Id6D4pEBZ6aB2WPS2aAJ/systems-engineer/
|
||||||
|
|
||||||
|
|
||||||
|
20240822
|
||||||
|
|
||||||
|
New in this release:
|
||||||
|
|
||||||
* --match Match input source with regexp to set replacement fields.
|
* --match Match input source with regexp to set replacement fields.
|
||||||
|
|
||||||
* {:%fmt} Use printf formatting of replacement strings.
|
* {:%fmt} Use printf formatting of replacement strings.
|
||||||
|
|
|
@ -4,9 +4,6 @@
|
||||||
|
|
||||||
Quote of the month:
|
Quote of the month:
|
||||||
|
|
||||||
Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts.
|
|
||||||
-- @mechanicker@twitter Dhruva
|
|
||||||
|
|
||||||
by extreme do you mean extremely slow? you could do it at least 100x faster using awk + grep + gnu parallel
|
by extreme do you mean extremely slow? you could do it at least 100x faster using awk + grep + gnu parallel
|
||||||
-- @ObssessedDev@twitter
|
-- @ObssessedDev@twitter
|
||||||
|
|
||||||
|
@ -227,6 +224,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html
|
||||||
|
|
||||||
=== Used ===
|
=== Used ===
|
||||||
|
|
||||||
|
Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts.
|
||||||
|
-- @mechanicker@twitter Dhruva
|
||||||
|
|
||||||
honestly the coolest software i've ever seen gotta be gnu parallel or
|
honestly the coolest software i've ever seen gotta be gnu parallel or
|
||||||
ffmpeg, nothing like them
|
ffmpeg, nothing like them
|
||||||
-- @scootykins scoot
|
-- @scootykins scoot
|
||||||
|
|
|
@ -268,34 +268,25 @@ from:tange@gnu.org
|
||||||
to:parallel@gnu.org, bug-parallel@gnu.org
|
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||||
|
|
||||||
Subject: GNU Parallel 20240822 ('Southport') released
|
Subject: GNU Parallel 20240922 ('Gold Apollo AR924') released
|
||||||
|
|
||||||
GNU Parallel 20240822 ('Southport') has been released. It is available for download at: lbry://@GnuParallel:4
|
GNU Parallel 20240922 ('Gold Apollo AR924') has been released. It is available for download at: lbry://@GnuParallel:4
|
||||||
|
|
||||||
Quote of the month:
|
Quote of the month:
|
||||||
|
|
||||||
honestly the coolest software i've ever seen gotta be gnu parallel or
|
Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts.
|
||||||
ffmpeg, nothing like them
|
-- @mechanicker@twitter Dhruva
|
||||||
-- @scootykins scoot
|
|
||||||
|
|
||||||
New in this release:
|
New in this release:
|
||||||
|
|
||||||
* --match Match input source with regexp to set replacement fields.
|
* --fast disables a lot of functionality to speed up running jobs.
|
||||||
|
|
||||||
* {:%fmt} Use printf formatting of replacement strings.
|
|
||||||
|
|
||||||
* Bug fixes and man page updates.
|
* Bug fixes and man page updates.
|
||||||
|
|
||||||
|
|
||||||
News about GNU Parallel:
|
News about GNU Parallel:
|
||||||
|
|
||||||
* Powerful GNU parallel, more than a loop https://www.linkedin.com/pulse/powerful-gnu-parallel-more-than-loop-zhenguo-zhang-18dxc
|
* Job requiring GNU Parallel knowledge
|
||||||
|
https://www.capgemini.com/ca-en/jobs/Id6D4pEBZ6aB2WPS2aAJ/systems-engineer/
|
||||||
* How To Increase File Transfer Speed Using Parallel Rsync? https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/
|
|
||||||
|
|
||||||
* Converting WebP Images to PNG Using parallel and dwebp https://bytefreaks.net/2024/07/27
|
|
||||||
|
|
||||||
* Turbocharging the Box CLI with GNU Parallel https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0
|
|
||||||
|
|
||||||
|
|
||||||
GNU Parallel - For people who live life in the parallel lane.
|
GNU Parallel - For people who live life in the parallel lane.
|
||||||
|
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -605,7 +605,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -570,7 +570,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
||||||
fi
|
fi
|
||||||
if [ "$_parset_NAME" = "--version" ] ; then
|
if [ "$_parset_NAME" = "--version" ] ; then
|
||||||
# shellcheck disable=SC2006
|
# shellcheck disable=SC2006
|
||||||
echo "parset 20240822 (GNU parallel `parallel --minversion 1`)"
|
echo "parset 20240823 (GNU parallel `parallel --minversion 1`)"
|
||||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||||
echo "Foundation, Inc."
|
echo "Foundation, Inc."
|
||||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||||
|
|
|
@ -26,7 +26,7 @@
|
||||||
use strict;
|
use strict;
|
||||||
use Getopt::Long;
|
use Getopt::Long;
|
||||||
$Global::progname="niceload";
|
$Global::progname="niceload";
|
||||||
$Global::version = 20240822;
|
$Global::version = 20240823;
|
||||||
Getopt::Long::Configure("bundling","require_order");
|
Getopt::Long::Configure("bundling","require_order");
|
||||||
get_options_from_array(\@ARGV) || die_usage();
|
get_options_from_array(\@ARGV) || die_usage();
|
||||||
if($opt::version) {
|
if($opt::version) {
|
||||||
|
|
102
src/parallel
102
src/parallel
|
@ -156,6 +156,91 @@ sub halt() {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub __FAST_MODE__() {}
|
||||||
|
|
||||||
|
|
||||||
|
sub fast() {
|
||||||
|
my $jobslots = $Global::host{':'}->user_requested_processes($opt::jobs);
|
||||||
|
my $i;
|
||||||
|
my $jobs_per_chunk;
|
||||||
|
my $buffer_len = 100;
|
||||||
|
my $splitstring = ("\n#\0\n");
|
||||||
|
# n = jobslots
|
||||||
|
# Read 100 jobs
|
||||||
|
# Split into n chunks
|
||||||
|
# Read 3*100 jobs
|
||||||
|
# Split into n chunks
|
||||||
|
# Read 3*3*100 jobs (up to at most 300K jobs)
|
||||||
|
# Split into n chunks
|
||||||
|
# Receiver: parallel --block 1k --pipe -N1 --recend '\n#\0\n' dash
|
||||||
|
# TODO --group? --halt? --tag? $PARALLEL_JOBSLOT
|
||||||
|
# _PARALLEL_TAG="foo"
|
||||||
|
# | tagger
|
||||||
|
# _PARALLEL_EXIT
|
||||||
|
# maybe_exit
|
||||||
|
my $executer_fh;
|
||||||
|
if($opt::D eq "fast") {
|
||||||
|
open($executer_fh, "|-", "cat") || die;
|
||||||
|
} else {
|
||||||
|
open($executer_fh, "|-", "parallel --plain -j $jobslots --block 1k --pipe -N1 --recend '\\n#\\0\\n' $Global::shell") || die;
|
||||||
|
}
|
||||||
|
# Do some testing of a stuck job - dns = 5 sec
|
||||||
|
my $block_start = q{
|
||||||
|
error() {
|
||||||
|
_exit=$((_exit+1))
|
||||||
|
# --halt-on-error stuff
|
||||||
|
if [ $_exit -gt 40 ]; then
|
||||||
|
echo "ERROR: More than 40"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
_tagger() {
|
||||||
|
{
|
||||||
|
rm -f "$1"
|
||||||
|
# This will fail for either /1 or /2
|
||||||
|
rmdir "$2" 2>/dev/null
|
||||||
|
# TODO use awk (see fasttag)
|
||||||
|
perl -pe 's{^}{$ENV{PARALLEL_TAGSTRING}}'
|
||||||
|
} < "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
_taggerwrap() {
|
||||||
|
_dir=`mktemp -d`
|
||||||
|
mkfifo "$_dir"/1 "$_dir"/2
|
||||||
|
_tagger "$_dir"/1 "$_dir" >&1 &
|
||||||
|
_tagger "$_dir"/2 "$_dir" >&2 &
|
||||||
|
# ( ... ) is needed to deal correctly with 'exit 1'
|
||||||
|
( eval $@ ; ) >"$_dir"/1 2>"$_dir"/2 || error
|
||||||
|
wait
|
||||||
|
}
|
||||||
|
PARALLEL_TAGSTRING=dummy
|
||||||
|
export PARALLEL_TAGSTRING
|
||||||
|
};
|
||||||
|
my $block_end = "\n# Define get exit value\necho EXIT=\$_exit\n".$splitstring."\n";
|
||||||
|
do {
|
||||||
|
$i = 0;
|
||||||
|
my @jobbuffer;
|
||||||
|
while(my $job = $Global::JobQueue->get()) {
|
||||||
|
push (@jobbuffer, "PARALLEL_TAGSTRING=".Q($job->tag()).
|
||||||
|
"\n_taggerwrap ". Q($job->replaced()));
|
||||||
|
($i++ < $buffer_len) or last;
|
||||||
|
}
|
||||||
|
my $jobs_per_chunk = 1 + ($i / $jobslots);
|
||||||
|
my $j = 0;
|
||||||
|
print $executer_fh $block_start;
|
||||||
|
for(@jobbuffer) {
|
||||||
|
print $executer_fh $_,"\n";
|
||||||
|
(++$j % $jobs_per_chunk) or print $executer_fh $block_end,$block_start;
|
||||||
|
}
|
||||||
|
print $executer_fh $block_end;
|
||||||
|
if($buffer_len < 100000) { $buffer_len *= 3; }
|
||||||
|
} while ($i);
|
||||||
|
close $executer_fh;
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
sub __PIPE_MODE__() {}
|
sub __PIPE_MODE__() {}
|
||||||
|
|
||||||
|
|
||||||
|
@ -2084,7 +2169,7 @@ sub options_completion_hash() {
|
||||||
"first line of a script. The content of the file will be treated ".
|
"first line of a script. The content of the file will be treated ".
|
||||||
"as inputsource]"
|
"as inputsource]"
|
||||||
=> \$opt::shebang),
|
=> \$opt::shebang),
|
||||||
("_pipe-means-argfiles[internal]"
|
("_pipe-means-argfiles[Internal: Called by --shebang-wrap]"
|
||||||
=> \$opt::_pipe_means_argfiles),
|
=> \$opt::_pipe_means_argfiles),
|
||||||
"Y" => \$opt::retired,
|
"Y" => \$opt::retired,
|
||||||
("skip-first-line|skipfirstline".
|
("skip-first-line|skipfirstline".
|
||||||
|
@ -2141,7 +2226,8 @@ sub options_completion_hash() {
|
||||||
"[Embed GNU parallel in a shell script]" => \$opt::combineexec,
|
"[Embed GNU parallel in a shell script]" => \$opt::combineexec,
|
||||||
("filter=s[Only run jobs where filter is true]:filter"
|
("filter=s[Only run jobs where filter is true]:filter"
|
||||||
=> \@opt::filter),
|
=> \@opt::filter),
|
||||||
"_parset=s[Generate shell code for parset]" => \$opt::_parset,
|
"fast[Run commands fast]" => \$opt::fast,
|
||||||
|
"_parset=s[Internal: Generate shell code for parset]" => \$opt::_parset,
|
||||||
("shell-completion|shellcompletion=s".
|
("shell-completion|shellcompletion=s".
|
||||||
"[Generate shell code for shell completion]:shell:(bash zsh)"
|
"[Generate shell code for shell completion]:shell:(bash zsh)"
|
||||||
=> \$opt::shellcompletion),
|
=> \$opt::shellcompletion),
|
||||||
|
@ -2248,8 +2334,12 @@ sub parse_options(@) {
|
||||||
#
|
#
|
||||||
## Shell
|
## Shell
|
||||||
#
|
#
|
||||||
$Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$)
|
if($opt::fast) {
|
||||||
|| $ENV{'SHELL'} || "/bin/sh";
|
$Global::shell = $ENV{'PARALLEL_SHELL'} || which("dash") || "/bin/sh";
|
||||||
|
} else {
|
||||||
|
$Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$)
|
||||||
|
|| $ENV{'SHELL'} || "/bin/sh";
|
||||||
|
}
|
||||||
if(not -x $Global::shell and not which($Global::shell)) {
|
if(not -x $Global::shell and not which($Global::shell)) {
|
||||||
::error("Shell '$Global::shell' not found.");
|
::error("Shell '$Global::shell' not found.");
|
||||||
wait_and_exit(255);
|
wait_and_exit(255);
|
||||||
|
@ -2798,7 +2888,7 @@ sub check_invalid_option_combinations() {
|
||||||
|
|
||||||
sub init_globals() {
|
sub init_globals() {
|
||||||
# Defaults:
|
# Defaults:
|
||||||
$Global::version = 20240822;
|
$Global::version = 20240823;
|
||||||
$Global::progname = 'parallel';
|
$Global::progname = 'parallel';
|
||||||
$::name = "GNU Parallel";
|
$::name = "GNU Parallel";
|
||||||
$Global::infinity = 2**31;
|
$Global::infinity = 2**31;
|
||||||
|
@ -15700,6 +15790,8 @@ sub main() {
|
||||||
# Fill all jobslots
|
# Fill all jobslots
|
||||||
while(start_more_jobs()) {}
|
while(start_more_jobs()) {}
|
||||||
spreadstdin();
|
spreadstdin();
|
||||||
|
} elsif($opt::fast) {
|
||||||
|
fast();
|
||||||
} else {
|
} else {
|
||||||
# Reap the finished jobs and start more
|
# Reap the finished jobs and start more
|
||||||
while(reapers() + start_more_jobs()) {}
|
while(reapers() + start_more_jobs()) {}
|
||||||
|
|
|
@ -134,7 +134,7 @@ B<Bash, Csh, or Tcsh aliases>: Use B<env_parallel>.
|
||||||
|
|
||||||
B<Zsh, Fish, Ksh, and Pdksh functions and aliases>: Use B<env_parallel>.
|
B<Zsh, Fish, Ksh, and Pdksh functions and aliases>: Use B<env_parallel>.
|
||||||
|
|
||||||
=item B<{}> (alpha testing)
|
=item B<{}> (beta testing)
|
||||||
|
|
||||||
Input line.
|
Input line.
|
||||||
|
|
||||||
|
@ -155,7 +155,7 @@ See also: B<--plus> B<{.}> B<{/}> B<{//}> B<{/.}> B<{#}> B<{%}>
|
||||||
B<{>I<n>B<}> B<{=>I<perl expression>B<=}>
|
B<{>I<n>B<}> B<{=>I<perl expression>B<=}>
|
||||||
|
|
||||||
|
|
||||||
=item B<{.}> (alpha testing)
|
=item B<{.}> (beta testing)
|
||||||
|
|
||||||
Input line without extension.
|
Input line without extension.
|
||||||
|
|
||||||
|
@ -173,7 +173,7 @@ The replacement string B<{.}> can be changed with B<--extensionreplace>
|
||||||
See also: B<{}> B<--extensionreplace>
|
See also: B<{}> B<--extensionreplace>
|
||||||
|
|
||||||
|
|
||||||
=item B<{/}> (alpha testing)
|
=item B<{/}> (beta testing)
|
||||||
|
|
||||||
Basename of input line.
|
Basename of input line.
|
||||||
|
|
||||||
|
@ -183,7 +183,7 @@ directory part removed.
|
||||||
See also: B<{}> B<--basenamereplace>
|
See also: B<{}> B<--basenamereplace>
|
||||||
|
|
||||||
|
|
||||||
=item B<{//}> (alpha testing)
|
=item B<{//}> (beta testing)
|
||||||
|
|
||||||
Dirname of input line.
|
Dirname of input line.
|
||||||
|
|
||||||
|
@ -193,7 +193,7 @@ line. See B<dirname>(1).
|
||||||
See also: B<{}> B<--dirnamereplace>
|
See also: B<{}> B<--dirnamereplace>
|
||||||
|
|
||||||
|
|
||||||
=item B<{/.}> (alpha testing)
|
=item B<{/.}> (beta testing)
|
||||||
|
|
||||||
Basename of input line without extension.
|
Basename of input line without extension.
|
||||||
|
|
||||||
|
@ -204,7 +204,7 @@ B<{/}> and B<{.}>.
|
||||||
See also: B<{}> B<--basenameextensionreplace>
|
See also: B<{}> B<--basenameextensionreplace>
|
||||||
|
|
||||||
|
|
||||||
=item B<{#}> (alpha testing)
|
=item B<{#}> (beta testing)
|
||||||
|
|
||||||
Sequence number of the job to run.
|
Sequence number of the job to run.
|
||||||
|
|
||||||
|
@ -214,7 +214,7 @@ job being run. It contains the same number as $PARALLEL_SEQ.
|
||||||
See also: B<{}> B<--seqreplace>
|
See also: B<{}> B<--seqreplace>
|
||||||
|
|
||||||
|
|
||||||
=item B<{%}> (alpha testing)
|
=item B<{%}> (beta testing)
|
||||||
|
|
||||||
Job slot number.
|
Job slot number.
|
||||||
|
|
||||||
|
@ -249,7 +249,7 @@ Notice how {%} and $PARALLEL_JOBSLOT differ in the retry run of C and D.
|
||||||
See also: B<{}> B<--jobs> B<--slotreplace>
|
See also: B<{}> B<--jobs> B<--slotreplace>
|
||||||
|
|
||||||
|
|
||||||
=item B<{>I<n>B<}> (alpha testing)
|
=item B<{>I<n>B<}> (beta testing)
|
||||||
|
|
||||||
Argument from input source I<n> or the I<n>'th argument.
|
Argument from input source I<n> or the I<n>'th argument.
|
||||||
|
|
||||||
|
@ -263,7 +263,7 @@ See also: B<{}> B<{>I<n>.B<}> B<{>I<n>/B<}> B<{>I<n>//B<}>
|
||||||
B<{>I<n>/.B<}> B<--colsep>
|
B<{>I<n>/.B<}> B<--colsep>
|
||||||
|
|
||||||
|
|
||||||
=item B<{>I<n>.B<}> (alpha testing)
|
=item B<{>I<n>.B<}> (beta testing)
|
||||||
|
|
||||||
Argument from input source I<n> or the I<n>'th argument without
|
Argument from input source I<n> or the I<n>'th argument without
|
||||||
extension.
|
extension.
|
||||||
|
@ -278,7 +278,7 @@ extension removed.
|
||||||
See also: B<{>I<n>B<}> B<{.}>
|
See also: B<{>I<n>B<}> B<{.}>
|
||||||
|
|
||||||
|
|
||||||
=item B<{>I<n>/B<}> (alpha testing)
|
=item B<{>I<n>/B<}> (beta testing)
|
||||||
|
|
||||||
Basename of argument from input source I<n> or the I<n>'th argument.
|
Basename of argument from input source I<n> or the I<n>'th argument.
|
||||||
|
|
||||||
|
@ -292,7 +292,7 @@ directory (if any) removed.
|
||||||
See also: B<{>I<n>B<}> B<{/}>
|
See also: B<{>I<n>B<}> B<{/}>
|
||||||
|
|
||||||
|
|
||||||
=item B<{>I<n>//B<}> (alpha testing)
|
=item B<{>I<n>//B<}> (beta testing)
|
||||||
|
|
||||||
Dirname of argument from input source I<n> or the I<n>'th argument.
|
Dirname of argument from input source I<n> or the I<n>'th argument.
|
||||||
|
|
||||||
|
@ -305,7 +305,7 @@ the I<n>'th argument (when used with B<-N>). See B<dirname>(1).
|
||||||
See also: B<{>I<n>B<}> B<{//}>
|
See also: B<{>I<n>B<}> B<{//}>
|
||||||
|
|
||||||
|
|
||||||
=item B<{>I<n>/.B<}> (alpha testing)
|
=item B<{>I<n>/.B<}> (beta testing)
|
||||||
|
|
||||||
Basename of argument from input source I<n> or the I<n>'th argument
|
Basename of argument from input source I<n> or the I<n>'th argument
|
||||||
without extension.
|
without extension.
|
||||||
|
@ -321,7 +321,7 @@ directory (if any) and extension removed.
|
||||||
See also: B<{>I<n>B<}> B<{/.}>
|
See also: B<{>I<n>B<}> B<{/.}>
|
||||||
|
|
||||||
|
|
||||||
=item B<{=>I<perl expression>B<=}> (alpha testing)
|
=item B<{=>I<perl expression>B<=}> (beta testing)
|
||||||
|
|
||||||
Replace with calculated I<perl expression>.
|
Replace with calculated I<perl expression>.
|
||||||
|
|
||||||
|
@ -438,7 +438,7 @@ See also: B<--rpl> B<--parens> B<{}> B<{=>I<n> I<perl expression>B<=}>
|
||||||
B<--filter>
|
B<--filter>
|
||||||
|
|
||||||
|
|
||||||
=item B<{=>I<n> I<perl expression>B<=}> (alpha testing)
|
=item B<{=>I<n> I<perl expression>B<=}> (beta testing)
|
||||||
|
|
||||||
Positional equivalent to B<{=>I<perl expression>B<=}>.
|
Positional equivalent to B<{=>I<perl expression>B<=}>.
|
||||||
|
|
||||||
|
@ -447,7 +447,7 @@ To understand positional replacement strings see B<{>I<n>B<}>.
|
||||||
See also: B<{=>I<perl expression>B<=}> B<{>I<n>B<}>
|
See also: B<{=>I<perl expression>B<=}> B<{>I<n>B<}>
|
||||||
|
|
||||||
|
|
||||||
=item B<{>I<rpl>:I<format>B<}> (alpha testing)
|
=item B<{>I<rpl>:I<format>B<}> (beta testing)
|
||||||
|
|
||||||
Format replacement string.
|
Format replacement string.
|
||||||
|
|
||||||
|
@ -1104,6 +1104,27 @@ Implies B<--progress>.
|
||||||
See also: B<--bar> B<--progress> B<--total-jobs>
|
See also: B<--bar> B<--progress> B<--total-jobs>
|
||||||
|
|
||||||
|
|
||||||
|
=item B<--fast> (alpha testing)
|
||||||
|
|
||||||
|
Run jobs fast.
|
||||||
|
|
||||||
|
This disables a lot of functionality of GNU B<parallel> to make jobs
|
||||||
|
run as fast as possible: Think of it as the nitro racing car compared
|
||||||
|
to the Volvo.
|
||||||
|
|
||||||
|
Useful for benchmarking and if you have 1000's of tiny jobs.
|
||||||
|
|
||||||
|
Compare:
|
||||||
|
|
||||||
|
time parallel echo ::: {1..10} ::: {1..10} ::: {1..10}
|
||||||
|
time parallel --fast echo ::: {1..10} ::: {1..10} ::: {1..10}
|
||||||
|
time parallel --fast echo ::: {1..100} ::: {1..100} ::: {1..100}
|
||||||
|
|
||||||
|
Supported options: B<--group> B<--keep-order>
|
||||||
|
|
||||||
|
If you need more options: File a bug.
|
||||||
|
|
||||||
|
|
||||||
=item B<--fg>
|
=item B<--fg>
|
||||||
|
|
||||||
Run command in foreground.
|
Run command in foreground.
|
||||||
|
@ -1796,7 +1817,7 @@ If in doubt use B<-X> as that will most likely do what is needed.
|
||||||
See also: B<-X> B<--xargs>
|
See also: B<-X> B<--xargs>
|
||||||
|
|
||||||
|
|
||||||
=item B<--match> I<regexp> (alpha testing)
|
=item B<--match> I<regexp> (beta testing)
|
||||||
|
|
||||||
Match input source with regexp to set replacement fields.
|
Match input source with regexp to set replacement fields.
|
||||||
|
|
||||||
|
|
|
@ -1555,11 +1555,11 @@ B<parallelion> is fast: 0.1 ms/job. Similar to B<parallel-bash>.
|
||||||
|
|
||||||
=head3 EXAMPLES FROM parallelion
|
=head3 EXAMPLES FROM parallelion
|
||||||
|
|
||||||
1$ parallelion -progress 'echo {}' {1..1000}
|
1$ parallelion --progress 'echo {}' {1..1000}
|
||||||
|
|
||||||
1$ parallel --bar echo {} ::: {1..1000}
|
1$ parallel --bar echo {} ::: {1..1000}
|
||||||
|
|
||||||
2$ parallelion -progress 'echo {}' $(seq 1 999)
|
2$ parallelion --progress 'echo {}' $(seq 1 999)
|
||||||
|
|
||||||
2$ seq 1 999 | parallel --bar echo
|
2$ seq 1 999 | parallel --bar echo
|
||||||
|
|
||||||
|
|
|
@ -137,7 +137,7 @@ GetOptions(
|
||||||
"help" => \$opt::dummy,
|
"help" => \$opt::dummy,
|
||||||
) || exit(255);
|
) || exit(255);
|
||||||
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
|
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
|
||||||
$Global::version = 20240822;
|
$Global::version = 20240823;
|
||||||
if($opt::version) { version(); exit 0; }
|
if($opt::version) { version(); exit 0; }
|
||||||
# Remove -D and --parallel=N
|
# Remove -D and --parallel=N
|
||||||
my @s = (grep { ! /^-D$|^--parallel=\S+$/ }
|
my @s = (grep { ! /^-D$|^--parallel=\S+$/ }
|
||||||
|
|
|
@ -97,7 +97,8 @@ sub pre3 {
|
||||||
# If =for not already there, insert:
|
# If =for not already there, insert:
|
||||||
# =for pod2rst next-code-block: bash
|
# =for pod2rst next-code-block: bash
|
||||||
$all = <STDIN>;
|
$all = <STDIN>;
|
||||||
@codepar = split/(?<=\n .{0,25}\S.{0,200}\n{2,10})(?=[0-9a-zA-Z=])/, $all;
|
# Split after code block (where indented lines => no indent)
|
||||||
|
@codepar = split/(?<=\n .{0,25}\S.{0,200}\n{2,10})(?=^[0-9a-zA-Z=])/m, $all;
|
||||||
for(@codepar) {
|
for(@codepar) {
|
||||||
/=for pod2rst next-code-block:/ and next;
|
/=for pod2rst next-code-block:/ and next;
|
||||||
s/((\n +\S))/\n=for pod2rst next-code-block: bash\n$1/;
|
s/((\n +\S))/\n=for pod2rst next-code-block: bash\n$1/;
|
||||||
|
|
2
src/sql
2
src/sql
|
@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
||||||
exit ($err);
|
exit ($err);
|
||||||
|
|
||||||
sub parse_options {
|
sub parse_options {
|
||||||
$Global::version = 20240822;
|
$Global::version = 20240823;
|
||||||
$Global::progname = 'sql';
|
$Global::progname = 'sql';
|
||||||
|
|
||||||
# This must be done first as this may exec myself
|
# This must be done first as this may exec myself
|
||||||
|
|
|
@ -123,7 +123,7 @@ stop=(stdout ping -w 1 -c 1 {} && \
|
||||||
(cd vagrant/*/{} && vagrant suspend) ) | \
|
(cd vagrant/*/{} && vagrant suspend) ) | \
|
||||||
grep -Ev "${ignore}"
|
grep -Ev "${ignore}"
|
||||||
|
|
||||||
namedservers=centos8 freebsd13 freebsd14 rhel8 centos3
|
namedservers=centos9 freebsd13 freebsd14 rhel8 centos3
|
||||||
servers=parallel-server1 parallel-server2 parallel-server3 parallel-server4 ${namedservers}
|
servers=parallel-server1 parallel-server2 parallel-server3 parallel-server4 ${namedservers}
|
||||||
|
|
||||||
startvm:
|
startvm:
|
||||||
|
|
|
@ -290,7 +290,7 @@ add_server_to_hosts() {
|
||||||
insert_in_etc_hosts 127.1.2.3 server
|
insert_in_etc_hosts 127.1.2.3 server
|
||||||
|
|
||||||
insert_in_etc_hosts 127.0.0.2 lo
|
insert_in_etc_hosts 127.0.0.2 lo
|
||||||
parallel add_single_vagrant_to_etc_hosts {} parallel-server{#} ::: centos8 freebsd13 freebsd14 rhel8 centos3
|
parallel add_single_vagrant_to_etc_hosts {} parallel-server{#} ::: centos9 freebsd13 freebsd14 rhel8 centos3
|
||||||
}
|
}
|
||||||
|
|
||||||
vagrant_up() {
|
vagrant_up() {
|
||||||
|
|
|
@ -984,10 +984,10 @@ par_sem_quote ### sem --quote should not add empty argument
|
||||||
par_sem_quote echo
|
par_sem_quote echo
|
||||||
par_sem_quote
|
par_sem_quote
|
||||||
par_shellcompletion ### --shellcompletion
|
par_shellcompletion ### --shellcompletion
|
||||||
par_shellcompletion bf790ce8a53d1e7e5f8e78b063dc65bc -
|
par_shellcompletion 1eba455fe1456f03376f19255aed28d3 -
|
||||||
par_shellcompletion bf790ce8a53d1e7e5f8e78b063dc65bc -
|
par_shellcompletion 1eba455fe1456f03376f19255aed28d3 -
|
||||||
par_shellcompletion 2230e8ac0593f660fccd983596377613 -
|
par_shellcompletion 1ea63315f24ede14c1e8d93c1b64150a -
|
||||||
par_shellcompletion 2230e8ac0593f660fccd983596377613 -
|
par_shellcompletion 1ea63315f24ede14c1e8d93c1b64150a -
|
||||||
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
|
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
|
||||||
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
|
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
|
||||||
par_slow_pipe_regexp 0 1 1
|
par_slow_pipe_regexp 0 1 1
|
||||||
|
|
|
@ -1855,7 +1855,7 @@ SQLITE par_empty p_wrapper par_empty $SQLITE
|
||||||
SQLITE par_empty Do nothing: TBL99999 does not exist because it is not created
|
SQLITE par_empty Do nothing: TBL99999 does not exist because it is not created
|
||||||
SQLITE par_empty Exit=0
|
SQLITE par_empty Exit=0
|
||||||
SQLITE par_empty Exit=0
|
SQLITE par_empty Exit=0
|
||||||
SQLITE par_empty Parse error near line 1: no such table: TBL99999 (1)
|
SQLITE par_empty Parse error near line 1: no such table: TBL99999
|
||||||
SQLITE par_no_table p_wrapper par_no_table $SQLITE
|
SQLITE par_no_table p_wrapper par_no_table $SQLITE
|
||||||
SQLITE par_no_table bug #50018: --dburl without table dies
|
SQLITE par_no_table bug #50018: --dburl without table dies
|
||||||
SQLITE par_no_table 255
|
SQLITE par_no_table 255
|
||||||
|
|
|
@ -720,6 +720,7 @@ BASH_FUNC_replace_tmpdir%%
|
||||||
BASH_FUNC_run_once%%
|
BASH_FUNC_run_once%%
|
||||||
BASH_FUNC_run_test%%
|
BASH_FUNC_run_test%%
|
||||||
_
|
_
|
||||||
|
mysqlrootpass
|
||||||
qTMPDIR
|
qTMPDIR
|
||||||
qqTMPDIR
|
qqTMPDIR
|
||||||
testsuitedir
|
testsuitedir
|
||||||
|
@ -1061,16 +1062,19 @@ cat: num_%header: No such file or directory
|
||||||
/usr/bin/bash: line 2: foo: command not found
|
/usr/bin/bash: line 2: foo: command not found
|
||||||
#!/usr/bin/perl
|
#!/usr/bin/perl
|
||||||
print "@ARGV\n"
|
print "@ARGV\n"
|
||||||
|
Warning: unknown mime-type for "@ARGV\n" -- using "application/octet-stream"
|
||||||
Error: no such file "@ARGV\n"
|
Error: no such file "@ARGV\n"
|
||||||
parallel perl_echo ::: foo bar
|
parallel perl_echo ::: foo bar
|
||||||
/usr/bin/bash: line 1: perl_echo: command not found
|
/usr/bin/bash: line 1: perl_echo: command not found
|
||||||
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
|
||||||
print "@ARGV\n"
|
print "@ARGV\n"
|
||||||
|
Warning: unknown mime-type for "@ARGV\n" -- using "application/octet-stream"
|
||||||
Error: no such file "@ARGV\n"
|
Error: no such file "@ARGV\n"
|
||||||
perl_echo foo bar
|
perl_echo foo bar
|
||||||
/usr/bin/bash: line 1: perl_echo: command not found
|
/usr/bin/bash: line 1: perl_echo: command not found
|
||||||
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
|
||||||
print "Arguments @ARGV\n";
|
print "Arguments @ARGV\n";
|
||||||
|
Warning: unknown mime-type for "Arguments @ARGV\n" -- using "application/octet-stream"
|
||||||
Error: no such file "Arguments @ARGV\n"
|
Error: no such file "Arguments @ARGV\n"
|
||||||
#!/usr/bin/parallel --shebang-wrap /usr/bin/python
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/python
|
||||||
print 'Arguments', str(sys.argv)
|
print 'Arguments', str(sys.argv)
|
||||||
|
@ -1091,6 +1095,7 @@ Arguments
|
||||||
#!/usr/bin/parallel --shebang-wrap /usr/bin/ruby
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/ruby
|
||||||
print "Arguments "
|
print "Arguments "
|
||||||
puts ARGV
|
puts ARGV
|
||||||
|
Warning: unknown mime-type for "Arguments " -- using "application/octet-stream"
|
||||||
Error: no such file "Arguments "
|
Error: no such file "Arguments "
|
||||||
/usr/bin/bash: line 4: puts: command not found
|
/usr/bin/bash: line 4: puts: command not found
|
||||||
#!/usr/bin/parallel --shebang-wrap /usr/bin/octave
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/octave
|
||||||
|
|
|
@ -3,28 +3,28 @@
|
||||||
134332 134332 940324
|
134332 134332 940324
|
||||||
1
|
1
|
||||||
1
|
1
|
||||||
centos8.localdomain
|
centos9s.localdomain
|
||||||
freebsd14.localdomain
|
freebsd14.localdomain
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
### Test $PARALLEL - multi line
|
### Test $PARALLEL - multi line
|
||||||
1
|
1
|
||||||
1
|
1
|
||||||
centos8.localdomain
|
centos9s.localdomain
|
||||||
freebsd14.localdomain
|
freebsd14.localdomain
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
### Test ~/.parallel/config - single line
|
### Test ~/.parallel/config - single line
|
||||||
1
|
1
|
||||||
1
|
1
|
||||||
centos8.localdomain
|
centos9s.localdomain
|
||||||
freebsd14.localdomain
|
freebsd14.localdomain
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
### Test ~/.parallel/config - multi line
|
### Test ~/.parallel/config - multi line
|
||||||
1
|
1
|
||||||
1
|
1
|
||||||
centos8.localdomain
|
centos9s.localdomain
|
||||||
freebsd14.localdomain
|
freebsd14.localdomain
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
hostname; echo 1
|
hostname; echo 1
|
||||||
|
|
|
@ -1,15 +1,15 @@
|
||||||
par_nonall ### Test --nonall
|
par_nonall ### Test --nonall
|
||||||
par_nonall centos8.localdomain
|
par_nonall centos9s.localdomain
|
||||||
par_nonall freebsd13.localdomain
|
par_nonall freebsd13.localdomain
|
||||||
par_nonall_basefile ### Test --nonall --basefile
|
par_nonall_basefile ### Test --nonall --basefile
|
||||||
par_nonall_basefile /tmp/nonall--basefile
|
par_nonall_basefile /tmp/nonall--basefile
|
||||||
par_nonall_basefile /tmp/nonall--basefile
|
par_nonall_basefile /tmp/nonall--basefile
|
||||||
par_nonall_sshloginfile_stdin ### Test read sshloginfile from STDIN
|
par_nonall_sshloginfile_stdin ### Test read sshloginfile from STDIN
|
||||||
par_nonall_sshloginfile_stdin centos8.localdomain
|
par_nonall_sshloginfile_stdin centos9s.localdomain
|
||||||
par_nonall_sshloginfile_stdin centos8.localdomain
|
par_nonall_sshloginfile_stdin centos9s.localdomain
|
||||||
par_nonall_u ### Test --nonall -u - should be interleaved x y x y
|
par_nonall_u ### Test --nonall -u - should be interleaved x y x y
|
||||||
par_nonall_u 1 centos8.localdomain
|
par_nonall_u 1 centos9s.localdomain
|
||||||
par_nonall_u 1 centos8.localdomain
|
par_nonall_u 1 centos9s.localdomain
|
||||||
par_nonall_u 1 freebsd13.localdomain
|
par_nonall_u 1 freebsd13.localdomain
|
||||||
par_nonall_u 1 freebsd13.localdomain
|
par_nonall_u 1 freebsd13.localdomain
|
||||||
par_onall ### Test --onall
|
par_onall ### Test --onall
|
||||||
|
|
Loading…
Reference in a new issue