mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-10-18 12:59:39 +00:00
parallel: {0%}/{0#} zero-padded jobslot/sequence number.
This commit is contained in:
Ole Tange
parent
fff769f675
commit
b46e90b7c5
11
src/parallel
11
src/parallel
|
|
@ -2212,6 +2212,12 @@ sub init_globals() {
|
|||
'{choose_k}' => 'for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } }',
|
||||
# {##} = number of jobs
|
||||
'{##}' => '$_=total_jobs()',
|
||||
# {0%} = 0-padded jobslot
|
||||
'{0%}' => '$f=1+int((log($Global::max_jobs_running||1)/log(10))); $_=sprintf("%0${f}d",slot())',
|
||||
# {0%} = 0-padded seq
|
||||
'{0#}' => '$f=1+int((log(total_jobs())/log(10))); $_=sprintf("%0${f}d",seq())',
|
||||
|
||||
## Bash inspired replacement strings
|
||||
# Bash ${a:-myval}
|
||||
'{:-([^}]+?)}' => '$_ ||= $$1',
|
||||
# Bash ${a:2}
|
||||
|
|
@ -6301,6 +6307,7 @@ sub set_max_jobs_running($$) {
|
|||
$Global::max_jobs_running -= $self->{'max_jobs_running'};
|
||||
}
|
||||
$self->{'max_jobs_running'} = shift;
|
||||
|
||||
if(defined $self->{'max_jobs_running'}) {
|
||||
# max_jobs_running could be resat if -j is a changed file
|
||||
$Global::max_jobs_running += $self->{'max_jobs_running'};
|
||||
|
|
@ -10811,12 +10818,13 @@ sub new($) {
|
|||
|
||||
sub flush_cache() {
|
||||
my $self = shift;
|
||||
for my $arglist (@{$self->{'arglist'}}) {
|
||||
for my $arglist (@{$self->{'arg_list'}}) {
|
||||
for my $arg (@$arglist) {
|
||||
$arg->flush_cache();
|
||||
}
|
||||
}
|
||||
$self->{'arg_queue'}->flush_cache();
|
||||
$self->{'replaced'} = undef;
|
||||
}
|
||||
|
||||
sub seq($) {
|
||||
|
|
@ -13657,6 +13665,7 @@ sub main() {
|
|||
# Compute $Global::max_jobs_running
|
||||
# Must be done after ungetting any --pipepart jobs.
|
||||
max_jobs_running();
|
||||
|
||||
init_run_jobs();
|
||||
my $sem;
|
||||
if($Global::semaphore) {
|
||||
|
|
|
|||
|
|
@ -299,14 +299,20 @@ directory (if any) and extension removed.
|
|||
To understand positional replacement strings see B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{=>I<perl expression>B<=}>
|
||||
=item B<{=>I<perl expression>B<=}> (alpha testing)
|
||||
|
||||
Replace with calculated I<perl expression>. B<$_> will contain the
|
||||
same as B<{}>. After evaluating I<perl expression> B<$_> will be used
|
||||
as the value. It is recommended to only change $_ but you have full
|
||||
access to all of GNU B<parallel>'s internal functions and data
|
||||
structures. A few convenience functions and data structures have been
|
||||
made:
|
||||
structures.
|
||||
|
||||
The expression must give the same result if evaluated twice -
|
||||
otherwise the behaviour is undefined. E.g. this will not work as expected:
|
||||
|
||||
parallel echo '{= $_= ++$wrong_counter =}' ::: a b c
|
||||
|
||||
A few convenience functions and data structures have been made:
|
||||
|
||||
=over 15
|
||||
|
||||
|
|
@ -870,7 +876,7 @@ replacement strings and Perl code. Example:
|
|||
|
||||
parallel --filter '{1} < {2}+1' echo ::: {1..3} ::: {1..3}
|
||||
|
||||
Runs: 1,1 1,2 1,3 2,2 2,3 3,3
|
||||
Outputs: 1,1 1,2 1,3 2,2 2,3 3,3
|
||||
|
||||
|
||||
=item B<--filter-hosts>
|
||||
|
|
@ -1625,11 +1631,9 @@ Activate additional replacement strings: {+/} {+.} {+..} {+...} {..}
|
|||
B<{##}> is the total number of jobs to be run. It is incompatible with
|
||||
B<-X>/B<-m>/B<--xargs>.
|
||||
|
||||
B<{0%}>, B<{00%}>, B<{000%}>, ... B<{0..0%}> prefix jobslot with
|
||||
zeros. (alpha testing)
|
||||
B<{0%}> zero-padded jobslot. (alpha testing)
|
||||
|
||||
B<{0#}>, B<{00#}>, B<{000#}>, ... B<{0..0#}> prefix sequence numbers
|
||||
with zeros. (alpha testing)
|
||||
B<{0#}> zero-padded sequence number. (alpha testing)
|
||||
|
||||
B<{choose_k}> is inspired by n choose k: Given a list of n elements,
|
||||
choose k. k is the number of input sources and n is the number of
|
||||
|
|
@ -2640,6 +2644,23 @@ will be replaced.
|
|||
|
||||
With B<--cleanup> the new file will be removed when the job is done.
|
||||
|
||||
If I<my.tmpl> contains this:
|
||||
|
||||
Xval: {x}
|
||||
Yval: {y}
|
||||
FixedValue: 9
|
||||
# x with 2 decimals
|
||||
DecimalX: {=x $_=sprintf("%.2f",$_) =}
|
||||
TenX: {=x $_=$_*10 =}
|
||||
RandomVal: {=1 $_=rand() =}
|
||||
|
||||
it can be used like this:
|
||||
|
||||
myprog() { echo Using "$@"; cat "$@"; }
|
||||
export -f myprog
|
||||
parallel --cleanup --header : --tmpl my.tmpl={#}.t myprog {#}.t \
|
||||
::: x 1.234 2.345 3.45678 ::: y 1 2 3
|
||||
|
||||
|
||||
=item B<--tty>
|
||||
|
||||
|
|
|
|||
|
|
@ -2862,10 +2862,8 @@ Distributed Resource Management Application API to make jobs run with
|
|||
different cluster software.
|
||||
|
||||
|
||||
=head3 Future support in GNU Parallel
|
||||
|
||||
B<Future> versions of GNU B<parallel> may support templates
|
||||
with replacement strings. Such as:
|
||||
GNU B<parallel> B<--tmpl> supports templates with replacement
|
||||
strings. Such as:
|
||||
|
||||
Xval: {x}
|
||||
Yval: {y}
|
||||
|
|
@ -2873,21 +2871,17 @@ with replacement strings. Such as:
|
|||
# x with 2 decimals
|
||||
DecimalX: {=x $_=sprintf("%.2f",$_) =}
|
||||
TenX: {=x $_=$_*10 =}
|
||||
RandomVal: {= $_=rand() =}
|
||||
RandomVal: {=1 $_=rand() =}
|
||||
|
||||
that can be used like:
|
||||
|
||||
parallel --header : --tmpl my.tmpl={#}.t myprog {#}.t \
|
||||
::: x 1 2 3 ::: y 1 2 3
|
||||
|
||||
Filtering may also be supported as:
|
||||
Filtering is supported as:
|
||||
|
||||
parallel --filter '{1} > {2}' echo ::: 1 2 3 ::: 1 2 3
|
||||
|
||||
which will basically do:
|
||||
|
||||
parallel echo '{= $arg[1] > $arg[2] and skip() =}' ::: 1 2 3 ::: 1 2 3
|
||||
|
||||
https://github.com/eviatarbach/parasweep (Last checked: 2021-01)
|
||||
|
||||
|
||||
|
|
@ -2936,6 +2930,28 @@ https://reposhub.com/python/command-line-tools/Akianonymus-parallel-bash.html
|
|||
(Last checked: 2021-02)
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN bash-concurrent AND GNU Parallel
|
||||
|
||||
B<bash-concurrent> is more an alternative to B<make> than to GNU
|
||||
B<parallel>. Its input is very similar to a Makefile, where jobs
|
||||
depend on other jobs.
|
||||
|
||||
It has a nice progress indicator where you can see which jobs
|
||||
completed successfully, which jobs are currently running, which jobs
|
||||
failed, and which jobs were skipped due to a depending job failed.
|
||||
The indicator does not deal well with resizing the window.
|
||||
|
||||
Output is cached in tempfiles on disk, but is only shown if there is
|
||||
an error, so it is not meant to be part of a UNIX pipeline. If
|
||||
B<bash-concurrent> crashes these tempfiles are not removed.
|
||||
|
||||
It uses an O(n*n) algorithm, so if you have 1000 independent jobs it
|
||||
takes 22 seconds to start it.
|
||||
|
||||
https://github.com/themattrix/bash-concurrent
|
||||
(Last checked: 2021-02)
|
||||
|
||||
|
||||
=head2 Todo
|
||||
|
||||
https://github.com/Nukesor/pueue
|
||||
|
|
|
|||
|
|
@ -69,12 +69,12 @@ EOF
|
|||
cat <<'EOF' > "$tmp2"
|
||||
2: Job:{#} Slot:{%} All:{} Arg[1]:{1} Arg[-1]:{-1} Perl({}+4):{=$_+=4=}
|
||||
EOF
|
||||
parallel -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \
|
||||
'cat t1.{#} t2.{%}' ::: 1 2 ::: a b
|
||||
parallel --colsep , -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \
|
||||
'sleep 0.{#}; cat t1.{#} t2.{%}' ::: 1,a 1,b 2,a 2,b
|
||||
echo should give no files
|
||||
ls t[12].*
|
||||
parallel --cleanup --colsep , -j2 --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \
|
||||
'cat t1.{#} t2.{%}' ::: 1,a 1,b 2,a 2,b
|
||||
parallel -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \
|
||||
'sleep 0.{#}; cat t1.{#} t2.{%}' ::: 1 2 ::: a b
|
||||
echo should give no files
|
||||
ls t[12].*
|
||||
}
|
||||
|
|
|
|||
|
|
@ -282,6 +282,10 @@ par_open_files_blocks 9 of 21
|
|||
par_open_files_blocks 10 of 21
|
||||
par_open_files_blocks 11 of 21
|
||||
par_open_files_blocks 12 of 21
|
||||
par_open_files_blocks parallel: Warning: Try running 'parallel -j0 -N 100 --pipe parallel -j0'
|
||||
par_open_files_blocks parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`)
|
||||
par_open_files_blocks parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf
|
||||
par_open_files_blocks parallel: Warning: or increasing /proc/sys/fs/file-max
|
||||
par_open_files_blocks 13 of 21
|
||||
par_open_files_blocks 14 of 21
|
||||
par_open_files_blocks 15 of 21
|
||||
|
|
@ -479,21 +483,21 @@ par_plus_slot_replacement Seq: 097 97
|
|||
par_plus_slot_replacement Seq: 098 98
|
||||
par_plus_slot_replacement Seq: 099 99
|
||||
par_plus_slot_replacement Seq: 100 100
|
||||
par_plus_slot_replacement Slot: 1 1
|
||||
par_plus_slot_replacement Slot: 01 1
|
||||
par_plus_slot_replacement Slot: 02 2
|
||||
par_plus_slot_replacement Slot: 03 3
|
||||
par_plus_slot_replacement Slot: 04 4
|
||||
par_plus_slot_replacement Slot: 05 5
|
||||
par_plus_slot_replacement Slot: 06 6
|
||||
par_plus_slot_replacement Slot: 07 7
|
||||
par_plus_slot_replacement Slot: 08 8
|
||||
par_plus_slot_replacement Slot: 09 9
|
||||
par_plus_slot_replacement Slot: 10 10
|
||||
par_plus_slot_replacement Slot: 11 11
|
||||
par_plus_slot_replacement Slot: 12 12
|
||||
par_plus_slot_replacement Slot: 13 13
|
||||
par_plus_slot_replacement Slot: 14 14
|
||||
par_plus_slot_replacement Slot: 15 15
|
||||
par_plus_slot_replacement Slot: 2 2
|
||||
par_plus_slot_replacement Slot: 3 3
|
||||
par_plus_slot_replacement Slot: 4 4
|
||||
par_plus_slot_replacement Slot: 5 5
|
||||
par_plus_slot_replacement Slot: 6 6
|
||||
par_plus_slot_replacement Slot: 7 7
|
||||
par_plus_slot_replacement Slot: 8 8
|
||||
par_plus_slot_replacement Slot: 9 9
|
||||
par_profiles_with_space ### bug #42902: profiles containing arguments with space
|
||||
par_profiles_with_space /bin/bash=/bin/bash
|
||||
par_profiles_with_space echo '/bin/bash=/bin/bash'
|
||||
|
|
|
|||
Loading…
Reference in a new issue