diff --git a/src/parallel b/src/parallel index 34700c7b..7f9375ea 100755 --- a/src/parallel +++ b/src/parallel @@ -2212,6 +2212,12 @@ sub init_globals() { '{choose_k}' => 'for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } }', # {##} = number of jobs '{##}' => '$_=total_jobs()', + # {0%} = 0-padded jobslot + '{0%}' => '$f=1+int((log($Global::max_jobs_running||1)/log(10))); $_=sprintf("%0${f}d",slot())', + # {0%} = 0-padded seq + '{0#}' => '$f=1+int((log(total_jobs())/log(10))); $_=sprintf("%0${f}d",seq())', + + ## Bash inspired replacement strings # Bash ${a:-myval} '{:-([^}]+?)}' => '$_ ||= $$1', # Bash ${a:2} @@ -6301,6 +6307,7 @@ sub set_max_jobs_running($$) { $Global::max_jobs_running -= $self->{'max_jobs_running'}; } $self->{'max_jobs_running'} = shift; + if(defined $self->{'max_jobs_running'}) { # max_jobs_running could be resat if -j is a changed file $Global::max_jobs_running += $self->{'max_jobs_running'}; @@ -10811,12 +10818,13 @@ sub new($) { sub flush_cache() { my $self = shift; - for my $arglist (@{$self->{'arglist'}}) { + for my $arglist (@{$self->{'arg_list'}}) { for my $arg (@$arglist) { $arg->flush_cache(); } } $self->{'arg_queue'}->flush_cache(); + $self->{'replaced'} = undef; } sub seq($) { @@ -13657,6 +13665,7 @@ sub main() { # Compute $Global::max_jobs_running # Must be done after ungetting any --pipepart jobs. max_jobs_running(); + init_run_jobs(); my $sem; if($Global::semaphore) { diff --git a/src/parallel.pod b/src/parallel.pod index df65ade5..fc8ea174 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -299,14 +299,20 @@ directory (if any) and extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{=>IB<=}> +=item B<{=>IB<=}> (alpha testing) Replace with calculated I. B<$_> will contain the same as B<{}>. After evaluating I B<$_> will be used as the value. It is recommended to only change $_ but you have full access to all of GNU B's internal functions and data -structures. A few convenience functions and data structures have been -made: +structures. + +The expression must give the same result if evaluated twice - +otherwise the behaviour is undefined. E.g. this will not work as expected: + + parallel echo '{= $_= ++$wrong_counter =}' ::: a b c + +A few convenience functions and data structures have been made: =over 15 @@ -870,7 +876,7 @@ replacement strings and Perl code. Example: parallel --filter '{1} < {2}+1' echo ::: {1..3} ::: {1..3} -Runs: 1,1 1,2 1,3 2,2 2,3 3,3 +Outputs: 1,1 1,2 1,3 2,2 2,3 3,3 =item B<--filter-hosts> @@ -1625,11 +1631,9 @@ Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} B<{##}> is the total number of jobs to be run. It is incompatible with B<-X>/B<-m>/B<--xargs>. -B<{0%}>, B<{00%}>, B<{000%}>, ... B<{0..0%}> prefix jobslot with -zeros. (alpha testing) +B<{0%}> zero-padded jobslot. (alpha testing) -B<{0#}>, B<{00#}>, B<{000#}>, ... B<{0..0#}> prefix sequence numbers -with zeros. (alpha testing) +B<{0#}> zero-padded sequence number. (alpha testing) B<{choose_k}> is inspired by n choose k: Given a list of n elements, choose k. k is the number of input sources and n is the number of @@ -2640,6 +2644,23 @@ will be replaced. With B<--cleanup> the new file will be removed when the job is done. +If I contains this: + + Xval: {x} + Yval: {y} + FixedValue: 9 + # x with 2 decimals + DecimalX: {=x $_=sprintf("%.2f",$_) =} + TenX: {=x $_=$_*10 =} + RandomVal: {=1 $_=rand() =} + +it can be used like this: + + myprog() { echo Using "$@"; cat "$@"; } + export -f myprog + parallel --cleanup --header : --tmpl my.tmpl={#}.t myprog {#}.t \ + ::: x 1.234 2.345 3.45678 ::: y 1 2 3 + =item B<--tty> diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 6e03ad12..6132376f 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -2862,10 +2862,8 @@ Distributed Resource Management Application API to make jobs run with different cluster software. -=head3 Future support in GNU Parallel - -B versions of GNU B may support templates -with replacement strings. Such as: +GNU B B<--tmpl> supports templates with replacement +strings. Such as: Xval: {x} Yval: {y} @@ -2873,21 +2871,17 @@ with replacement strings. Such as: # x with 2 decimals DecimalX: {=x $_=sprintf("%.2f",$_) =} TenX: {=x $_=$_*10 =} - RandomVal: {= $_=rand() =} + RandomVal: {=1 $_=rand() =} that can be used like: parallel --header : --tmpl my.tmpl={#}.t myprog {#}.t \ ::: x 1 2 3 ::: y 1 2 3 -Filtering may also be supported as: +Filtering is supported as: parallel --filter '{1} > {2}' echo ::: 1 2 3 ::: 1 2 3 -which will basically do: - - parallel echo '{= $arg[1] > $arg[2] and skip() =}' ::: 1 2 3 ::: 1 2 3 - https://github.com/eviatarbach/parasweep (Last checked: 2021-01) @@ -2936,6 +2930,28 @@ https://reposhub.com/python/command-line-tools/Akianonymus-parallel-bash.html (Last checked: 2021-02) +=head2 DIFFERENCES BETWEEN bash-concurrent AND GNU Parallel + +B is more an alternative to B than to GNU +B. Its input is very similar to a Makefile, where jobs +depend on other jobs. + +It has a nice progress indicator where you can see which jobs +completed successfully, which jobs are currently running, which jobs +failed, and which jobs were skipped due to a depending job failed. +The indicator does not deal well with resizing the window. + +Output is cached in tempfiles on disk, but is only shown if there is +an error, so it is not meant to be part of a UNIX pipeline. If +B crashes these tempfiles are not removed. + +It uses an O(n*n) algorithm, so if you have 1000 independent jobs it +takes 22 seconds to start it. + +https://github.com/themattrix/bash-concurrent +(Last checked: 2021-02) + + =head2 Todo https://github.com/Nukesor/pueue diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index bc39e6f1..36eb75fb 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -69,12 +69,12 @@ EOF cat <<'EOF' > "$tmp2" 2: Job:{#} Slot:{%} All:{} Arg[1]:{1} Arg[-1]:{-1} Perl({}+4):{=$_+=4=} EOF - parallel -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \ - 'cat t1.{#} t2.{%}' ::: 1 2 ::: a b + parallel --colsep , -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \ + 'sleep 0.{#}; cat t1.{#} t2.{%}' ::: 1,a 1,b 2,a 2,b echo should give no files ls t[12].* - parallel --cleanup --colsep , -j2 --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \ - 'cat t1.{#} t2.{%}' ::: 1,a 1,b 2,a 2,b + parallel -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \ + 'sleep 0.{#}; cat t1.{#} t2.{%}' ::: 1 2 ::: a b echo should give no files ls t[12].* } diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 9187ae5f..1e74499c 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -282,6 +282,10 @@ par_open_files_blocks 9 of 21 par_open_files_blocks 10 of 21 par_open_files_blocks 11 of 21 par_open_files_blocks 12 of 21 +par_open_files_blocks parallel: Warning: Try running 'parallel -j0 -N 100 --pipe parallel -j0' +par_open_files_blocks parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`) +par_open_files_blocks parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf +par_open_files_blocks parallel: Warning: or increasing /proc/sys/fs/file-max par_open_files_blocks 13 of 21 par_open_files_blocks 14 of 21 par_open_files_blocks 15 of 21 @@ -479,21 +483,21 @@ par_plus_slot_replacement Seq: 097 97 par_plus_slot_replacement Seq: 098 98 par_plus_slot_replacement Seq: 099 99 par_plus_slot_replacement Seq: 100 100 -par_plus_slot_replacement Slot: 1 1 +par_plus_slot_replacement Slot: 01 1 +par_plus_slot_replacement Slot: 02 2 +par_plus_slot_replacement Slot: 03 3 +par_plus_slot_replacement Slot: 04 4 +par_plus_slot_replacement Slot: 05 5 +par_plus_slot_replacement Slot: 06 6 +par_plus_slot_replacement Slot: 07 7 +par_plus_slot_replacement Slot: 08 8 +par_plus_slot_replacement Slot: 09 9 par_plus_slot_replacement Slot: 10 10 par_plus_slot_replacement Slot: 11 11 par_plus_slot_replacement Slot: 12 12 par_plus_slot_replacement Slot: 13 13 par_plus_slot_replacement Slot: 14 14 par_plus_slot_replacement Slot: 15 15 -par_plus_slot_replacement Slot: 2 2 -par_plus_slot_replacement Slot: 3 3 -par_plus_slot_replacement Slot: 4 4 -par_plus_slot_replacement Slot: 5 5 -par_plus_slot_replacement Slot: 6 6 -par_plus_slot_replacement Slot: 7 7 -par_plus_slot_replacement Slot: 8 8 -par_plus_slot_replacement Slot: 9 9 par_profiles_with_space ### bug #42902: profiles containing arguments with space par_profiles_with_space /bin/bash=/bin/bash par_profiles_with_space echo '/bin/bash=/bin/bash'