tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #62310: xargs compatibility --process-slot-var.

Browse source
This commit is contained in:
Ole Tange 2022-04-30 01:48:26 +02:00
parent 612083626f
commit 57f385d54a
8 changed files with 59 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/Makefile.am
View file

@ -258,39 +258,39 @@ parallel_tutorial.rst: parallel_tutorial.pod
|| echo "Warning: pod2rst not found. Using old parallel_tutorial.rst" || echo "Warning: pod2rst not found. Using old parallel_tutorial.rst"
parallel_book.rst: parallel_book.pod parallel_book.rst: parallel_book.pod
./pod2rst-fix "$(srcdir)"/parallel_book.pod > "$(srcdir)"/parallel_book.rst \ ./pod2rst-fix < "$(srcdir)"/parallel_book.pod > "$(srcdir)"/parallel_book.rst \
|| echo "Warning: pod2rst not found. Using old parallel_book.rst" || echo "Warning: pod2rst not found. Using old parallel_book.rst"
parallel_design.rst: parallel_design.pod parallel_design.rst: parallel_design.pod
./pod2rst-fix "$(srcdir)"/parallel_design.pod > "$(srcdir)"/parallel_design.rst \ ./pod2rst-fix < "$(srcdir)"/parallel_design.pod > "$(srcdir)"/parallel_design.rst \
|| echo "Warning: pod2rst not found. Using old parallel_design.rst" || echo "Warning: pod2rst not found. Using old parallel_design.rst"
parallel_alternatives.rst: parallel_alternatives.pod parallel_alternatives.rst: parallel_alternatives.pod
./pod2rst-fix "$(srcdir)"/parallel_alternatives.pod > "$(srcdir)"/parallel_alternatives.rst \ ./pod2rst-fix < "$(srcdir)"/parallel_alternatives.pod > "$(srcdir)"/parallel_alternatives.rst \
|| echo "Warning: pod2rst not found. Using old parallel_alternatives.rst" || echo "Warning: pod2rst not found. Using old parallel_alternatives.rst"
sem.rst: sem.pod sem.rst: sem.pod
./pod2rst-fix "$(srcdir)"/sem.pod > "$(srcdir)"/sem.rst \ ./pod2rst-fix < "$(srcdir)"/sem.pod > "$(srcdir)"/sem.rst \
|| echo "Warning: pod2rst not found. Using old sem.rst" || echo "Warning: pod2rst not found. Using old sem.rst"
sql.rst: sql sql.rst: sql
./pod2rst-fix "$(srcdir)"/sql > "$(srcdir)"/sql.rst \ ./pod2rst-fix < "$(srcdir)"/sql > "$(srcdir)"/sql.rst \
|| echo "Warning: pod2rst not found. Using old sql.rst" || echo "Warning: pod2rst not found. Using old sql.rst"
niceload.rst: niceload.pod niceload.rst: niceload.pod
./pod2rst-fix "$(srcdir)"/niceload.pod > "$(srcdir)"/niceload.rst \ ./pod2rst-fix < "$(srcdir)"/niceload.pod > "$(srcdir)"/niceload.rst \
|| echo "Warning: pod2rst not found. Using old niceload.rst" || echo "Warning: pod2rst not found. Using old niceload.rst"
parcat.rst: parcat.pod parcat.rst: parcat.pod
./pod2rst-fix "$(srcdir)"/parcat.pod > "$(srcdir)"/parcat.rst \ ./pod2rst-fix < "$(srcdir)"/parcat.pod > "$(srcdir)"/parcat.rst \
|| echo "Warning: pod2rst not found. Using old parcat.rst" || echo "Warning: pod2rst not found. Using old parcat.rst"
parset.rst: parset.pod parset.rst: parset.pod
./pod2rst-fix "$(srcdir)"/parset.pod > "$(srcdir)"/parset.rst \ ./pod2rst-fix < "$(srcdir)"/parset.pod > "$(srcdir)"/parset.rst \
|| echo "Warning: pod2rst not found. Using old parset.rst" || echo "Warning: pod2rst not found. Using old parset.rst"
parsort.rst: parsort parsort.rst: parsort
./pod2rst-fix "$(srcdir)"/parsort > "$(srcdir)"/parsort.rst \ ./pod2rst-fix < "$(srcdir)"/parsort > "$(srcdir)"/parsort.rst \
|| echo "Warning: pod2rst not found. Using old parsort.rst" || echo "Warning: pod2rst not found. Using old parsort.rst"
parallel.pdf: parallel.pod parallel.pdf: parallel.pod

16
src/parallel
View file

@ -1933,6 +1933,9 @@ sub options_completion_hash() {
"E=s" => \$opt::eof, "E=s" => \$opt::eof,
("eof|e:s[Set the end of file string to eof-str]:eof-str" ("eof|e:s[Set the end of file string to eof-str]:eof-str"
=> \$opt::eof), => \$opt::eof),
("process-slot-var|processslotvar=s".
"[Set this variable to job slot number]:varname"
=> \$opt::process_slot_var),
("max-args|maxargs|n=s". ("max-args|maxargs|n=s".
"[Use at most max-args arguments per command line]:max-args" "[Use at most max-args arguments per command line]:max-args"
=> \$opt::max_args), => \$opt::max_args),
@ -2212,6 +2215,9 @@ sub parse_options(@) {
if(defined $opt::arg_file_sep) { if(defined $opt::arg_file_sep) {
$Global::arg_file_sep = $opt::arg_file_sep; $Global::arg_file_sep = $opt::arg_file_sep;
} }
if(not defined $opt::process_slot_var) {
$opt::process_slot_var = 'PARALLEL_JOBSLOT0';
}
if(defined $opt::number_of_sockets) { if(defined $opt::number_of_sockets) {
print SSHLogin::no_of_sockets(),"\n"; wait_and_exit(0); print SSHLogin::no_of_sockets(),"\n"; wait_and_exit(0);
} }
@ -9910,8 +9916,8 @@ sub sshlogin_wrap($) {
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
"PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST",
"PARALLEL_HOSTGROUPS", "PARALLEL_ARGHOSTGROUPS", "PARALLEL_HOSTGROUPS", "PARALLEL_ARGHOSTGROUPS",
"PARALLEL_JOBSLOT", map { ("BASH_FUNC_$_()", "PARALLEL_JOBSLOT", $opt::process_slot_var,
"BASH_FUNC_$_%%") } @vars); map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
# Keep only defined variables # Keep only defined variables
return grep { defined($ENV{$_}) } @vars; return grep { defined($ENV{$_}) } @vars;
} }
@ -9967,7 +9973,8 @@ sub sshlogin_wrap($) {
if(not defined $self->{'sshlogin_wrap'}{$command}) { if(not defined $self->{'sshlogin_wrap'}{$command}) {
my $sshlogin = $self->sshlogin(); my $sshlogin = $self->sshlogin();
$ENV{'PARALLEL_SEQ'} = $self->seq(); $ENV{'PARALLEL_SEQ'} = $self->seq();
$ENV{'PARALLEL_JOBSLOT'} = $self->slot(); $ENV{$opt::process_slot_var} = -1 +
($ENV{'PARALLEL_JOBSLOT'} = $self->slot());
$ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string(); $ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string();
$ENV{'PARALLEL_SSHHOST'} = $sshlogin->host(); $ENV{'PARALLEL_SSHHOST'} = $sshlogin->host();
if ($opt::hostgroups) { if ($opt::hostgroups) {
@ -10474,7 +10481,8 @@ sub start($) {
if($opt::dryrun or $opt::sqlmaster) { $command = "true"; } if($opt::dryrun or $opt::sqlmaster) { $command = "true"; }
$ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_SEQ'} = $job->seq();
$ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_PID'} = $$;
$ENV{'PARALLEL_JOBSLOT'} = $job->slot(); $ENV{$opt::process_slot_var} = -1 +
($ENV{'PARALLEL_JOBSLOT'} = $job->slot());
$ENV{'PARALLEL_TMP'} = ::tmpname("par"); $ENV{'PARALLEL_TMP'} = ::tmpname("par");
$job->add_rm($ENV{'PARALLEL_TMP'}); $job->add_rm($ENV{'PARALLEL_TMP'});
$job->fill_templates(); $job->fill_templates();

7
src/parallel.pod
View file

@ -1929,6 +1929,13 @@ inspired by bash's parameter expansion:
See also: B<--rpl> B<{}> See also: B<--rpl> B<{}>
=item B<--process-slot-var> I<varname> (alpha testing)
Set the environment variable I<varname> to the jobslot number-1.
seq 10 | parallel --process-slot-var=name echo '$name' {}
=item B<--progress> =item B<--progress>
Show progress of computations. Show progress of computations.

1
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -8,7 +8,6 @@
# Each should be taking 1-3s and be possible to run in parallel # Each should be taking 1-3s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_commandline_with_newline() { par_commandline_with_newline() {
echo 'bug #51299: --retry-failed with command with newline' echo 'bug #51299: --retry-failed with command with newline'
echo 'The format must remain the same' echo 'The format must remain the same'

10
testsuite/tests-to-run/parallel-local-3s.sh
View file

@ -8,6 +8,16 @@
# Each should be taking 3-10s and be possible to run in parallel # Each should be taking 3-10s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_process_slot_var() {
echo '### bug #62310: xargs compatibility: --process-slot-var=name'
seq 0.1 0.1 0.5 |
parallel -n1 -P4 --process-slot-var=name -q bash -c 'sleep $1; echo "$name"' _
seq 0.1 0.1 0.5 |
xargs -n1 -P4 --process-slot-var=name bash -c 'sleep $1; echo "$name"' _
seq 0.1 0.1 0.5 |
parallel -P4 --process-slot-var=name sleep {}\; echo '$name'
}
par_retries_0() { par_retries_0() {
echo '--retries 0 = inf' echo '--retries 0 = inf'
echo this wraps at 256 and should retry until it wraps echo this wraps at 256 and should retry until it wraps

8
testsuite/wanted-results/parallel-local-0.3s
View file

@ -953,10 +953,10 @@ par_sem_quote ### sem --quote should not add empty argument
par_sem_quote echo par_sem_quote echo
par_sem_quote par_sem_quote
par_shellcompletion ### --shellcompletion par_shellcompletion ### --shellcompletion
par_shellcompletion 920d653deec2b96528d7dcfe396f771f - par_shellcompletion f4aa0b2d61cf5e6836b738bf403c1d96 -
par_shellcompletion 920d653deec2b96528d7dcfe396f771f - par_shellcompletion f4aa0b2d61cf5e6836b738bf403c1d96 -
par_shellcompletion 5a53f2595ceeefbf3fb70d0cae4af71c - par_shellcompletion be5b6ba646ba3f082315afd8e205c954 -
par_shellcompletion 5a53f2595ceeefbf3fb70d0cae4af71c - par_shellcompletion be5b6ba646ba3f082315afd8e205c954 -
par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks
par_slow_pipe_regexp This should take a few ms, but took more than 2 hours par_slow_pipe_regexp This should take a few ms, but took more than 2 hours
par_slow_pipe_regexp 0 1 1 par_slow_pipe_regexp 0 1 1

16
testsuite/wanted-results/parallel-local-3s
View file

@ -569,6 +569,22 @@ par_prefix_for_L_n_N_s 244
par_prefix_for_L_n_N_s 246 par_prefix_for_L_n_N_s 246
par_prefix_for_L_n_N_s 246 par_prefix_for_L_n_N_s 246
par_prefix_for_L_n_N_s 273 par_prefix_for_L_n_N_s 273
par_process_slot_var ### bug #62310: xargs compatibility: --process-slot-var=name
par_process_slot_var 0
par_process_slot_var 1
par_process_slot_var 2
par_process_slot_var 3
par_process_slot_var 0
par_process_slot_var 0
par_process_slot_var 1
par_process_slot_var 2
par_process_slot_var 3
par_process_slot_var 0
par_process_slot_var 0
par_process_slot_var 1
par_process_slot_var 2
par_process_slot_var 3
par_process_slot_var 0
par_progress ### Test of --progress par_progress ### Test of --progress
par_progress 16 par_progress 16
par_progress ### Test of --progress with no jobs par_progress ### Test of --progress with no jobs

2
testsuite/wanted-results/parallel-tutorial
View file

@ -847,7 +847,7 @@ For details: see man env_parallel
export -f my_func3 export -f my_func3
parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \ parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \
-S $SERVER1 my_func3 {} ::: abc-file -S $SERVER1 my_func3 {} ::: abc-file
ssh -l parallel lo -- exec mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -X -e GNU_Parallel_worker,eval+pack+q/H10000000/,join+q//,@ARGV BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 -rlDzR -e'ssh -l parallel' --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -- lo:./abc-file.out ./.;ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec rm -rf .TMPWORKDIR;exit $_EXIT_status; ssh -l parallel lo -- exec mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -X -e GNU_Parallel_worker,eval+pack+q/H10000000/,join+q//,@ARGV BASE64.BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 -rlDzR -e'ssh -l parallel' --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -- lo:./abc-file.out ./.;ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec rm -rf .TMPWORKDIR;exit $_EXIT_status;
parset myvar1,myvar2 echo ::: a b parset myvar1,myvar2 echo ::: a b
echo $myvar1 echo $myvar1
echo $myvar2 echo $myvar2