From 57f385d54a66854042ea6e92bf43cbe848ce5afd Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 30 Apr 2022 01:48:26 +0200 Subject: [PATCH] parallel: Fixed bug #62310: xargs compatibility --process-slot-var. --- src/Makefile.am | 18 +++++++++--------- src/parallel | 16 ++++++++++++---- src/parallel.pod | 7 +++++++ testsuite/tests-to-run/parallel-local-1s.sh | 1 - testsuite/tests-to-run/parallel-local-3s.sh | 10 ++++++++++ testsuite/wanted-results/parallel-local-0.3s | 8 ++++---- testsuite/wanted-results/parallel-local-3s | 16 ++++++++++++++++ testsuite/wanted-results/parallel-tutorial | 2 +- 8 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 787e12eb..a77dff06 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -258,39 +258,39 @@ parallel_tutorial.rst: parallel_tutorial.pod || echo "Warning: pod2rst not found. Using old parallel_tutorial.rst" parallel_book.rst: parallel_book.pod - ./pod2rst-fix "$(srcdir)"/parallel_book.pod > "$(srcdir)"/parallel_book.rst \ + ./pod2rst-fix < "$(srcdir)"/parallel_book.pod > "$(srcdir)"/parallel_book.rst \ || echo "Warning: pod2rst not found. Using old parallel_book.rst" parallel_design.rst: parallel_design.pod - ./pod2rst-fix "$(srcdir)"/parallel_design.pod > "$(srcdir)"/parallel_design.rst \ + ./pod2rst-fix < "$(srcdir)"/parallel_design.pod > "$(srcdir)"/parallel_design.rst \ || echo "Warning: pod2rst not found. Using old parallel_design.rst" parallel_alternatives.rst: parallel_alternatives.pod - ./pod2rst-fix "$(srcdir)"/parallel_alternatives.pod > "$(srcdir)"/parallel_alternatives.rst \ + ./pod2rst-fix < "$(srcdir)"/parallel_alternatives.pod > "$(srcdir)"/parallel_alternatives.rst \ || echo "Warning: pod2rst not found. Using old parallel_alternatives.rst" sem.rst: sem.pod - ./pod2rst-fix "$(srcdir)"/sem.pod > "$(srcdir)"/sem.rst \ + ./pod2rst-fix < "$(srcdir)"/sem.pod > "$(srcdir)"/sem.rst \ || echo "Warning: pod2rst not found. Using old sem.rst" sql.rst: sql - ./pod2rst-fix "$(srcdir)"/sql > "$(srcdir)"/sql.rst \ + ./pod2rst-fix < "$(srcdir)"/sql > "$(srcdir)"/sql.rst \ || echo "Warning: pod2rst not found. Using old sql.rst" niceload.rst: niceload.pod - ./pod2rst-fix "$(srcdir)"/niceload.pod > "$(srcdir)"/niceload.rst \ + ./pod2rst-fix < "$(srcdir)"/niceload.pod > "$(srcdir)"/niceload.rst \ || echo "Warning: pod2rst not found. Using old niceload.rst" parcat.rst: parcat.pod - ./pod2rst-fix "$(srcdir)"/parcat.pod > "$(srcdir)"/parcat.rst \ + ./pod2rst-fix < "$(srcdir)"/parcat.pod > "$(srcdir)"/parcat.rst \ || echo "Warning: pod2rst not found. Using old parcat.rst" parset.rst: parset.pod - ./pod2rst-fix "$(srcdir)"/parset.pod > "$(srcdir)"/parset.rst \ + ./pod2rst-fix < "$(srcdir)"/parset.pod > "$(srcdir)"/parset.rst \ || echo "Warning: pod2rst not found. Using old parset.rst" parsort.rst: parsort - ./pod2rst-fix "$(srcdir)"/parsort > "$(srcdir)"/parsort.rst \ + ./pod2rst-fix < "$(srcdir)"/parsort > "$(srcdir)"/parsort.rst \ || echo "Warning: pod2rst not found. Using old parsort.rst" parallel.pdf: parallel.pod diff --git a/src/parallel b/src/parallel index ccf9ad65..72633895 100755 --- a/src/parallel +++ b/src/parallel @@ -1933,6 +1933,9 @@ sub options_completion_hash() { "E=s" => \$opt::eof, ("eof|e:s[Set the end of file string to eof-str]:eof-str" => \$opt::eof), + ("process-slot-var|processslotvar=s". + "[Set this variable to job slot number]:varname" + => \$opt::process_slot_var), ("max-args|maxargs|n=s". "[Use at most max-args arguments per command line]:max-args" => \$opt::max_args), @@ -2212,6 +2215,9 @@ sub parse_options(@) { if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; } + if(not defined $opt::process_slot_var) { + $opt::process_slot_var = 'PARALLEL_JOBSLOT0'; + } if(defined $opt::number_of_sockets) { print SSHLogin::no_of_sockets(),"\n"; wait_and_exit(0); } @@ -9910,8 +9916,8 @@ sub sshlogin_wrap($) { push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_HOSTGROUPS", "PARALLEL_ARGHOSTGROUPS", - "PARALLEL_JOBSLOT", map { ("BASH_FUNC_$_()", - "BASH_FUNC_$_%%") } @vars); + "PARALLEL_JOBSLOT", $opt::process_slot_var, + map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars); # Keep only defined variables return grep { defined($ENV{$_}) } @vars; } @@ -9967,7 +9973,8 @@ sub sshlogin_wrap($) { if(not defined $self->{'sshlogin_wrap'}{$command}) { my $sshlogin = $self->sshlogin(); $ENV{'PARALLEL_SEQ'} = $self->seq(); - $ENV{'PARALLEL_JOBSLOT'} = $self->slot(); + $ENV{$opt::process_slot_var} = -1 + + ($ENV{'PARALLEL_JOBSLOT'} = $self->slot()); $ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string(); $ENV{'PARALLEL_SSHHOST'} = $sshlogin->host(); if ($opt::hostgroups) { @@ -10474,7 +10481,8 @@ sub start($) { if($opt::dryrun or $opt::sqlmaster) { $command = "true"; } $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; - $ENV{'PARALLEL_JOBSLOT'} = $job->slot(); + $ENV{$opt::process_slot_var} = -1 + + ($ENV{'PARALLEL_JOBSLOT'} = $job->slot()); $ENV{'PARALLEL_TMP'} = ::tmpname("par"); $job->add_rm($ENV{'PARALLEL_TMP'}); $job->fill_templates(); diff --git a/src/parallel.pod b/src/parallel.pod index 4917b8a2..aa446812 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1929,6 +1929,13 @@ inspired by bash's parameter expansion: See also: B<--rpl> B<{}> +=item B<--process-slot-var> I (alpha testing) + +Set the environment variable I to the jobslot number-1. + + seq 10 | parallel --process-slot-var=name echo '$name' {} + + =item B<--progress> Show progress of computations. diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 3df3bf3c..0bf3362a 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -8,7 +8,6 @@ # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins - par_commandline_with_newline() { echo 'bug #51299: --retry-failed with command with newline' echo 'The format must remain the same' diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index e6fb059c..724254e1 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -8,6 +8,16 @@ # Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins +par_process_slot_var() { + echo '### bug #62310: xargs compatibility: --process-slot-var=name' + seq 0.1 0.1 0.5 | + parallel -n1 -P4 --process-slot-var=name -q bash -c 'sleep $1; echo "$name"' _ + seq 0.1 0.1 0.5 | + xargs -n1 -P4 --process-slot-var=name bash -c 'sleep $1; echo "$name"' _ + seq 0.1 0.1 0.5 | + parallel -P4 --process-slot-var=name sleep {}\; echo '$name' +} + par_retries_0() { echo '--retries 0 = inf' echo this wraps at 256 and should retry until it wraps diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 07df3bbb..ee55a5c5 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -953,10 +953,10 @@ par_sem_quote ### sem --quote should not add empty argument par_sem_quote echo par_sem_quote par_shellcompletion ### --shellcompletion -par_shellcompletion 920d653deec2b96528d7dcfe396f771f - -par_shellcompletion 920d653deec2b96528d7dcfe396f771f - -par_shellcompletion 5a53f2595ceeefbf3fb70d0cae4af71c - -par_shellcompletion 5a53f2595ceeefbf3fb70d0cae4af71c - +par_shellcompletion f4aa0b2d61cf5e6836b738bf403c1d96 - +par_shellcompletion f4aa0b2d61cf5e6836b738bf403c1d96 - +par_shellcompletion be5b6ba646ba3f082315afd8e205c954 - +par_shellcompletion be5b6ba646ba3f082315afd8e205c954 - par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks par_slow_pipe_regexp This should take a few ms, but took more than 2 hours par_slow_pipe_regexp 0 1 1 diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 5383cba7..fd22fbf8 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -569,6 +569,22 @@ par_prefix_for_L_n_N_s 244 par_prefix_for_L_n_N_s 246 par_prefix_for_L_n_N_s 246 par_prefix_for_L_n_N_s 273 +par_process_slot_var ### bug #62310: xargs compatibility: --process-slot-var=name +par_process_slot_var 0 +par_process_slot_var 1 +par_process_slot_var 2 +par_process_slot_var 3 +par_process_slot_var 0 +par_process_slot_var 0 +par_process_slot_var 1 +par_process_slot_var 2 +par_process_slot_var 3 +par_process_slot_var 0 +par_process_slot_var 0 +par_process_slot_var 1 +par_process_slot_var 2 +par_process_slot_var 3 +par_process_slot_var 0 par_progress ### Test of --progress par_progress 16 par_progress ### Test of --progress with no jobs diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 967974f1..5f90d342 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -847,7 +847,7 @@ For details: see man env_parallel export -f my_func3 parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \ -S $SERVER1 my_func3 {} ::: abc-file -ssh -l parallel lo -- exec mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -X -e GNU_Parallel_worker,eval+pack+q/H10000000/,join+q//,@ARGV BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 -rlDzR -e'ssh -l parallel' --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -- lo:./abc-file.out ./.;ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec rm -rf .TMPWORKDIR;exit $_EXIT_status; +ssh -l parallel lo -- exec mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -X -e GNU_Parallel_worker,eval+pack+q/H10000000/,join+q//,@ARGV BASE64.BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 -rlDzR -e'ssh -l parallel' --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -- lo:./abc-file.out ./.;ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec rm -rf .TMPWORKDIR;exit $_EXIT_status; parset myvar1,myvar2 echo ::: a b echo $myvar1 echo $myvar2