env_parallel.csh: If env_parallel.csh not in $PATH => Error.

doc/release_new_version

@@ -284,6 +284,13 @@ New in this release:
 
 News about GNU Parallel:
 
+* Scientific Workflows at Scale using GNU Parallel https://www.escience-conference.org/2023/tutorials/gnu_parallel/
+
+https://web.cvent.com/event/f318e73c-2230-432a-a044-b75625020543/websitePage:afd80266-008e-414b-9f94-2fd9b4dd1924?session=fe79a785-ec60-414c-8d2b-c29208f53d4c&shareLink=true
+
+?https://www.jvt.me/posts/2022/04/28/shell-queue/
+?https://github.com/tfmoraes/blender_gnu_parallel_render
+
 https://v2thegreat.com/2024/06/19/lessons-learned-from-scaling-to-multi-terabyte-datasets/
 
 https://medium.com/@nfrozi/efisiensi-maksimal-cara-paralelisasi-perintah-di-cli-linux-f4fda3afe2a0

src/env_parallel.csh

@@ -30,8 +30,14 @@
 
 set _parallel_exit_CODE=0
 if ("`alias env_parallel`" == '' || ! $?PARALLEL_CSH) then
-  # Activate alias
-  alias env_parallel '(setenv PARALLEL_CSH "\!*"; source `which env_parallel.csh`)'
+  set _parallel_eNv_pAtH=`which env_parallel.csh`
+  if ( $status == 0 ) then
+    # Activate alias
+    alias env_parallel '(setenv PARALLEL_CSH "\!*"; source '$_parallel_eNv_pAtH')'
+    unset _parallel_eNv_pAtH
+  else
+    echo 'env_parallel: Error: env_parallel.csh must be in $PATH ('$PATH')'
+  endif
 else
   # Get the --env variables if set
   # --env _ should be ignored

src/env_parallel.pod

@@ -434,6 +434,8 @@ B<--session> is supported.
 
 =item arrays
 
+Arrays normally work, but fail intermitently.
+
   set myarray arrays work, too
   env_parallel -k echo '$myarray[{}]' ::: 1 2 3
   env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3

src/env_parallel.tcsh

@@ -30,8 +30,15 @@
 
 set _parallel_exit_CODE=0
 if ("`alias env_parallel`" == '' || ! $?PARALLEL_CSH) then
-  # Activate alias
-  alias env_parallel '(setenv PARALLEL_CSH "\!*"; source `which env_parallel.csh`)'
+  set _parallel_eNv_pAtH=`which env_parallel.tcsh`
+  if ( $status == 0 ) then
+    # Activate alias
+    alias env_parallel '(setenv PARALLEL_CSH "\!*"; source '$_parallel_eNv_pAtH')'
+    unset _parallel_eNv_pAtH
+  else
+    echo 'env_parallel: Error: env_parallel.tcsh must be in $PATH ('$PATH')'
+    unset _parallel_eNv_pAtH
+  endif
 else
   # Get the --env variables if set
   # --env _ should be ignored

src/parallel

@@ -1980,7 +1980,7 @@ sub options_completion_hash() {
 	 "gnu[Behave like GNU parallel]" => \$opt::gnu,
 	 "link|xapply[Link input sources]" => \$opt::link,
 	 "linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource,
-	 # Before changing these lines, please read
+	 # Before changing these lines,  please read
 	 # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice
 	 # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
 	 # You accept to be put in a public hall-of-shame by removing
@@ -2561,7 +2561,7 @@ sub parse_options(@) {
 
     # If you want GNU Parallel to be maintained in the future you
     # should keep this.
-    # *YOU* will be harming free software by removing the notice.
+    # _YOU_ will be harming free software by removing the notice.
     #
     # Funding a free software project is hard. GNU Parallel is no
     # exception. On top of that it seems the less visible a project
@@ -2603,7 +2603,7 @@ sub parse_options(@) {
     # list. Ideas that will cost work and which have not been tested
     # are, however, unlikely to be prioritized.
     #
-    # *YOU* will be harming free software by removing the notice.  You
+    # _YOU_ will be harming free software by removing the notice.  You
     # accept to be added to a public hall of shame by removing the
     # line.  That includes you, George and Andreas.
     #
@@ -2638,7 +2638,7 @@ sub parse_options(@) {
     # This is because _YOU_ actively make it harder to justify
     # spending time developing GNU Parallel by removing it.
 
-    # If you disagree, please read (especially 77-):
+    # If you disagree,  please read (especially 77-):
     # https://www.fordfoundation.org/media/2976/roads-and-bridges-the-unseen-labor-behind-our-digital-infrastructure.pdf
     parse_halt();
 
@@ -2793,7 +2793,7 @@ sub check_invalid_option_combinations() {
 
 sub init_globals() {
     # Defaults:
-    $Global::version = 20240623;
+    $Global::version = 20240624;
     $Global::progname = 'parallel';
     $::name = "GNU Parallel";
     $Global::infinity = 2**31;
@@ -5850,7 +5850,7 @@ sub usage() {
 	 "  Tange, O. (2024, June 22). GNU Parallel 20240622 ('34 counts').",
 	 "  Zenodo. https://doi.org/10.5281/zenodo.12518196",
 	 "",
-	 # Before changing these lines, please read
+	 # Before changing these lines,  please read
 	 # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice
 	 # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
 	 # You accept to be put in a  public hall of shame  by removing
@@ -5883,11 +5883,10 @@ sub citation_notice() {
 	     "  Tange, O. (2024, June 22). GNU Parallel 20240622 ('34 counts').",
 	     "  Zenodo. https://doi.org/10.5281/zenodo.12518196",
 	     "",
-	     # Before changing these line,  please read
+	     # Before changing these line,   please read
 	     # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and
 	     # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
-	     # You accept to be put in a public hall of shame by
-	     # removing these lines
+	     # You accept to be put in a public hall of shame by removing these lines
 	     "This helps funding further development; AND IT WON'T COST YOU A CENT.",
 	     "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
 	     "",
@@ -8526,7 +8525,8 @@ sub user_requested_processes($) {
     my $opt_P = shift;
     my $processes;
     if(defined $opt_P) {
-	if (-f $opt_P) {
+	if (-f $opt_P and not $opt_P =~ /^[-+]?\d+%?$/) {
+	    # This is a file. Ignore files called +10% 4 -3
 	    $Global::max_procs_file = $opt_P;
 	    my $opt_P_file = ::slurp_or_exit($Global::max_procs_file);
 	    if($opt_P_file !~ /\S/) {

testsuite/REQUIREMENTS

@@ -132,9 +132,16 @@ install_oracle_client() {
     fi
     # libaio
     sudo apt install libaio1t64
+    sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
     perl -MCPAN -e 'install DBD::Oracle'
     # TODO set up vagrant oracle server
     git clone https://github.com/oracle/vagrant-projects.git
+    (
+	cd vagrant-projects/OracleDatabase/21.3.0-XE
+	export VM_ORACLE_PWD=`goodpasswd`
+	echo export VM_ORACLE_PWD="$VM_ORACLE_PWD" >> ~/.passwords
+	vagrant up
+    )
     # TODO set up default passwd
     echo sudo su - oracle -c "'/home/oracle/setPassword.sh $oracle_password'" |
 	vagrant ssh

testsuite/tests-to-run/parallel-local-1s.sh

@@ -8,6 +8,19 @@
 # Each should be taking 1-3s and be possible to run in parallel
 # I.e.: No race conditions, no logins
 
+par_keep_order_make_job_1_output_fast() {
+    echo '# EXAMPLE: Keep order, but make job 1 output fast'
+    doit() {
+      echo "$@" ERR >&2
+      echo "$@" out
+      sleep 0.$1
+      echo "$@" ERR >&2
+      echo "$@" out
+    }
+    export -f doit
+    parallel -k -u doit {= 'seq() > 1 and $opt::ungroup = 0' =} ::: 9 1 2 3
+}
+
 par_citation_no_config_dir() {
     echo '### bug #64329: parallel --citation will loop forever unless the config dir exists'
     t=$(mktemp -d)

testsuite/tests-to-run/parallel-local-ssh1.sh

@@ -39,13 +39,13 @@ par__sshpass_with_password() {
     echo '### Crazy passwords: `date>>/tmp/trap`;(|<*&)'"'"
     > /tmp/trap
     (
-	# These fail. The important part is that /tmp/trap is empty
+	echo '# These fail. The important part is that /tmp/trap is empty'
 	stdout parallel --onall -S5/user:'`date>>/tmp/trap`;'@host echo ::: A
 	stdout parallel --onall -S5/user:'`date>>/tmp/trap`;(|<*&)'"'"@host echo ::: A
     ) | perl -pe 's/([a-f0-9]{100,} )+[a-f0-9]{1,}/HEX/g; s/\r/\n/g;' |
 	perl -pe 's/(ssh: Could not resolve hostname host:) .*/$1/'
     echo This must stay empty:
-    cat /tmp/trap
+    grep . /tmp/trap || echo OK
 }
 
 par_--ssh_ssh_in_ssh() {

testsuite/tests-to-run/parallel-local-ssh4.sh

@@ -236,5 +236,5 @@ par_z_multiple_hosts_repeat_arg() {
 
 export -f $(compgen -A function | grep par_)
 compgen -A function | G par_ "$@" | LC_ALL=C sort |
-    parallel --timeout 200 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
+    parallel --timeout 250 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
     perl -pe 's:/usr/bin:/bin:g;'

testsuite/tests-to-run/parallel-local-ssh7.sh

@@ -281,13 +281,10 @@ par__man_fish() {
 
     set myarray arrays 'with  = & " !'" '" work, too
     echo $myarray[1] $myarray[2] $myarray[3] $myarray[4]
-    sleep 0.1
+    echo "# these 4 fail often. Race condition?"
     env_parallel -k echo '$myarray[{}]' ::: 1 2 3 4
-    sleep 0.1
     env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 4
-    sleep 0.1
     env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 4
-    sleep 0.1
     env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 4
 
     env_parallel --argsep --- env_parallel -k echo ::: multi level --- env_parallel

testsuite/tests-to-run/parallel-local7.sh

@@ -29,7 +29,7 @@ export -f par_tmux
 # echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground'
 # stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null |  perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"'
 
-cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 20 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
+cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 30 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
 
 echo '### tmux-1.9'
   seq 510 512 | PARALLEL_TMUX=tmux-1.9 par_tmux

testsuite/tests-to-run/parallel-tutorial.sh

@@ -23,7 +23,7 @@ srcdir=$(echo "$testsuitedir" | perl -pe 's=$ENV{HOME}==')
 export SERVER1=parallel@lo
 export SERVER2=csh@lo
 export PARALLEL=-k
-perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' "$testsuitedir"/../src/parallel_tutorial.pod |
+perl -ne '$/="\n\n"; /^Output/../^[^O=]\S/ and next; /^  / and print;' "$testsuitedir"/../src/parallel_tutorial.pod |
     egrep -v 'curl|tty|parallel_tutorial|interactive|example.(com|net)' |
     egrep -v 'shellquote|works|num128|--filter-hosts|--tmux|my_id' |
     perl -pe 's/username@//;s/user@//;

testsuite/tests-to-run/sql01.sh

@@ -139,14 +139,17 @@ par_listproc() {
     (
 	sql --show-processlist "$DBURL" | wc -lw
 	sql --show-processlist "$DBURL" | wc -lw
+	sql --show-processlist "$DBURL" | wc -lw
     ) | sort | head -n1
     (
 	sql --proclist "$DBURL" | wc -lw
 	sql --proclist "$DBURL" | wc -lw
+	sql --proclist "$DBURL" | wc -lw
     ) | sort | head -n1
     (
 	sql --listproc "$DBURL" | wc -lw
 	sql --listproc "$DBURL" | wc -lw
+	sql --listproc "$DBURL" | wc -lw
     ) | sort | head -n1
 }
 

testsuite/wanted-results/parallel-local-1s

@@ -211,6 +211,23 @@ par_keep_order	### Bug made 3 go before 1 2
 par_keep_order	1
 par_keep_order	2
 par_keep_order	3
+par_keep_order_make_job_1_output_fast	# EXAMPLE: Keep order, but make job 1 output fast
+par_keep_order_make_job_1_output_fast	9 ERR
+par_keep_order_make_job_1_output_fast	9 out
+par_keep_order_make_job_1_output_fast	9 ERR
+par_keep_order_make_job_1_output_fast	9 out
+par_keep_order_make_job_1_output_fast	1 out
+par_keep_order_make_job_1_output_fast	1 out
+par_keep_order_make_job_1_output_fast	1 ERR
+par_keep_order_make_job_1_output_fast	1 ERR
+par_keep_order_make_job_1_output_fast	2 out
+par_keep_order_make_job_1_output_fast	2 out
+par_keep_order_make_job_1_output_fast	2 ERR
+par_keep_order_make_job_1_output_fast	2 ERR
+par_keep_order_make_job_1_output_fast	3 out
+par_keep_order_make_job_1_output_fast	3 out
+par_keep_order_make_job_1_output_fast	3 ERR
+par_keep_order_make_job_1_output_fast	3 ERR
 par_long_input	### Long input lines should not fail if they are not used
 par_long_input	a a a a
 par_long_input	b b b b

testsuite/wanted-results/parallel-local-ssh1

@@ -75,15 +75,17 @@ par__sshpass_with_password	parallel: Warning: Using password or SSHPASS with --(
 par__sshpass_with_password	parallel: Warning: on the command line, making it visible to local users via `ps`.
 par__sshpass_with_password	OK
 par__sshpass_with_password	### Crazy passwords: `date>>/tmp/trap`;(|<*&)'
+par__sshpass_with_password	# These fail. The important part is that /tmp/trap is empty
 par__sshpass_with_password	parallel: Warning: Using password or SSHPASS with --(n)onall exposes the password
 par__sshpass_with_password	parallel: Warning: on the command line, making it visible to local users via `ps`.
-par__sshpass_with_password	ssh: Could not resolve hostname host:
+par__sshpass_with_password	Permission denied, please try again.
 par__sshpass_with_password	
 par__sshpass_with_password	parallel: Warning: Using password or SSHPASS with --(n)onall exposes the password
 par__sshpass_with_password	parallel: Warning: on the command line, making it visible to local users via `ps`.
-par__sshpass_with_password	ssh: Could not resolve hostname host:
+par__sshpass_with_password	Permission denied, please try again.
 par__sshpass_with_password	
 par__sshpass_with_password	This must stay empty:
+par__sshpass_with_password	OK
 par_csh_variable_newline	### Can csh propagate a variable containing \n
 par_csh_variable_newline	1
 par_csh_variable_newline	2

testsuite/wanted-results/parallel-local-ssh7

@@ -453,10 +453,11 @@ par__man_fish	    variables with  = & " ! ' work
 par__man_fish	multiline
 par__man_fish	    variables with  = & " ! ' work
 par__man_fish	arrays with  = & " ! ' work, too
-par__man_fish	
+par__man_fish	# these 4 fail often. Race condition?
+par__man_fish	arrays
+par__man_fish	with  = & " ! '
 par__man_fish	work,
 par__man_fish	too
-par__man_fish	
 par__man_fish	arrays
 par__man_fish	with  = & " ! '
 par__man_fish	work,

testsuite/wanted-results/parallel-local-ssh8

@@ -34,7 +34,8 @@ par_path_remote_csh	StArT
 par_path_remote_csh	CSH Path before: /bin:/usr/bin with no parallel
 par_path_remote_csh	parallel: Command not found.
 par_path_remote_csh	^^^^^^^^ Not found is OK
-par_path_remote_csh	/bin:/usr/bin:/tmp OK
+par_path_remote_csh	parallel: Warning: Could not figure out number of cpus on lo (). Using 1.
+par_path_remote_csh	OK: /bin:/usr/bin:/tmp
 par_path_remote_csh	Done
 par_retries_1	### Test of --retries - it should run 13 jobs in total
 par_retries_1	OK