From eedb8b61b5a0e0e1a0ffdf2d85bef9e6c4b8b6f7 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 16 Jul 2024 18:42:28 +0200 Subject: [PATCH] env_parallel.csh: If env_parallel.csh not in $PATH => Error. --- doc/release_new_version | 7 +++++++ src/env_parallel.csh | 10 ++++++++-- src/env_parallel.pod | 2 ++ src/env_parallel.tcsh | 11 ++++++++-- src/parallel | 20 +++++++++---------- testsuite/REQUIREMENTS | 7 +++++++ testsuite/tests-to-run/parallel-local-1s.sh | 13 ++++++++++++ testsuite/tests-to-run/parallel-local-ssh1.sh | 4 ++-- testsuite/tests-to-run/parallel-local-ssh4.sh | 2 +- testsuite/tests-to-run/parallel-local-ssh7.sh | 5 +---- testsuite/tests-to-run/parallel-local7.sh | 2 +- testsuite/tests-to-run/parallel-tutorial.sh | 2 +- testsuite/tests-to-run/sql01.sh | 3 +++ testsuite/wanted-results/parallel-local-1s | 17 ++++++++++++++++ testsuite/wanted-results/parallel-local-ssh1 | 6 ++++-- testsuite/wanted-results/parallel-local-ssh7 | 5 +++-- testsuite/wanted-results/parallel-local-ssh8 | 3 ++- 17 files changed, 91 insertions(+), 28 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 613753cf..d8eebdbb 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -284,6 +284,13 @@ New in this release: News about GNU Parallel: +* Scientific Workflows at Scale using GNU Parallel https://www.escience-conference.org/2023/tutorials/gnu_parallel/ + +https://web.cvent.com/event/f318e73c-2230-432a-a044-b75625020543/websitePage:afd80266-008e-414b-9f94-2fd9b4dd1924?session=fe79a785-ec60-414c-8d2b-c29208f53d4c&shareLink=true + +?https://www.jvt.me/posts/2022/04/28/shell-queue/ +?https://github.com/tfmoraes/blender_gnu_parallel_render + https://v2thegreat.com/2024/06/19/lessons-learned-from-scaling-to-multi-terabyte-datasets/ https://medium.com/@nfrozi/efisiensi-maksimal-cara-paralelisasi-perintah-di-cli-linux-f4fda3afe2a0 diff --git a/src/env_parallel.csh b/src/env_parallel.csh index 3f00e3ab..ed247651 100755 --- a/src/env_parallel.csh +++ b/src/env_parallel.csh @@ -30,8 +30,14 @@ set _parallel_exit_CODE=0 if ("`alias env_parallel`" == '' || ! $?PARALLEL_CSH) then - # Activate alias - alias env_parallel '(setenv PARALLEL_CSH "\!*"; source `which env_parallel.csh`)' + set _parallel_eNv_pAtH=`which env_parallel.csh` + if ( $status == 0 ) then + # Activate alias + alias env_parallel '(setenv PARALLEL_CSH "\!*"; source '$_parallel_eNv_pAtH')' + unset _parallel_eNv_pAtH + else + echo 'env_parallel: Error: env_parallel.csh must be in $PATH ('$PATH')' + endif else # Get the --env variables if set # --env _ should be ignored diff --git a/src/env_parallel.pod b/src/env_parallel.pod index c59ba5d8..ad5675a4 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -434,6 +434,8 @@ B<--session> is supported. =item arrays +Arrays normally work, but fail intermitently. + set myarray arrays work, too env_parallel -k echo '$myarray[{}]' ::: 1 2 3 env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index dcb7a0b2..4782e91c 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -30,8 +30,15 @@ set _parallel_exit_CODE=0 if ("`alias env_parallel`" == '' || ! $?PARALLEL_CSH) then - # Activate alias - alias env_parallel '(setenv PARALLEL_CSH "\!*"; source `which env_parallel.csh`)' + set _parallel_eNv_pAtH=`which env_parallel.tcsh` + if ( $status == 0 ) then + # Activate alias + alias env_parallel '(setenv PARALLEL_CSH "\!*"; source '$_parallel_eNv_pAtH')' + unset _parallel_eNv_pAtH + else + echo 'env_parallel: Error: env_parallel.tcsh must be in $PATH ('$PATH')' + unset _parallel_eNv_pAtH + endif else # Get the --env variables if set # --env _ should be ignored diff --git a/src/parallel b/src/parallel index 26d7be02..b9d32004 100755 --- a/src/parallel +++ b/src/parallel @@ -1980,7 +1980,7 @@ sub options_completion_hash() { "gnu[Behave like GNU parallel]" => \$opt::gnu, "link|xapply[Link input sources]" => \$opt::link, "linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource, - # Before changing these lines, please read + # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt # You accept to be put in a public hall-of-shame by removing @@ -2561,7 +2561,7 @@ sub parse_options(@) { # If you want GNU Parallel to be maintained in the future you # should keep this. - # *YOU* will be harming free software by removing the notice. + # _YOU_ will be harming free software by removing the notice. # # Funding a free software project is hard. GNU Parallel is no # exception. On top of that it seems the less visible a project @@ -2603,7 +2603,7 @@ sub parse_options(@) { # list. Ideas that will cost work and which have not been tested # are, however, unlikely to be prioritized. # - # *YOU* will be harming free software by removing the notice. You + # _YOU_ will be harming free software by removing the notice. You # accept to be added to a public hall of shame by removing the # line. That includes you, George and Andreas. # @@ -2638,7 +2638,7 @@ sub parse_options(@) { # This is because _YOU_ actively make it harder to justify # spending time developing GNU Parallel by removing it. - # If you disagree, please read (especially 77-): + # If you disagree, please read (especially 77-): # https://www.fordfoundation.org/media/2976/roads-and-bridges-the-unseen-labor-behind-our-digital-infrastructure.pdf parse_halt(); @@ -2793,7 +2793,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20240623; + $Global::version = 20240624; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -5850,7 +5850,7 @@ sub usage() { " Tange, O. (2024, June 22). GNU Parallel 20240622 ('34 counts').", " Zenodo. https://doi.org/10.5281/zenodo.12518196", "", - # Before changing these lines, please read + # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt # You accept to be put in a public hall of shame by removing @@ -5883,11 +5883,10 @@ sub citation_notice() { " Tange, O. (2024, June 22). GNU Parallel 20240622 ('34 counts').", " Zenodo. https://doi.org/10.5281/zenodo.12518196", "", - # Before changing these line, please read + # Before changing these line, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt - # You accept to be put in a public hall of shame by - # removing these lines + # You accept to be put in a public hall of shame by removing these lines "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "", @@ -8526,7 +8525,8 @@ sub user_requested_processes($) { my $opt_P = shift; my $processes; if(defined $opt_P) { - if (-f $opt_P) { + if (-f $opt_P and not $opt_P =~ /^[-+]?\d+%?$/) { + # This is a file. Ignore files called +10% 4 -3 $Global::max_procs_file = $opt_P; my $opt_P_file = ::slurp_or_exit($Global::max_procs_file); if($opt_P_file !~ /\S/) { diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index 19d1e149..de1227e6 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -132,9 +132,16 @@ install_oracle_client() { fi # libaio sudo apt install libaio1t64 + sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 perl -MCPAN -e 'install DBD::Oracle' # TODO set up vagrant oracle server git clone https://github.com/oracle/vagrant-projects.git + ( + cd vagrant-projects/OracleDatabase/21.3.0-XE + export VM_ORACLE_PWD=`goodpasswd` + echo export VM_ORACLE_PWD="$VM_ORACLE_PWD" >> ~/.passwords + vagrant up + ) # TODO set up default passwd echo sudo su - oracle -c "'/home/oracle/setPassword.sh $oracle_password'" | vagrant ssh diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 358f38eb..fbf8951e 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -8,6 +8,19 @@ # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins +par_keep_order_make_job_1_output_fast() { + echo '# EXAMPLE: Keep order, but make job 1 output fast' + doit() { + echo "$@" ERR >&2 + echo "$@" out + sleep 0.$1 + echo "$@" ERR >&2 + echo "$@" out + } + export -f doit + parallel -k -u doit {= 'seq() > 1 and $opt::ungroup = 0' =} ::: 9 1 2 3 +} + par_citation_no_config_dir() { echo '### bug #64329: parallel --citation will loop forever unless the config dir exists' t=$(mktemp -d) diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index c374a174..cd809475 100755 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -39,13 +39,13 @@ par__sshpass_with_password() { echo '### Crazy passwords: `date>>/tmp/trap`;(|<*&)'"'" > /tmp/trap ( - # These fail. The important part is that /tmp/trap is empty + echo '# These fail. The important part is that /tmp/trap is empty' stdout parallel --onall -S5/user:'`date>>/tmp/trap`;'@host echo ::: A stdout parallel --onall -S5/user:'`date>>/tmp/trap`;(|<*&)'"'"@host echo ::: A ) | perl -pe 's/([a-f0-9]{100,} )+[a-f0-9]{1,}/HEX/g; s/\r/\n/g;' | perl -pe 's/(ssh: Could not resolve hostname host:) .*/$1/' echo This must stay empty: - cat /tmp/trap + grep . /tmp/trap || echo OK } par_--ssh_ssh_in_ssh() { diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 8b076402..79bc04f5 100755 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -236,5 +236,5 @@ par_z_multiple_hosts_repeat_arg() { export -f $(compgen -A function | grep par_) compgen -A function | G par_ "$@" | LC_ALL=C sort | - parallel --timeout 200 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | + parallel --timeout 250 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | perl -pe 's:/usr/bin:/bin:g;' diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 6755410e..3be7fd12 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -281,13 +281,10 @@ par__man_fish() { set myarray arrays 'with = & " !'" '" work, too echo $myarray[1] $myarray[2] $myarray[3] $myarray[4] - sleep 0.1 + echo "# these 4 fail often. Race condition?" env_parallel -k echo '$myarray[{}]' ::: 1 2 3 4 - sleep 0.1 env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 4 - sleep 0.1 env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 4 - sleep 0.1 env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 4 env_parallel --argsep --- env_parallel -k echo ::: multi level --- env_parallel diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index ff40116e..82c23381 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -29,7 +29,7 @@ export -f par_tmux # echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground' # stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null | perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 20 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 30 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### tmux-1.9' seq 510 512 | PARALLEL_TMUX=tmux-1.9 par_tmux diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 0de7de8d..3e9528fa 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -23,7 +23,7 @@ srcdir=$(echo "$testsuitedir" | perl -pe 's=$ENV{HOME}==') export SERVER1=parallel@lo export SERVER2=csh@lo export PARALLEL=-k -perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' "$testsuitedir"/../src/parallel_tutorial.pod | +perl -ne '$/="\n\n"; /^Output/../^[^O=]\S/ and next; /^ / and print;' "$testsuitedir"/../src/parallel_tutorial.pod | egrep -v 'curl|tty|parallel_tutorial|interactive|example.(com|net)' | egrep -v 'shellquote|works|num128|--filter-hosts|--tmux|my_id' | perl -pe 's/username@//;s/user@//; diff --git a/testsuite/tests-to-run/sql01.sh b/testsuite/tests-to-run/sql01.sh index 2a2ebaa7..9d4a5a1c 100755 --- a/testsuite/tests-to-run/sql01.sh +++ b/testsuite/tests-to-run/sql01.sh @@ -139,14 +139,17 @@ par_listproc() { ( sql --show-processlist "$DBURL" | wc -lw sql --show-processlist "$DBURL" | wc -lw + sql --show-processlist "$DBURL" | wc -lw ) | sort | head -n1 ( sql --proclist "$DBURL" | wc -lw sql --proclist "$DBURL" | wc -lw + sql --proclist "$DBURL" | wc -lw ) | sort | head -n1 ( sql --listproc "$DBURL" | wc -lw sql --listproc "$DBURL" | wc -lw + sql --listproc "$DBURL" | wc -lw ) | sort | head -n1 } diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 4c890e1e..32291dc8 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -211,6 +211,23 @@ par_keep_order ### Bug made 3 go before 1 2 par_keep_order 1 par_keep_order 2 par_keep_order 3 +par_keep_order_make_job_1_output_fast # EXAMPLE: Keep order, but make job 1 output fast +par_keep_order_make_job_1_output_fast 9 ERR +par_keep_order_make_job_1_output_fast 9 out +par_keep_order_make_job_1_output_fast 9 ERR +par_keep_order_make_job_1_output_fast 9 out +par_keep_order_make_job_1_output_fast 1 out +par_keep_order_make_job_1_output_fast 1 out +par_keep_order_make_job_1_output_fast 1 ERR +par_keep_order_make_job_1_output_fast 1 ERR +par_keep_order_make_job_1_output_fast 2 out +par_keep_order_make_job_1_output_fast 2 out +par_keep_order_make_job_1_output_fast 2 ERR +par_keep_order_make_job_1_output_fast 2 ERR +par_keep_order_make_job_1_output_fast 3 out +par_keep_order_make_job_1_output_fast 3 out +par_keep_order_make_job_1_output_fast 3 ERR +par_keep_order_make_job_1_output_fast 3 ERR par_long_input ### Long input lines should not fail if they are not used par_long_input a a a a par_long_input b b b b diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index d95151f8..74dfc13a 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -75,15 +75,17 @@ par__sshpass_with_password parallel: Warning: Using password or SSHPASS with --( par__sshpass_with_password parallel: Warning: on the command line, making it visible to local users via `ps`. par__sshpass_with_password OK par__sshpass_with_password ### Crazy passwords: `date>>/tmp/trap`;(|<*&)' +par__sshpass_with_password # These fail. The important part is that /tmp/trap is empty par__sshpass_with_password parallel: Warning: Using password or SSHPASS with --(n)onall exposes the password par__sshpass_with_password parallel: Warning: on the command line, making it visible to local users via `ps`. -par__sshpass_with_password ssh: Could not resolve hostname host: +par__sshpass_with_password Permission denied, please try again. par__sshpass_with_password par__sshpass_with_password parallel: Warning: Using password or SSHPASS with --(n)onall exposes the password par__sshpass_with_password parallel: Warning: on the command line, making it visible to local users via `ps`. -par__sshpass_with_password ssh: Could not resolve hostname host: +par__sshpass_with_password Permission denied, please try again. par__sshpass_with_password par__sshpass_with_password This must stay empty: +par__sshpass_with_password OK par_csh_variable_newline ### Can csh propagate a variable containing \n par_csh_variable_newline 1 par_csh_variable_newline 2 diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 8aece247..6e6787cf 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -453,10 +453,11 @@ par__man_fish variables with = & " ! ' work par__man_fish multiline par__man_fish variables with = & " ! ' work par__man_fish arrays with = & " ! ' work, too -par__man_fish +par__man_fish # these 4 fail often. Race condition? +par__man_fish arrays +par__man_fish with = & " ! ' par__man_fish work, par__man_fish too -par__man_fish par__man_fish arrays par__man_fish with = & " ! ' par__man_fish work, diff --git a/testsuite/wanted-results/parallel-local-ssh8 b/testsuite/wanted-results/parallel-local-ssh8 index d2de9406..c5bca8fc 100644 --- a/testsuite/wanted-results/parallel-local-ssh8 +++ b/testsuite/wanted-results/parallel-local-ssh8 @@ -34,7 +34,8 @@ par_path_remote_csh StArT par_path_remote_csh CSH Path before: /bin:/usr/bin with no parallel par_path_remote_csh parallel: Command not found. par_path_remote_csh ^^^^^^^^ Not found is OK -par_path_remote_csh /bin:/usr/bin:/tmp OK +par_path_remote_csh parallel: Warning: Could not figure out number of cpus on lo (). Using 1. +par_path_remote_csh OK: /bin:/usr/bin:/tmp par_path_remote_csh Done par_retries_1 ### Test of --retries - it should run 13 jobs in total par_retries_1 OK