env_parallel.csh: If env_parallel.csh not in $PATH => Error.

This commit is contained in:
Ole Tange 2024-07-16 18:42:28 +02:00
parent 9321ac863a
commit eedb8b61b5
17 changed files with 91 additions and 28 deletions

View file

@ -284,6 +284,13 @@ New in this release:
News about GNU Parallel:
* Scientific Workflows at Scale using GNU Parallel https://www.escience-conference.org/2023/tutorials/gnu_parallel/
https://web.cvent.com/event/f318e73c-2230-432a-a044-b75625020543/websitePage:afd80266-008e-414b-9f94-2fd9b4dd1924?session=fe79a785-ec60-414c-8d2b-c29208f53d4c&shareLink=true
?https://www.jvt.me/posts/2022/04/28/shell-queue/
?https://github.com/tfmoraes/blender_gnu_parallel_render
https://v2thegreat.com/2024/06/19/lessons-learned-from-scaling-to-multi-terabyte-datasets/
https://medium.com/@nfrozi/efisiensi-maksimal-cara-paralelisasi-perintah-di-cli-linux-f4fda3afe2a0

View file

@ -30,8 +30,14 @@
set _parallel_exit_CODE=0
if ("`alias env_parallel`" == '' || ! $?PARALLEL_CSH) then
set _parallel_eNv_pAtH=`which env_parallel.csh`
if ( $status == 0 ) then
# Activate alias
alias env_parallel '(setenv PARALLEL_CSH "\!*"; source `which env_parallel.csh`)'
alias env_parallel '(setenv PARALLEL_CSH "\!*"; source '$_parallel_eNv_pAtH')'
unset _parallel_eNv_pAtH
else
echo 'env_parallel: Error: env_parallel.csh must be in $PATH ('$PATH')'
endif
else
# Get the --env variables if set
# --env _ should be ignored

View file

@ -434,6 +434,8 @@ B<--session> is supported.
=item arrays
Arrays normally work, but fail intermitently.
set myarray arrays work, too
env_parallel -k echo '$myarray[{}]' ::: 1 2 3
env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3

View file

@ -30,8 +30,15 @@
set _parallel_exit_CODE=0
if ("`alias env_parallel`" == '' || ! $?PARALLEL_CSH) then
set _parallel_eNv_pAtH=`which env_parallel.tcsh`
if ( $status == 0 ) then
# Activate alias
alias env_parallel '(setenv PARALLEL_CSH "\!*"; source `which env_parallel.csh`)'
alias env_parallel '(setenv PARALLEL_CSH "\!*"; source '$_parallel_eNv_pAtH')'
unset _parallel_eNv_pAtH
else
echo 'env_parallel: Error: env_parallel.tcsh must be in $PATH ('$PATH')'
unset _parallel_eNv_pAtH
endif
else
# Get the --env variables if set
# --env _ should be ignored

View file

@ -2561,7 +2561,7 @@ sub parse_options(@) {
# If you want GNU Parallel to be maintained in the future you
# should keep this.
# *YOU* will be harming free software by removing the notice.
# _YOU_ will be harming free software by removing the notice.
#
# Funding a free software project is hard. GNU Parallel is no
# exception. On top of that it seems the less visible a project
@ -2603,7 +2603,7 @@ sub parse_options(@) {
# list. Ideas that will cost work and which have not been tested
# are, however, unlikely to be prioritized.
#
# *YOU* will be harming free software by removing the notice. You
# _YOU_ will be harming free software by removing the notice. You
# accept to be added to a public hall of shame by removing the
# line. That includes you, George and Andreas.
#
@ -2793,7 +2793,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20240623;
$Global::version = 20240624;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -5886,8 +5886,7 @@ sub citation_notice() {
# Before changing these line, please read
# https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
# You accept to be put in a public hall of shame by
# removing these lines
# You accept to be put in a public hall of shame by removing these lines
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
"",
@ -8526,7 +8525,8 @@ sub user_requested_processes($) {
my $opt_P = shift;
my $processes;
if(defined $opt_P) {
if (-f $opt_P) {
if (-f $opt_P and not $opt_P =~ /^[-+]?\d+%?$/) {
# This is a file. Ignore files called +10% 4 -3
$Global::max_procs_file = $opt_P;
my $opt_P_file = ::slurp_or_exit($Global::max_procs_file);
if($opt_P_file !~ /\S/) {

View file

@ -132,9 +132,16 @@ install_oracle_client() {
fi
# libaio
sudo apt install libaio1t64
sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
perl -MCPAN -e 'install DBD::Oracle'
# TODO set up vagrant oracle server
git clone https://github.com/oracle/vagrant-projects.git
(
cd vagrant-projects/OracleDatabase/21.3.0-XE
export VM_ORACLE_PWD=`goodpasswd`
echo export VM_ORACLE_PWD="$VM_ORACLE_PWD" >> ~/.passwords
vagrant up
)
# TODO set up default passwd
echo sudo su - oracle -c "'/home/oracle/setPassword.sh $oracle_password'" |
vagrant ssh

View file

@ -8,6 +8,19 @@
# Each should be taking 1-3s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_keep_order_make_job_1_output_fast() {
echo '# EXAMPLE: Keep order, but make job 1 output fast'
doit() {
echo "$@" ERR >&2
echo "$@" out
sleep 0.$1
echo "$@" ERR >&2
echo "$@" out
}
export -f doit
parallel -k -u doit {= 'seq() > 1 and $opt::ungroup = 0' =} ::: 9 1 2 3
}
par_citation_no_config_dir() {
echo '### bug #64329: parallel --citation will loop forever unless the config dir exists'
t=$(mktemp -d)

View file

@ -39,13 +39,13 @@ par__sshpass_with_password() {
echo '### Crazy passwords: `date>>/tmp/trap`;(|<*&)'"'"
> /tmp/trap
(
# These fail. The important part is that /tmp/trap is empty
echo '# These fail. The important part is that /tmp/trap is empty'
stdout parallel --onall -S5/user:'`date>>/tmp/trap`;'@host echo ::: A
stdout parallel --onall -S5/user:'`date>>/tmp/trap`;(|<*&)'"'"@host echo ::: A
) | perl -pe 's/([a-f0-9]{100,} )+[a-f0-9]{1,}/HEX/g; s/\r/\n/g;' |
perl -pe 's/(ssh: Could not resolve hostname host:) .*/$1/'
echo This must stay empty:
cat /tmp/trap
grep . /tmp/trap || echo OK
}
par_--ssh_ssh_in_ssh() {

View file

@ -236,5 +236,5 @@ par_z_multiple_hosts_repeat_arg() {
export -f $(compgen -A function | grep par_)
compgen -A function | G par_ "$@" | LC_ALL=C sort |
parallel --timeout 200 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
parallel --timeout 250 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
perl -pe 's:/usr/bin:/bin:g;'

View file

@ -281,13 +281,10 @@ par__man_fish() {
set myarray arrays 'with = & " !'" '" work, too
echo $myarray[1] $myarray[2] $myarray[3] $myarray[4]
sleep 0.1
echo "# these 4 fail often. Race condition?"
env_parallel -k echo '$myarray[{}]' ::: 1 2 3 4
sleep 0.1
env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 4
sleep 0.1
env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 4
sleep 0.1
env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 4
env_parallel --argsep --- env_parallel -k echo ::: multi level --- env_parallel

View file

@ -29,7 +29,7 @@ export -f par_tmux
# echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground'
# stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null | perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"'
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 20 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 30 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
echo '### tmux-1.9'
seq 510 512 | PARALLEL_TMUX=tmux-1.9 par_tmux

View file

@ -23,7 +23,7 @@ srcdir=$(echo "$testsuitedir" | perl -pe 's=$ENV{HOME}==')
export SERVER1=parallel@lo
export SERVER2=csh@lo
export PARALLEL=-k
perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' "$testsuitedir"/../src/parallel_tutorial.pod |
perl -ne '$/="\n\n"; /^Output/../^[^O=]\S/ and next; /^ / and print;' "$testsuitedir"/../src/parallel_tutorial.pod |
egrep -v 'curl|tty|parallel_tutorial|interactive|example.(com|net)' |
egrep -v 'shellquote|works|num128|--filter-hosts|--tmux|my_id' |
perl -pe 's/username@//;s/user@//;

View file

@ -139,14 +139,17 @@ par_listproc() {
(
sql --show-processlist "$DBURL" | wc -lw
sql --show-processlist "$DBURL" | wc -lw
sql --show-processlist "$DBURL" | wc -lw
) | sort | head -n1
(
sql --proclist "$DBURL" | wc -lw
sql --proclist "$DBURL" | wc -lw
sql --proclist "$DBURL" | wc -lw
) | sort | head -n1
(
sql --listproc "$DBURL" | wc -lw
sql --listproc "$DBURL" | wc -lw
sql --listproc "$DBURL" | wc -lw
) | sort | head -n1
}

View file

@ -211,6 +211,23 @@ par_keep_order ### Bug made 3 go before 1 2
par_keep_order 1
par_keep_order 2
par_keep_order 3
par_keep_order_make_job_1_output_fast # EXAMPLE: Keep order, but make job 1 output fast
par_keep_order_make_job_1_output_fast 9 ERR
par_keep_order_make_job_1_output_fast 9 out
par_keep_order_make_job_1_output_fast 9 ERR
par_keep_order_make_job_1_output_fast 9 out
par_keep_order_make_job_1_output_fast 1 out
par_keep_order_make_job_1_output_fast 1 out
par_keep_order_make_job_1_output_fast 1 ERR
par_keep_order_make_job_1_output_fast 1 ERR
par_keep_order_make_job_1_output_fast 2 out
par_keep_order_make_job_1_output_fast 2 out
par_keep_order_make_job_1_output_fast 2 ERR
par_keep_order_make_job_1_output_fast 2 ERR
par_keep_order_make_job_1_output_fast 3 out
par_keep_order_make_job_1_output_fast 3 out
par_keep_order_make_job_1_output_fast 3 ERR
par_keep_order_make_job_1_output_fast 3 ERR
par_long_input ### Long input lines should not fail if they are not used
par_long_input a a a a
par_long_input b b b b

View file

@ -75,15 +75,17 @@ par__sshpass_with_password parallel: Warning: Using password or SSHPASS with --(
par__sshpass_with_password parallel: Warning: on the command line, making it visible to local users via `ps`.
par__sshpass_with_password OK
par__sshpass_with_password ### Crazy passwords: `date>>/tmp/trap`;(|<*&)'
par__sshpass_with_password # These fail. The important part is that /tmp/trap is empty
par__sshpass_with_password parallel: Warning: Using password or SSHPASS with --(n)onall exposes the password
par__sshpass_with_password parallel: Warning: on the command line, making it visible to local users via `ps`.
par__sshpass_with_password ssh: Could not resolve hostname host:
par__sshpass_with_password Permission denied, please try again.
par__sshpass_with_password
par__sshpass_with_password parallel: Warning: Using password or SSHPASS with --(n)onall exposes the password
par__sshpass_with_password parallel: Warning: on the command line, making it visible to local users via `ps`.
par__sshpass_with_password ssh: Could not resolve hostname host:
par__sshpass_with_password Permission denied, please try again.
par__sshpass_with_password
par__sshpass_with_password This must stay empty:
par__sshpass_with_password OK
par_csh_variable_newline ### Can csh propagate a variable containing \n
par_csh_variable_newline 1
par_csh_variable_newline 2

View file

@ -453,10 +453,11 @@ par__man_fish variables with = & " ! ' work
par__man_fish multiline
par__man_fish variables with = & " ! ' work
par__man_fish arrays with = & " ! ' work, too
par__man_fish
par__man_fish # these 4 fail often. Race condition?
par__man_fish arrays
par__man_fish with = & " ! '
par__man_fish work,
par__man_fish too
par__man_fish
par__man_fish arrays
par__man_fish with = & " ! '
par__man_fish work,

View file

@ -34,7 +34,8 @@ par_path_remote_csh StArT
par_path_remote_csh CSH Path before: /bin:/usr/bin with no parallel
par_path_remote_csh parallel: Command not found.
par_path_remote_csh ^^^^^^^^ Not found is OK
par_path_remote_csh /bin:/usr/bin:/tmp OK
par_path_remote_csh parallel: Warning: Could not figure out number of cpus on lo (). Using 1.
par_path_remote_csh OK: /bin:/usr/bin:/tmp
par_path_remote_csh Done
par_retries_1 ### Test of --retries - it should run 13 jobs in total
par_retries_1 OK