From 088b43b5b090223db6af755e8da776da833c8169 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 27 Jun 2019 15:49:49 +0200 Subject: [PATCH] parallel: $PARALLEL_SSHLOGIN can be used in command line. --- src/parallel | 3 +- src/parallel.pod | 32 +++++++--- testsuite/tests-to-run/parallel-local-10s.sh | 39 +++++++------ testsuite/tests-to-run/parallel-local-30s.sh | 32 +++------- testsuite/tests-to-run/parallel-local-ssh9.sh | 8 +++ testsuite/tests-to-run/parallel-local164.sh | 5 -- testsuite/wanted-results/parallel-local-10s | 58 +++++++++++++++---- testsuite/wanted-results/parallel-local-30s | 55 +++--------------- testsuite/wanted-results/parallel-local-ssh1 | 2 +- testsuite/wanted-results/parallel-local-ssh9 | 11 ++++ testsuite/wanted-results/parallel-local164 | 4 -- 11 files changed, 131 insertions(+), 118 deletions(-) diff --git a/src/parallel b/src/parallel index d1ce2a4d..9732bfe4 100755 --- a/src/parallel +++ b/src/parallel @@ -8779,7 +8779,7 @@ sub sshlogin_wrap($) { } # Duplicate vars as BASH functions to include post-shellshock functions (v1+v2) # So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%% - push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", + push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", "PARALLEL_SSHLOGIN", map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars); # Keep only defined variables return grep { defined($ENV{$_}) } @vars; @@ -8838,6 +8838,7 @@ sub sshlogin_wrap($) { my $serverlogin = $sshlogin->serverlogin(); my $quoted_remote_command; $ENV{'PARALLEL_SEQ'} = $self->seq(); + $ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string(); $ENV{'PARALLEL_PID'} = $$; if($serverlogin eq ":") { if($opt::workdir) { diff --git a/src/parallel.pod b/src/parallel.pod index 023a2ada..086f33a9 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1219,9 +1219,9 @@ Similar to B<--memfree>. =back -=item B<--line-buffer> +=item B<--line-buffer> (alpha testing) -=item B<--lb> +=item B<--lb> (alpha testing) Buffer output on line basis. B<--group> will keep the output together for a whole job. B<--ungroup> allows output to mixup with half a line @@ -1240,8 +1240,9 @@ B<--files>. This will make it as slow as B<--group> and will limit output to the available disk space. With B<--keep-order> B<--line-buffer> will output lines from the first -job while it is running, then lines from the second job while that is -running. It will buffer full lines, but jobs will not mix. Compare: +job continuously while it is running, then lines from the second job +while that is running. It will buffer full lines, but jobs will not +mix. Compare: parallel -j0 'echo {};sleep {};echo {}' ::: 1 3 2 4 parallel -j0 --lb 'echo {};sleep {};echo {}' ::: 1 3 2 4 @@ -1607,9 +1608,9 @@ B<,,>: See also: B<--rpl> B<{= perl expression =}> -=item B<--profile> I +=item B<--profile> I (alpha testing) -=item B<-J> I +=item B<-J> I (alpha testing) Use profile I for options. This is useful if you want to have multiple profiles. You could have one profile for running jobs in @@ -2494,7 +2495,7 @@ the lines will be prepended with the sshlogin instead. B<--tag> is ignored when using B<-u>. -=item B<--tagstring> I +=item B<--tagstring> I (alpha testing) Tag lines with a string. Each output line will be prepended with I and TAB (\t). I can contain replacement strings such as @@ -4740,6 +4741,19 @@ B<--ssh>. It can also be set on a per server basis (see B<--sshlogin>). +=item $PARALLEL_SSHLOGIN (alpha testing) + +The environment variable $PARALLEL_SSHLOGIN is set by GNU B +and is visible to the jobs started from GNU B. The value is +the sshlogin line with number of cores removed. E.g. + + 4//usr/bin/specialssh user@host + +becomes: + + /usr/bin/specialssh user@host + + =item $PARALLEL_SEQ $PARALLEL_SEQ will be set to the sequence number of the job @@ -4808,6 +4822,10 @@ If B<--profile> set, GNU B will read the profile from that file rather than the global or user configuration files. You can have multiple B<--profiles>. +Profiles are searched for in B<~/.parallel>. If the name starts with +B it is seen as an absolute path. If the name starts with B<./> it +is seen as a relative path from current dir. + Example: Profile for running a command on every sshlogin in ~/.ssh/sshlogins and prepend the output with the sshlogin: diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index e7cbe54a..2b5cd3ba 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -331,23 +331,6 @@ par__pipepart_tee() { rm $tmp } -par__memleak() { - echo "### Test memory consumption stays (almost) the same for 30 and 300 jobs" - echo "should give 1 == true" - - mem30=$( nice stdout time -f %M parallel -j2 true :::: <(perl -e '$a="x"x60000;for(1..30){print $a,"\n"}') ); - mem300=$( nice stdout time -f %M parallel -j2 true :::: <(perl -e '$a="x"x60000;for(1..300){print $a,"\n"}') ); - echo "Memory use should not depend very much on the total number of jobs run\n"; - echo "Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 110% "; - echo $(($mem300*100 < $mem30 * 110)) -} - -par_slow_total_jobs() { - echo 'bug #51006: Slow total_jobs() eats job' - (echo a; sleep 15; echo b; sleep 15; seq 2) | - parallel -k echo '{=total_jobs()=}' 2> >(perl -pe 's/\d/X/g') -} - par_interactive() { echo '### Test -p --interactive' cat >/tmp/parallel-script-for-expect <<_EOF @@ -507,6 +490,28 @@ par_long_line_remote() { parallel -j1 -S lo -N 10000 echo {} |wc } +par_shellquote() { + echo '### Test --shellquote in all shells' + doit() { + # Run --shellquote for ascii 1..255 in a shell + shell="$1" + "$shell" -c perl\ -e\ \'print\ pack\(\"c\*\",1..255\)\'\ \|\ parallel\ -0\ --shellquote + } + export -f doit + parallel --tag -q -k doit {} ::: ash bash csh dash fish fizsh ksh ksh93 lksh mksh posh rzsh sash sh static-sh tcsh yash zsh csh tcsh +} + +par_tmp_full() { + # Assume /tmp/shm is easy to fill up + export SHM=/tmp/shm/parallel + mkdir -p $SHM + sudo umount -l $SHM 2>/dev/null + sudo mount -t tmpfs -o size=10% none $SHM + + echo "### Test --tmpdir running full. bug #40733 was caused by this" + stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 787eb630..1cf99138 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -22,17 +22,6 @@ par_race_condition1() { rm /tmp/parallel_race_cond } -par_tmp_full() { - # Assume /tmp/shm is easy to fill up - export SHM=/tmp/shm/parallel - mkdir -p $SHM - sudo umount -l $SHM 2>/dev/null - sudo mount -t tmpfs -o size=10% none $SHM - - echo "### Test --tmpdir running full. bug #40733 was caused by this" - stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy -} - par_memory_leak() { a_run() { seq $1 |time -v parallel true 2>&1 | @@ -41,7 +30,7 @@ par_memory_leak() { } export -f a_run echo "### Test for memory leaks" - echo "Of 100 runs of 1 job at least one should be bigger than a 3000 job run" + echo "Of 100 runs of 1 job none should be bigger than a 3000 job run" small_max=$(seq 100 | parallel a_run 1 | jq -s max) big=$(a_run 3000) if [ $small_max -lt $big ] ; then @@ -51,6 +40,12 @@ par_memory_leak() { fi } +par_slow_total_jobs() { + echo 'bug #51006: Slow total_jobs() eats job' + (echo a; sleep 15; echo b; sleep 15; seq 2) | + parallel -k echo '{=total_jobs()=}' 2> >(perl -pe 's/\d/X/g') +} + linebuffer_matters() { echo "### (--linebuffer) --compress $TAG should give different output" nolbfile=$(mktemp) @@ -131,17 +126,6 @@ par_memfree() { grep -v TERM | grep -v ps/display.c } -par_shellquote() { - echo '### Test --shellquote in all shells' - doit() { - # Run --shellquote for ascii 1..255 in a shell - shell="$1" - "$shell" -c perl\ -e\ \'print\ pack\(\"c\*\",1..255\)\'\ \|\ parallel\ -0\ --shellquote - } - export -f doit - parallel --tag -q -k doit {} ::: ash bash csh dash fish fizsh ksh ksh93 lksh mksh posh rzsh sash sh static-sh tcsh yash zsh csh tcsh -} - par_test_detected_shell() { echo '### bug #42913: Dont use $SHELL but the shell currently running' @@ -349,4 +333,4 @@ par_keeporder_roundrobin() { export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | - parallel --delay 0.3 -j0 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' + parallel --delay 0.3 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-ssh9.sh b/testsuite/tests-to-run/parallel-local-ssh9.sh index d8a7eb6e..dbd78f54 100644 --- a/testsuite/tests-to-run/parallel-local-ssh9.sh +++ b/testsuite/tests-to-run/parallel-local-ssh9.sh @@ -194,6 +194,14 @@ par_no_route_to_host() { ) | perl -pe 's/(\d+\.\d+\.\d+\.\d+)/i.p.n.r/' | puniq } +par_PARALLEL_SSHLOGIN() { + echo '### bug #56554: Introduce $PARALLEL_SSHLOGIN' + (echo lo; echo zsh@lo; echo /usr/bin/ssh csh@lo; echo 1/sh@lo; + echo 1//usr/bin/ssh tcsh@lo) | + parallel --tag --nonall -S - 'whoami;echo $PARALLEL_SSHLOGIN' | + sort +} + export -f $(compgen -A function | grep par_) #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' #compgen -A function | grep par_ | sort | diff --git a/testsuite/tests-to-run/parallel-local164.sh b/testsuite/tests-to-run/parallel-local164.sh index 7c713c64..1962bc15 100644 --- a/testsuite/tests-to-run/parallel-local164.sh +++ b/testsuite/tests-to-run/parallel-local164.sh @@ -2,11 +2,6 @@ # -L1 will join lines ending in ' ' cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r -echo '### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name' - echo "echo echo from ./local_test_profile" > local_test_profile; - parallel --profile local_test_profile echo ::: 1; - rm local_test_profile - echo "### Test --delay" seq 9 | /usr/bin/time -f %e parallel -j3 --delay 0.57 true {} 2>&1 | perl -ne '$_ > 3.3 and print "More than 3.3 secs: OK\n"' diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 887502e6..549d7a5d 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -16,11 +16,6 @@ par_END include this par_END include this par_END include this par_END include this -par__memleak ### Test memory consumption stays (almost) the same for 30 and 300 jobs -par__memleak should give 1 == true -par__memleak Memory use should not depend very much on the total number of jobs run\n -par__memleak Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 110% -par__memleak 1 par__pipe_tee bug #45479: --pipe/--pipepart --tee par__pipe_tee --pipe --tee par__pipe_tee 314572800 @@ -705,12 +700,47 @@ par_semaphore job3a 4 par_semaphore job2b 5 par_semaphore job3b 6 par_semaphore done -par_slow_total_jobs bug #51006: Slow total_jobs() eats job -par_slow_total_jobs a -par_slow_total_jobs b -par_slow_total_jobs 1 -par_slow_total_jobs 2 -par_slow_total_jobs parallel: Warning: Reading X arguments took longer than XX seconds. +par_shellquote ### Test --shellquote in all shells +par_shellquote ash ' +par_shellquote ash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote bash ' +par_shellquote bash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote csh \\\\\\\\ "\ +par_shellquote csh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_shellquote dash ' +par_shellquote dash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote fish ' +par_shellquote fish  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote fizsh ' +par_shellquote fizsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote ksh ' +par_shellquote ksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote ksh93 ' +par_shellquote ksh93  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote lksh ' +par_shellquote lksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote mksh ' +par_shellquote mksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote posh ' +par_shellquote posh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote rzsh ' +par_shellquote rzsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote sash ' +par_shellquote sash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote sh ' +par_shellquote sh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote static-sh ' +par_shellquote static-sh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote tcsh \\\\\\\\ "\ +par_shellquote tcsh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_shellquote yash ' +par_shellquote yash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote zsh ' +par_shellquote zsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' +par_shellquote csh \\\\\\\\ "\ +par_shellquote csh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_shellquote tcsh \\\\\\\\ "\ +par_shellquote tcsh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_sockets_cores_threads ### Test --number-of-sockets/cores/threads par_sockets_cores_threads 1 par_sockets_cores_threads 2 @@ -720,6 +750,12 @@ par_sockets_cores_threads ### Test --use-sockets-instead-of-threads par_sockets_cores_threads threads done par_sockets_cores_threads sockets done par_sockets_cores_threads Threads should complete first on machines with less than 8 sockets +par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this +par_tmp_full parallel: Error: Output is incomplete. +par_tmp_full parallel: Error: Cannot append to buffer file in /tmp/shm/parallel. +par_tmp_full parallel: Error: Is the disk full? +par_tmp_full parallel: Error: Change $TMPDIR with --tmpdir or use --compress. +par_tmp_full Warning: unable to close filehandle properly: No space left on device during global destruction. par_tmux_fg bug #50107: --tmux --fg should also write how to access it par_tmux_fg See output with: tmux -S tmp attach par_tmux_fg open terminal failed: not a terminal diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index ecacf90e..b694058d 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -113,7 +113,7 @@ par_memfree Free mem: 1k par_memfree parallel: Warning: This job was killed because it timed out: par_memfree parallel: Warning: parallel --memfree 1t echo Free mem: ::: 1t par_memory_leak ### Test for memory leaks -par_memory_leak Of 100 runs of 1 job at least one should be bigger than a 3000 job run +par_memory_leak Of 100 runs of 1 job none should be bigger than a 3000 job run par_memory_leak Good: No memleak detected. par_no_newline_compress bug #41613: --compress --line-buffer - no newline par_no_newline_compress tagstring=--tagstring {#} compress=--compress @@ -209,47 +209,6 @@ par_race_condition1 9 par_race_condition1 10 par_round_robin_blocks bug #49664: --round-robin does not complete par_round_robin_blocks 8 -par_shellquote ### Test --shellquote in all shells -par_shellquote ash ' -par_shellquote ash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote bash ' -par_shellquote bash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote csh \\\\\\\\ "\ -par_shellquote csh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ -par_shellquote dash ' -par_shellquote dash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote fish ' -par_shellquote fish  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote fizsh ' -par_shellquote fizsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote ksh ' -par_shellquote ksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote ksh93 ' -par_shellquote ksh93  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote lksh ' -par_shellquote lksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote mksh ' -par_shellquote mksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote posh ' -par_shellquote posh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote rzsh ' -par_shellquote rzsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote sash ' -par_shellquote sash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote sh ' -par_shellquote sh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote static-sh ' -par_shellquote static-sh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote tcsh \\\\\\\\ "\ -par_shellquote tcsh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ -par_shellquote yash ' -par_shellquote yash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote zsh ' -par_shellquote zsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote csh \\\\\\\\ "\ -par_shellquote csh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ -par_shellquote tcsh \\\\\\\\ "\ -par_shellquote tcsh "\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_sighup ### Test SIGHUP par_sighup 1 par_sighup 10 @@ -268,6 +227,12 @@ par_sighup parallel: sleep 15; echo 6 par_sighup parallel: sleep 15; echo 7 par_sighup parallel: sleep 15; echo 8 par_sighup parallel: sleep 15; echo 9 +par_slow_total_jobs bug #51006: Slow total_jobs() eats job +par_slow_total_jobs a +par_slow_total_jobs b +par_slow_total_jobs 1 +par_slow_total_jobs 2 +par_slow_total_jobs parallel: Warning: Reading X arguments took longer than XX seconds. par_test_detected_shell ### bug #42913: Dont use $SHELL but the shell currently running par_test_detected_shell test_unknown_shell ash Global::shell /bin/bash par_test_detected_shell test_unknown_shell bash Global::shell /bin/bash @@ -329,9 +294,3 @@ par_test_detected_shell test_known_shell_pipe static-sh Global::shell /bin/stati par_test_detected_shell test_known_shell_pipe tcsh Global::shell /usr/bin/tcsh par_test_detected_shell test_known_shell_pipe yash Global::shell /usr/bin/yash par_test_detected_shell test_known_shell_pipe zsh Global::shell /usr/bin/zsh -par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this -par_tmp_full parallel: Error: Output is incomplete. -par_tmp_full parallel: Error: Cannot append to buffer file in /tmp/shm/parallel. -par_tmp_full parallel: Error: Is the disk full? -par_tmp_full parallel: Error: Change $TMPDIR with --tmpdir or use --compress. -par_tmp_full Warning: unable to close filehandle properly: No space left on device during global destruction. diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 5a916c80..146b8a8f 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -72,7 +72,7 @@ par_remote_load parallel: Warning: This job was killed because it timed out: par_remote_load parallel: Warning: perl -e 'while(1){ }' par_remote_load OK par_remote_nice ### Test --nice remote -par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; +par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; par_remote_nice a b par_ssh ### use --ssh par_ssh Run through BARSSH? diff --git a/testsuite/wanted-results/parallel-local-ssh9 b/testsuite/wanted-results/parallel-local-ssh9 index 955a84e4..d12c1867 100644 --- a/testsuite/wanted-results/parallel-local-ssh9 +++ b/testsuite/wanted-results/parallel-local-ssh9 @@ -136,3 +136,14 @@ par_ash_embed your par_ash_embed code par_ash_embed here par_ash_embed Put your code here +par_PARALLEL_SSHLOGIN ### bug #56554: Introduce $PARALLEL_SSHLOGIN +par_PARALLEL_SSHLOGIN /usr/bin/ssh csh@lo /usr/bin/ssh csh@lo +par_PARALLEL_SSHLOGIN /usr/bin/ssh csh@lo csh +par_PARALLEL_SSHLOGIN /usr/bin/ssh tcsh@lo /usr/bin/ssh tcsh@lo +par_PARALLEL_SSHLOGIN /usr/bin/ssh tcsh@lo tcsh +par_PARALLEL_SSHLOGIN lo lo +par_PARALLEL_SSHLOGIN lo tange +par_PARALLEL_SSHLOGIN sh@lo sh +par_PARALLEL_SSHLOGIN sh@lo sh@lo +par_PARALLEL_SSHLOGIN zsh@lo zsh +par_PARALLEL_SSHLOGIN zsh@lo zsh@lo diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index c266210b..893f9fff 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -1,7 +1,3 @@ -echo '### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name' -### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name - echo "echo echo from ./local_test_profile" > local_test_profile; parallel --profile local_test_profile echo ::: 1; rm local_test_profile -echo from ./local_test_profile echo 1 echo "### Test --delay" ### Test --delay seq 9 | /usr/bin/time -f %e parallel -j3 --delay 0.57 true {} 2>&1 | perl -ne '$_ > 3.3 and print "More than 3.3 secs: OK\n"'