diff --git a/src/parallel b/src/parallel index 34049aec..283bc999 100755 --- a/src/parallel +++ b/src/parallel @@ -1277,62 +1277,53 @@ sub start_more_jobs { # Returns: # number of jobs started my $jobs_started = 0; - if(not $Global::start_no_new_jobs) { - if($Global::max_procs_file) { - my $mtime = (stat($Global::max_procs_file))[9]; - if($mtime > $Global::max_procs_file_last_mod) { - $Global::max_procs_file_last_mod = $mtime; - for my $sshlogin (values %Global::host) { - $sshlogin->set_max_jobs_running(undef); - } - } - } - if($Global::max_load_file) { - my $mtime = (stat($Global::max_load_file))[9]; - if($mtime > $Global::max_load_file_last_mod) { - $Global::max_load_file_last_mod = $mtime; - for my $sshlogin (values %Global::host) { - $sshlogin->set_max_loadavg(undef); - } - } - } - - my $sleep = 0.0001; # 0.01 ms - better performance on highend - for my $sshlogin (values %Global::host) { - debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n"); - while ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) { - if($opt::load and $sshlogin->loadavg_too_high()) { - # The load is too high or unknown - $sleep = ::reap_usleep($sleep); - next; - } - if($opt::noswap and $sshlogin->swapping()) { - # The server is swapping - $sleep = ::reap_usleep($sleep); - next; - } - if($sshlogin->too_fast_remote_login()) { - next; - } - if($Global::JobQueue->empty() and not $opt::pipe) { - last; - } - debug($sshlogin->string()." has ".$sshlogin->jobs_running() - . " out of " . $sshlogin->max_jobs_running() - . " jobs running. Start another.\n"); - if(start_another_job($sshlogin) == 0) { - # No more jobs to start on this $sshlogin - debug("No jobs started on ".$sshlogin->string()."\n"); - last; - } - debug("Job started on ".$sshlogin->string()."\n"); - $sshlogin->inc_jobs_running(); - $sshlogin->set_last_login_at(::hires_time()); - $jobs_started++; - } - debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running() - ." of ".$sshlogin->max_jobs_running() ."\n"); - } + if($Global::start_no_new_jobs) { + return $jobs_started; + } + if($Global::max_procs_file) { + my $mtime = (stat($Global::max_procs_file))[9]; + if($mtime > $Global::max_procs_file_last_mod) { + $Global::max_procs_file_last_mod = $mtime; + for my $sshlogin (values %Global::host) { + $sshlogin->set_max_jobs_running(undef); + } + } + } + my $sleep = 0.0001; # 0.01 ms - better performance on highend + for my $sshlogin (values %Global::host) { + debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n"); + while ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) { + if($opt::load and $sshlogin->loadavg_too_high()) { + # The load is too high or unknown + $sleep = ::reap_usleep($sleep); + next; + } + if($opt::noswap and $sshlogin->swapping()) { + # The server is swapping + $sleep = ::reap_usleep($sleep); + next; + } + if($sshlogin->too_fast_remote_login()) { + next; + } + if($Global::JobQueue->empty() and not $opt::pipe) { + last; + } + debug($sshlogin->string()." has ".$sshlogin->jobs_running() + . " out of " . $sshlogin->max_jobs_running() + . " jobs running. Start another.\n"); + if(start_another_job($sshlogin) == 0) { + # No more jobs to start on this $sshlogin + debug("No jobs started on ".$sshlogin->string()."\n"); + last; + } + debug("Job started on ".$sshlogin->string()."\n"); + $sshlogin->inc_jobs_running(); + $sshlogin->set_last_login_at(::hires_time()); + $jobs_started++; + } + debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running() + ." of ".$sshlogin->max_jobs_running() ."\n"); } return $jobs_started; } @@ -2523,6 +2514,16 @@ sub loadavg { sub max_loadavg { my $self = shift; + # If --load is a file it might be changed + if($Global::max_load_file) { + my $mtime = (stat($Global::max_load_file))[9]; + if($mtime > $Global::max_load_file_last_mod) { + $Global::max_load_file_last_mod = $mtime; + for my $sshlogin (values %Global::host) { + $sshlogin->set_max_loadavg(undef); + } + } + } if(not defined $self->{'max_loadavg'}) { $self->{'max_loadavg'} = $self->compute_max_loadavg($opt::load); @@ -3766,7 +3767,7 @@ sub sshreturn { my @cmd = (); my $rsync_destdir = ($relpath ? "./" : "/"); my $ret_file = $file; - my $cd; + my $cd = ""; if($relpath) { # rsync -avR /foo/./bar/baz.c remote:/tmp/ # == (on old systems) diff --git a/testsuite/tests-to-run/test45.sh b/testsuite/tests-to-run/test45.sh index 58895ad6..4266d82f 100644 --- a/testsuite/tests-to-run/test45.sh +++ b/testsuite/tests-to-run/test45.sh @@ -1,125 +1,40 @@ #!/bin/bash -rsync -Ha --delete input-files/segfault/ tmp/ -cd tmp - SERVER1=parallel-server3 -SERVER2=parallel-server2 +SERVER2=parallel-server1 # -L1 will join lines ending in ' ' cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1 -echo '### Test --load locally'; - echo '# This will force the loadavg > 10'; - while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; +echo "### BUG: The length for -X is not close to max (131072)"; + seq 1 60000 | nice parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc + seq 1 60000 | nice parallel -X echo a{}b{}c |head -n 1 |wc + seq 1 60000 | nice parallel -X echo |head -n 1 |wc + seq 1 60000 | nice parallel -X echo a{}b{}c {} |head -n 1 |wc + seq 1 60000 | nice parallel -X echo {}aa{} |head -n 1 |wc + seq 1 60000 | nice parallel -X echo {} aa {} |head -n 1 |wc + +echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin' + seq 1 150 | stdout parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null" + +echo '### Test --load locally - should take >10s' + echo '# This will run 10 processes in parallel for 10s'; + seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" & stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' -echo '### Test --load remote'; - ssh parallel@$SERVER2 "while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done"; +echo '### Test --load remote' + ssh parallel@$SERVER2 'seq 10 | parallel --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null"' & stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' -echo '### Test --load read from a file - more than 3s'; - echo '# This will force the loadavg > 10'; - while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; - ( echo 8 > /tmp/parallel_load_file; sleep 3; echo 100 > /tmp/parallel_load_file ) & - stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"' +echo '### Test --load read from a file - more than 3s' + echo '# This will run 10 processes in parallel for 10s'; + seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" & + ( echo 8 > /tmp/parallel_load_file; sleep 4; echo 1000 > /tmp/parallel_load_file ) & + sleep 1;stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"' echo '### Test --load read from a file - less than 10s'; - echo '# This will force the loadavg > 10'; - while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; - ( echo 8 > /tmp/parallel_load_file2; sleep 3; echo 100 > /tmp/parallel_load_file2 ) & + echo '# This will run 10 processes in parallel for 10s'; + seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" & + ( echo 8 > /tmp/parallel_load_file2; sleep 4; echo 1000 > /tmp/parallel_load_file2 ) & stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 10 and print "OK\n"' -echo '### Bug in --load'; - parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3 - -echo '### Test --timeout'; - parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 6.6 7.7 8.8 - -echo '### Test retired'; - stdout parallel -B; - stdout parallel -g; - stdout parallel -H; - stdout parallel -T; - stdout parallel -U; - stdout parallel -W; - stdout parallel -Y; - -echo '### Test --joblog followed by --resume --joblog'; - rm -f /tmp/joblog; - timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; - parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4; - cat /tmp/joblog | wc -lw; - rm -f /tmp/joblog; - -echo '### Test --resume --joblog followed by --resume --joblog'; - rm -f /tmp/joblog2; - timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; - parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4; - cat /tmp/joblog2 | wc -lw; - rm -f /tmp/joblog2; - -echo '### Test --header'; - printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.}; - -echo '### 64-bit wierdness - this did not complete on a 64-bit machine'; - seq 1 2 | parallel -j1 'seq 1 1 | parallel true' - -echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'"; - stdout bash -c 'parallel -k -a <(seq 1 3) echo' - -echo "### BUG: The length for -X is not close to max (131072)"; - seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc; - seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc; - seq 1 60000 | parallel -X echo |head -n 1 |wc; - seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc; - seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc; - seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc; - -echo "### bug #35268: shell_quote doesn't treats [] brackets correctly"; - touch /tmp/foo1; - stdout parallel echo ::: '/tmp/foo[123]' - -echo '### Test make .deb package'; - cd ~/privat/parallel/packager/debian; - stdout make | grep 'To install the GNU Parallel Debian package, run:' - -echo '### Test of segfaulting issue'; - echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; - echo 'before adding wait() before exit'; - seq 1 300 | stdout parallel ./trysegfault - -echo '### Test basic --arg-sep'; - parallel -k echo ::: a b - -echo '### Run commands using --arg-sep'; - parallel -kv ::: 'echo a' 'echo b' - -echo '### Change --arg-sep'; - parallel --arg-sep ::: -kv ::: 'echo a' 'echo b'; - parallel --arg-sep .--- -kv .--- 'echo a' 'echo b'; - parallel --argsep ::: -kv ::: 'echo a' 'echo b'; - parallel --argsep .--- -kv .--- 'echo a' 'echo b' - -echo '### Test stdin goes to first command only' -echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b' -echo via cat |parallel -kv ::: 'cat' 'echo b' - -echo '### Bug made 4 5 go before 1 2 3'; - parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5" - -echo '### Bug made 3 go before 1 2'; - parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3" - -echo '### Bug did not quote'; - echo '>' | parallel -v echo; - parallel -v echo ::: '>'; - (echo '>'; echo 2) | parallel -j1 -vX echo; - parallel -X -j1 echo ::: '>' 2 - -echo '### Must not quote'; - echo 'echo | wc -l' | parallel -v; - parallel -v ::: 'echo | wc -l'; - echo 'echo a b c | wc -w' | parallel -v; - parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w' - EOF diff --git a/testsuite/wanted-results/test45 b/testsuite/wanted-results/test45 index a05d03c7..f2dc1e8c 100644 --- a/testsuite/wanted-results/test45 +++ b/testsuite/wanted-results/test45 @@ -1,57 +1,3 @@ -### Test --load locally -# This will force the loadavg > 10 -OK -### Test --load remote -OK -### Test --load read from a file - more than 3s -# This will force the loadavg > 10 -OK -### Test --load read from a file - less than 10s -# This will force the loadavg > 10 -OK -### Bug in --load -1 -2 -3 -### Test --timeout -1.1 -1.1 -6.6 -7.7 -8.8 -### Test retired -parallel: Error: -g has been retired. Use --group. -parallel: Error: -B has been retired. Use --bf. -parallel: Error: -T has been retired. Use --tty. -parallel: Error: -U has been retired. Use --er. -parallel: Error: -W has been retired. Use --wd. -parallel: Error: -Y has been retired. Use --shebang. -parallel: Error: -H has been retired. Use --halt. -parallel: Error: -g has been retired. Use --group. -parallel: Error: -B has been retired. Use --bf. -parallel: Error: -T has been retired. Use --tty. -parallel: Error: -U has been retired. Use --er. -parallel: Error: -W has been retired. Use --wd. -parallel: Error: -Y has been retired. Use --shebang. -parallel: Error: -H has been retired. Use --halt. -parallel: Error: -g has been retired. Use --group. -parallel: Error: -B has been retired. Use --bf. -parallel: Error: -T has been retired. Use --tty. -parallel: Error: -U has been retired. Use --er. -parallel: Error: -W has been retired. Use --wd. -parallel: Error: -Y has been retired. Use --shebang. -parallel: Error: -H has been retired. Use --halt. -### Test --joblog followed by --resume --joblog - 5 49 -### Test --resume --joblog followed by --resume --joblog - 5 49 -### Test --header -3/4.5 1.2 3/4 4.5 3 4 -### 64-bit wierdness - this did not complete on a 64-bit machine -### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo' -1 -2 -3 ### BUG: The length for -X is not close to max (131072) 1 12821 131060 1 10948 131060 @@ -59,73 +5,15 @@ parallel: Error: -H has been retired. Use --halt. 1 15810 131064 1 11790 131058 1 25545 131055 -### bug #35268: shell_quote doesn't treats [] brackets correctly -/tmp/foo[123] -### Test make .deb package -To install the GNU Parallel Debian package, run: -### Test of segfaulting issue -This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 -before adding wait() before exit -### Test basic --arg-sep -a -b -### Run commands using --arg-sep -echo a -a -echo b -b -### Change --arg-sep -echo a -a -echo b -b -echo a -a -echo b -b -echo a -a -echo b -b -echo a -a -echo b -b -### Test stdin goes to first command only -cat -via cat -echo b -b -cat -via cat -echo b -b -### Bug made 4 5 go before 1 2 3 -1 -2 -3 -4 -5 -### Bug made 3 go before 1 2 -1 -2 -3 -### Bug did not quote -echo \> -> -echo \> -> -echo \> 2 -> 2 -> 2 -### Must not quote -echo | wc -l -1 -echo | wc -l -1 -echo a b c | wc -w -3 -echo a b c | wc -w -3 -echo a b | wc -w -2 +### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin +### Test --load locally - should take >10s +# This will run 10 processes in parallel for 10s +OK +### Test --load remote +OK +### Test --load read from a file - more than 3s +# This will run 10 processes in parallel for 10s +OK +### Test --load read from a file - less than 10s +# This will run 10 processes in parallel for 10s +OK