src/parallel: If --load is a file: Recheck it every time max_loadavg is called.

This commit is contained in:
Ole Tange 2013-03-23 21:36:17 +01:00
parent 3d26864477
commit 7a8bd820ac
3 changed files with 95 additions and 291 deletions

View file

@ -1277,62 +1277,53 @@ sub start_more_jobs {
# Returns: # Returns:
# number of jobs started # number of jobs started
my $jobs_started = 0; my $jobs_started = 0;
if(not $Global::start_no_new_jobs) { if($Global::start_no_new_jobs) {
if($Global::max_procs_file) { return $jobs_started;
my $mtime = (stat($Global::max_procs_file))[9]; }
if($mtime > $Global::max_procs_file_last_mod) { if($Global::max_procs_file) {
$Global::max_procs_file_last_mod = $mtime; my $mtime = (stat($Global::max_procs_file))[9];
for my $sshlogin (values %Global::host) { if($mtime > $Global::max_procs_file_last_mod) {
$sshlogin->set_max_jobs_running(undef); $Global::max_procs_file_last_mod = $mtime;
} for my $sshlogin (values %Global::host) {
} $sshlogin->set_max_jobs_running(undef);
} }
if($Global::max_load_file) { }
my $mtime = (stat($Global::max_load_file))[9]; }
if($mtime > $Global::max_load_file_last_mod) { my $sleep = 0.0001; # 0.01 ms - better performance on highend
$Global::max_load_file_last_mod = $mtime; for my $sshlogin (values %Global::host) {
for my $sshlogin (values %Global::host) { debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n");
$sshlogin->set_max_loadavg(undef); while ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
} if($opt::load and $sshlogin->loadavg_too_high()) {
} # The load is too high or unknown
} $sleep = ::reap_usleep($sleep);
next;
my $sleep = 0.0001; # 0.01 ms - better performance on highend }
for my $sshlogin (values %Global::host) { if($opt::noswap and $sshlogin->swapping()) {
debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n"); # The server is swapping
while ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) { $sleep = ::reap_usleep($sleep);
if($opt::load and $sshlogin->loadavg_too_high()) { next;
# The load is too high or unknown }
$sleep = ::reap_usleep($sleep); if($sshlogin->too_fast_remote_login()) {
next; next;
} }
if($opt::noswap and $sshlogin->swapping()) { if($Global::JobQueue->empty() and not $opt::pipe) {
# The server is swapping last;
$sleep = ::reap_usleep($sleep); }
next; debug($sshlogin->string()." has ".$sshlogin->jobs_running()
} . " out of " . $sshlogin->max_jobs_running()
if($sshlogin->too_fast_remote_login()) { . " jobs running. Start another.\n");
next; if(start_another_job($sshlogin) == 0) {
} # No more jobs to start on this $sshlogin
if($Global::JobQueue->empty() and not $opt::pipe) { debug("No jobs started on ".$sshlogin->string()."\n");
last; last;
} }
debug($sshlogin->string()." has ".$sshlogin->jobs_running() debug("Job started on ".$sshlogin->string()."\n");
. " out of " . $sshlogin->max_jobs_running() $sshlogin->inc_jobs_running();
. " jobs running. Start another.\n"); $sshlogin->set_last_login_at(::hires_time());
if(start_another_job($sshlogin) == 0) { $jobs_started++;
# No more jobs to start on this $sshlogin }
debug("No jobs started on ".$sshlogin->string()."\n"); debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
last; ." of ".$sshlogin->max_jobs_running() ."\n");
}
debug("Job started on ".$sshlogin->string()."\n");
$sshlogin->inc_jobs_running();
$sshlogin->set_last_login_at(::hires_time());
$jobs_started++;
}
debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
." of ".$sshlogin->max_jobs_running() ."\n");
}
} }
return $jobs_started; return $jobs_started;
} }
@ -2523,6 +2514,16 @@ sub loadavg {
sub max_loadavg { sub max_loadavg {
my $self = shift; my $self = shift;
# If --load is a file it might be changed
if($Global::max_load_file) {
my $mtime = (stat($Global::max_load_file))[9];
if($mtime > $Global::max_load_file_last_mod) {
$Global::max_load_file_last_mod = $mtime;
for my $sshlogin (values %Global::host) {
$sshlogin->set_max_loadavg(undef);
}
}
}
if(not defined $self->{'max_loadavg'}) { if(not defined $self->{'max_loadavg'}) {
$self->{'max_loadavg'} = $self->{'max_loadavg'} =
$self->compute_max_loadavg($opt::load); $self->compute_max_loadavg($opt::load);
@ -3766,7 +3767,7 @@ sub sshreturn {
my @cmd = (); my @cmd = ();
my $rsync_destdir = ($relpath ? "./" : "/"); my $rsync_destdir = ($relpath ? "./" : "/");
my $ret_file = $file; my $ret_file = $file;
my $cd; my $cd = "";
if($relpath) { if($relpath) {
# rsync -avR /foo/./bar/baz.c remote:/tmp/ # rsync -avR /foo/./bar/baz.c remote:/tmp/
# == (on old systems) # == (on old systems)

View file

@ -1,125 +1,40 @@
#!/bin/bash #!/bin/bash
rsync -Ha --delete input-files/segfault/ tmp/
cd tmp
SERVER1=parallel-server3 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server1
# -L1 will join lines ending in ' ' # -L1 will join lines ending in ' '
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1
echo '### Test --load locally'; echo "### BUG: The length for -X is not close to max (131072)";
echo '# This will force the loadavg > 10'; seq 1 60000 | nice parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc
while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; seq 1 60000 | nice parallel -X echo a{}b{}c |head -n 1 |wc
seq 1 60000 | nice parallel -X echo |head -n 1 |wc
seq 1 60000 | nice parallel -X echo a{}b{}c {} |head -n 1 |wc
seq 1 60000 | nice parallel -X echo {}aa{} |head -n 1 |wc
seq 1 60000 | nice parallel -X echo {} aa {} |head -n 1 |wc
echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin'
seq 1 150 | stdout parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null"
echo '### Test --load locally - should take >10s'
echo '# This will run 10 processes in parallel for 10s';
seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
echo '### Test --load remote'; echo '### Test --load remote'
ssh parallel@$SERVER2 "while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done"; ssh parallel@$SERVER2 'seq 10 | parallel --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null"' &
stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
echo '### Test --load read from a file - more than 3s'; echo '### Test --load read from a file - more than 3s'
echo '# This will force the loadavg > 10'; echo '# This will run 10 processes in parallel for 10s';
while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
( echo 8 > /tmp/parallel_load_file; sleep 3; echo 100 > /tmp/parallel_load_file ) & ( echo 8 > /tmp/parallel_load_file; sleep 4; echo 1000 > /tmp/parallel_load_file ) &
stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"' sleep 1;stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"'
echo '### Test --load read from a file - less than 10s'; echo '### Test --load read from a file - less than 10s';
echo '# This will force the loadavg > 10'; echo '# This will run 10 processes in parallel for 10s';
while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
( echo 8 > /tmp/parallel_load_file2; sleep 3; echo 100 > /tmp/parallel_load_file2 ) & ( echo 8 > /tmp/parallel_load_file2; sleep 4; echo 1000 > /tmp/parallel_load_file2 ) &
stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 10 and print "OK\n"' stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 10 and print "OK\n"'
echo '### Bug in --load';
parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3
echo '### Test --timeout';
parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 6.6 7.7 8.8
echo '### Test retired';
stdout parallel -B;
stdout parallel -g;
stdout parallel -H;
stdout parallel -T;
stdout parallel -U;
stdout parallel -W;
stdout parallel -Y;
echo '### Test --joblog followed by --resume --joblog';
rm -f /tmp/joblog;
timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null;
parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4;
cat /tmp/joblog | wc -lw;
rm -f /tmp/joblog;
echo '### Test --resume --joblog followed by --resume --joblog';
rm -f /tmp/joblog2;
timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null;
parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4;
cat /tmp/joblog2 | wc -lw;
rm -f /tmp/joblog2;
echo '### Test --header';
printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.};
echo '### 64-bit wierdness - this did not complete on a 64-bit machine';
seq 1 2 | parallel -j1 'seq 1 1 | parallel true'
echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'";
stdout bash -c 'parallel -k -a <(seq 1 3) echo'
echo "### BUG: The length for -X is not close to max (131072)";
seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc;
seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc;
seq 1 60000 | parallel -X echo |head -n 1 |wc;
seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc;
seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc;
seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc;
echo "### bug #35268: shell_quote doesn't treats [] brackets correctly";
touch /tmp/foo1;
stdout parallel echo ::: '/tmp/foo[123]'
echo '### Test make .deb package';
cd ~/privat/parallel/packager/debian;
stdout make | grep 'To install the GNU Parallel Debian package, run:'
echo '### Test of segfaulting issue';
echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1';
echo 'before adding wait() before exit';
seq 1 300 | stdout parallel ./trysegfault
echo '### Test basic --arg-sep';
parallel -k echo ::: a b
echo '### Run commands using --arg-sep';
parallel -kv ::: 'echo a' 'echo b'
echo '### Change --arg-sep';
parallel --arg-sep ::: -kv ::: 'echo a' 'echo b';
parallel --arg-sep .--- -kv .--- 'echo a' 'echo b';
parallel --argsep ::: -kv ::: 'echo a' 'echo b';
parallel --argsep .--- -kv .--- 'echo a' 'echo b'
echo '### Test stdin goes to first command only'
echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b'
echo via cat |parallel -kv ::: 'cat' 'echo b'
echo '### Bug made 4 5 go before 1 2 3';
parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5"
echo '### Bug made 3 go before 1 2';
parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3"
echo '### Bug did not quote';
echo '>' | parallel -v echo;
parallel -v echo ::: '>';
(echo '>'; echo 2) | parallel -j1 -vX echo;
parallel -X -j1 echo ::: '>' 2
echo '### Must not quote';
echo 'echo | wc -l' | parallel -v;
parallel -v ::: 'echo | wc -l';
echo 'echo a b c | wc -w' | parallel -v;
parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w'
EOF EOF

View file

@ -1,57 +1,3 @@
### Test --load locally
# This will force the loadavg > 10
OK
### Test --load remote
OK
### Test --load read from a file - more than 3s
# This will force the loadavg > 10
OK
### Test --load read from a file - less than 10s
# This will force the loadavg > 10
OK
### Bug in --load
1
2
3
### Test --timeout
1.1
1.1
6.6
7.7
8.8
### Test retired
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
### Test --joblog followed by --resume --joblog
5 49
### Test --resume --joblog followed by --resume --joblog
5 49
### Test --header
3/4.5 1.2 3/4 4.5 3 4
### 64-bit wierdness - this did not complete on a 64-bit machine
### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'
1
2
3
### BUG: The length for -X is not close to max (131072) ### BUG: The length for -X is not close to max (131072)
1 12821 131060 1 12821 131060
1 10948 131060 1 10948 131060
@ -59,73 +5,15 @@ parallel: Error: -H has been retired. Use --halt.
1 15810 131064 1 15810 131064
1 11790 131058 1 11790 131058
1 25545 131055 1 25545 131055
### bug #35268: shell_quote doesn't treats [] brackets correctly ### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin
/tmp/foo[123] ### Test --load locally - should take >10s
### Test make .deb package # This will run 10 processes in parallel for 10s
To install the GNU Parallel Debian package, run: OK
### Test of segfaulting issue ### Test --load remote
This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 OK
before adding wait() before exit ### Test --load read from a file - more than 3s
### Test basic --arg-sep # This will run 10 processes in parallel for 10s
a OK
b ### Test --load read from a file - less than 10s
### Run commands using --arg-sep # This will run 10 processes in parallel for 10s
echo a OK
a
echo b
b
### Change --arg-sep
echo a
a
echo b
b
echo a
a
echo b
b
echo a
a
echo b
b
echo a
a
echo b
b
### Test stdin goes to first command only
cat
via cat
echo b
b
cat
via cat
echo b
b
### Bug made 4 5 go before 1 2 3
1
2
3
4
5
### Bug made 3 go before 1 2
1
2
3
### Bug did not quote
echo \>
>
echo \>
>
echo \> 2
> 2
> 2
### Must not quote
echo | wc -l
1
echo | wc -l
1
echo a b c | wc -w
3
echo a b c | wc -w
3
echo a b | wc -w
2