mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 20:57:53 +00:00
src/parallel: If --load is a file: Recheck it every time max_loadavg is called.
This commit is contained in:
Ole Tange
parent
3d26864477
commit
7a8bd820ac
115
src/parallel
115
src/parallel
|
|
@ -1277,62 +1277,53 @@ sub start_more_jobs {
|
|||
# Returns:
|
||||
# number of jobs started
|
||||
my $jobs_started = 0;
|
||||
if(not $Global::start_no_new_jobs) {
|
||||
if($Global::max_procs_file) {
|
||||
my $mtime = (stat($Global::max_procs_file))[9];
|
||||
if($mtime > $Global::max_procs_file_last_mod) {
|
||||
$Global::max_procs_file_last_mod = $mtime;
|
||||
for my $sshlogin (values %Global::host) {
|
||||
$sshlogin->set_max_jobs_running(undef);
|
||||
}
|
||||
}
|
||||
}
|
||||
if($Global::max_load_file) {
|
||||
my $mtime = (stat($Global::max_load_file))[9];
|
||||
if($mtime > $Global::max_load_file_last_mod) {
|
||||
$Global::max_load_file_last_mod = $mtime;
|
||||
for my $sshlogin (values %Global::host) {
|
||||
$sshlogin->set_max_loadavg(undef);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $sleep = 0.0001; # 0.01 ms - better performance on highend
|
||||
for my $sshlogin (values %Global::host) {
|
||||
debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n");
|
||||
while ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
|
||||
if($opt::load and $sshlogin->loadavg_too_high()) {
|
||||
# The load is too high or unknown
|
||||
$sleep = ::reap_usleep($sleep);
|
||||
next;
|
||||
}
|
||||
if($opt::noswap and $sshlogin->swapping()) {
|
||||
# The server is swapping
|
||||
$sleep = ::reap_usleep($sleep);
|
||||
next;
|
||||
}
|
||||
if($sshlogin->too_fast_remote_login()) {
|
||||
next;
|
||||
}
|
||||
if($Global::JobQueue->empty() and not $opt::pipe) {
|
||||
last;
|
||||
}
|
||||
debug($sshlogin->string()." has ".$sshlogin->jobs_running()
|
||||
. " out of " . $sshlogin->max_jobs_running()
|
||||
. " jobs running. Start another.\n");
|
||||
if(start_another_job($sshlogin) == 0) {
|
||||
# No more jobs to start on this $sshlogin
|
||||
debug("No jobs started on ".$sshlogin->string()."\n");
|
||||
last;
|
||||
}
|
||||
debug("Job started on ".$sshlogin->string()."\n");
|
||||
$sshlogin->inc_jobs_running();
|
||||
$sshlogin->set_last_login_at(::hires_time());
|
||||
$jobs_started++;
|
||||
}
|
||||
debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
|
||||
." of ".$sshlogin->max_jobs_running() ."\n");
|
||||
}
|
||||
if($Global::start_no_new_jobs) {
|
||||
return $jobs_started;
|
||||
}
|
||||
if($Global::max_procs_file) {
|
||||
my $mtime = (stat($Global::max_procs_file))[9];
|
||||
if($mtime > $Global::max_procs_file_last_mod) {
|
||||
$Global::max_procs_file_last_mod = $mtime;
|
||||
for my $sshlogin (values %Global::host) {
|
||||
$sshlogin->set_max_jobs_running(undef);
|
||||
}
|
||||
}
|
||||
}
|
||||
my $sleep = 0.0001; # 0.01 ms - better performance on highend
|
||||
for my $sshlogin (values %Global::host) {
|
||||
debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n");
|
||||
while ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
|
||||
if($opt::load and $sshlogin->loadavg_too_high()) {
|
||||
# The load is too high or unknown
|
||||
$sleep = ::reap_usleep($sleep);
|
||||
next;
|
||||
}
|
||||
if($opt::noswap and $sshlogin->swapping()) {
|
||||
# The server is swapping
|
||||
$sleep = ::reap_usleep($sleep);
|
||||
next;
|
||||
}
|
||||
if($sshlogin->too_fast_remote_login()) {
|
||||
next;
|
||||
}
|
||||
if($Global::JobQueue->empty() and not $opt::pipe) {
|
||||
last;
|
||||
}
|
||||
debug($sshlogin->string()." has ".$sshlogin->jobs_running()
|
||||
. " out of " . $sshlogin->max_jobs_running()
|
||||
. " jobs running. Start another.\n");
|
||||
if(start_another_job($sshlogin) == 0) {
|
||||
# No more jobs to start on this $sshlogin
|
||||
debug("No jobs started on ".$sshlogin->string()."\n");
|
||||
last;
|
||||
}
|
||||
debug("Job started on ".$sshlogin->string()."\n");
|
||||
$sshlogin->inc_jobs_running();
|
||||
$sshlogin->set_last_login_at(::hires_time());
|
||||
$jobs_started++;
|
||||
}
|
||||
debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
|
||||
." of ".$sshlogin->max_jobs_running() ."\n");
|
||||
}
|
||||
return $jobs_started;
|
||||
}
|
||||
|
|
@ -2523,6 +2514,16 @@ sub loadavg {
|
|||
|
||||
sub max_loadavg {
|
||||
my $self = shift;
|
||||
# If --load is a file it might be changed
|
||||
if($Global::max_load_file) {
|
||||
my $mtime = (stat($Global::max_load_file))[9];
|
||||
if($mtime > $Global::max_load_file_last_mod) {
|
||||
$Global::max_load_file_last_mod = $mtime;
|
||||
for my $sshlogin (values %Global::host) {
|
||||
$sshlogin->set_max_loadavg(undef);
|
||||
}
|
||||
}
|
||||
}
|
||||
if(not defined $self->{'max_loadavg'}) {
|
||||
$self->{'max_loadavg'} =
|
||||
$self->compute_max_loadavg($opt::load);
|
||||
|
|
@ -3766,7 +3767,7 @@ sub sshreturn {
|
|||
my @cmd = ();
|
||||
my $rsync_destdir = ($relpath ? "./" : "/");
|
||||
my $ret_file = $file;
|
||||
my $cd;
|
||||
my $cd = "";
|
||||
if($relpath) {
|
||||
# rsync -avR /foo/./bar/baz.c remote:/tmp/
|
||||
# == (on old systems)
|
||||
|
|
|
|||
|
|
@ -1,125 +1,40 @@
|
|||
#!/bin/bash
|
||||
|
||||
rsync -Ha --delete input-files/segfault/ tmp/
|
||||
cd tmp
|
||||
|
||||
SERVER1=parallel-server3
|
||||
SERVER2=parallel-server2
|
||||
SERVER2=parallel-server1
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1
|
||||
echo '### Test --load locally';
|
||||
echo '# This will force the loadavg > 10';
|
||||
while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done;
|
||||
echo "### BUG: The length for -X is not close to max (131072)";
|
||||
seq 1 60000 | nice parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc
|
||||
seq 1 60000 | nice parallel -X echo a{}b{}c |head -n 1 |wc
|
||||
seq 1 60000 | nice parallel -X echo |head -n 1 |wc
|
||||
seq 1 60000 | nice parallel -X echo a{}b{}c {} |head -n 1 |wc
|
||||
seq 1 60000 | nice parallel -X echo {}aa{} |head -n 1 |wc
|
||||
seq 1 60000 | nice parallel -X echo {} aa {} |head -n 1 |wc
|
||||
|
||||
echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin'
|
||||
seq 1 150 | stdout parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null"
|
||||
|
||||
echo '### Test --load locally - should take >10s'
|
||||
echo '# This will run 10 processes in parallel for 10s';
|
||||
seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||
|
||||
echo '### Test --load remote';
|
||||
ssh parallel@$SERVER2 "while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done";
|
||||
echo '### Test --load remote'
|
||||
ssh parallel@$SERVER2 'seq 10 | parallel --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null"' &
|
||||
stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||
|
||||
echo '### Test --load read from a file - more than 3s';
|
||||
echo '# This will force the loadavg > 10';
|
||||
while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done;
|
||||
( echo 8 > /tmp/parallel_load_file; sleep 3; echo 100 > /tmp/parallel_load_file ) &
|
||||
stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"'
|
||||
echo '### Test --load read from a file - more than 3s'
|
||||
echo '# This will run 10 processes in parallel for 10s';
|
||||
seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
( echo 8 > /tmp/parallel_load_file; sleep 4; echo 1000 > /tmp/parallel_load_file ) &
|
||||
sleep 1;stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"'
|
||||
|
||||
echo '### Test --load read from a file - less than 10s';
|
||||
echo '# This will force the loadavg > 10';
|
||||
while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done;
|
||||
( echo 8 > /tmp/parallel_load_file2; sleep 3; echo 100 > /tmp/parallel_load_file2 ) &
|
||||
echo '# This will run 10 processes in parallel for 10s';
|
||||
seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
( echo 8 > /tmp/parallel_load_file2; sleep 4; echo 1000 > /tmp/parallel_load_file2 ) &
|
||||
stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 10 and print "OK\n"'
|
||||
|
||||
echo '### Bug in --load';
|
||||
parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3
|
||||
|
||||
echo '### Test --timeout';
|
||||
parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 6.6 7.7 8.8
|
||||
|
||||
echo '### Test retired';
|
||||
stdout parallel -B;
|
||||
stdout parallel -g;
|
||||
stdout parallel -H;
|
||||
stdout parallel -T;
|
||||
stdout parallel -U;
|
||||
stdout parallel -W;
|
||||
stdout parallel -Y;
|
||||
|
||||
echo '### Test --joblog followed by --resume --joblog';
|
||||
rm -f /tmp/joblog;
|
||||
timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null;
|
||||
parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4;
|
||||
cat /tmp/joblog | wc -lw;
|
||||
rm -f /tmp/joblog;
|
||||
|
||||
echo '### Test --resume --joblog followed by --resume --joblog';
|
||||
rm -f /tmp/joblog2;
|
||||
timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null;
|
||||
parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4;
|
||||
cat /tmp/joblog2 | wc -lw;
|
||||
rm -f /tmp/joblog2;
|
||||
|
||||
echo '### Test --header';
|
||||
printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.};
|
||||
|
||||
echo '### 64-bit wierdness - this did not complete on a 64-bit machine';
|
||||
seq 1 2 | parallel -j1 'seq 1 1 | parallel true'
|
||||
|
||||
echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'";
|
||||
stdout bash -c 'parallel -k -a <(seq 1 3) echo'
|
||||
|
||||
echo "### BUG: The length for -X is not close to max (131072)";
|
||||
seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc;
|
||||
seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc;
|
||||
seq 1 60000 | parallel -X echo |head -n 1 |wc;
|
||||
seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc;
|
||||
seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc;
|
||||
seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc;
|
||||
|
||||
echo "### bug #35268: shell_quote doesn't treats [] brackets correctly";
|
||||
touch /tmp/foo1;
|
||||
stdout parallel echo ::: '/tmp/foo[123]'
|
||||
|
||||
echo '### Test make .deb package';
|
||||
cd ~/privat/parallel/packager/debian;
|
||||
stdout make | grep 'To install the GNU Parallel Debian package, run:'
|
||||
|
||||
echo '### Test of segfaulting issue';
|
||||
echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1';
|
||||
echo 'before adding wait() before exit';
|
||||
seq 1 300 | stdout parallel ./trysegfault
|
||||
|
||||
echo '### Test basic --arg-sep';
|
||||
parallel -k echo ::: a b
|
||||
|
||||
echo '### Run commands using --arg-sep';
|
||||
parallel -kv ::: 'echo a' 'echo b'
|
||||
|
||||
echo '### Change --arg-sep';
|
||||
parallel --arg-sep ::: -kv ::: 'echo a' 'echo b';
|
||||
parallel --arg-sep .--- -kv .--- 'echo a' 'echo b';
|
||||
parallel --argsep ::: -kv ::: 'echo a' 'echo b';
|
||||
parallel --argsep .--- -kv .--- 'echo a' 'echo b'
|
||||
|
||||
echo '### Test stdin goes to first command only'
|
||||
echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b'
|
||||
echo via cat |parallel -kv ::: 'cat' 'echo b'
|
||||
|
||||
echo '### Bug made 4 5 go before 1 2 3';
|
||||
parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5"
|
||||
|
||||
echo '### Bug made 3 go before 1 2';
|
||||
parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3"
|
||||
|
||||
echo '### Bug did not quote';
|
||||
echo '>' | parallel -v echo;
|
||||
parallel -v echo ::: '>';
|
||||
(echo '>'; echo 2) | parallel -j1 -vX echo;
|
||||
parallel -X -j1 echo ::: '>' 2
|
||||
|
||||
echo '### Must not quote';
|
||||
echo 'echo | wc -l' | parallel -v;
|
||||
parallel -v ::: 'echo | wc -l';
|
||||
echo 'echo a b c | wc -w' | parallel -v;
|
||||
parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w'
|
||||
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -1,57 +1,3 @@
|
|||
### Test --load locally
|
||||
# This will force the loadavg > 10
|
||||
OK
|
||||
### Test --load remote
|
||||
OK
|
||||
### Test --load read from a file - more than 3s
|
||||
# This will force the loadavg > 10
|
||||
OK
|
||||
### Test --load read from a file - less than 10s
|
||||
# This will force the loadavg > 10
|
||||
OK
|
||||
### Bug in --load
|
||||
1
|
||||
2
|
||||
3
|
||||
### Test --timeout
|
||||
1.1
|
||||
1.1
|
||||
6.6
|
||||
7.7
|
||||
8.8
|
||||
### Test retired
|
||||
parallel: Error: -g has been retired. Use --group.
|
||||
parallel: Error: -B has been retired. Use --bf.
|
||||
parallel: Error: -T has been retired. Use --tty.
|
||||
parallel: Error: -U has been retired. Use --er.
|
||||
parallel: Error: -W has been retired. Use --wd.
|
||||
parallel: Error: -Y has been retired. Use --shebang.
|
||||
parallel: Error: -H has been retired. Use --halt.
|
||||
parallel: Error: -g has been retired. Use --group.
|
||||
parallel: Error: -B has been retired. Use --bf.
|
||||
parallel: Error: -T has been retired. Use --tty.
|
||||
parallel: Error: -U has been retired. Use --er.
|
||||
parallel: Error: -W has been retired. Use --wd.
|
||||
parallel: Error: -Y has been retired. Use --shebang.
|
||||
parallel: Error: -H has been retired. Use --halt.
|
||||
parallel: Error: -g has been retired. Use --group.
|
||||
parallel: Error: -B has been retired. Use --bf.
|
||||
parallel: Error: -T has been retired. Use --tty.
|
||||
parallel: Error: -U has been retired. Use --er.
|
||||
parallel: Error: -W has been retired. Use --wd.
|
||||
parallel: Error: -Y has been retired. Use --shebang.
|
||||
parallel: Error: -H has been retired. Use --halt.
|
||||
### Test --joblog followed by --resume --joblog
|
||||
5 49
|
||||
### Test --resume --joblog followed by --resume --joblog
|
||||
5 49
|
||||
### Test --header
|
||||
3/4.5 1.2 3/4 4.5 3 4
|
||||
### 64-bit wierdness - this did not complete on a 64-bit machine
|
||||
### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'
|
||||
1
|
||||
2
|
||||
3
|
||||
### BUG: The length for -X is not close to max (131072)
|
||||
1 12821 131060
|
||||
1 10948 131060
|
||||
|
|
@ -59,73 +5,15 @@ parallel: Error: -H has been retired. Use --halt.
|
|||
1 15810 131064
|
||||
1 11790 131058
|
||||
1 25545 131055
|
||||
### bug #35268: shell_quote doesn't treats [] brackets correctly
|
||||
/tmp/foo[123]
|
||||
### Test make .deb package
|
||||
To install the GNU Parallel Debian package, run:
|
||||
### Test of segfaulting issue
|
||||
This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1
|
||||
before adding wait() before exit
|
||||
### Test basic --arg-sep
|
||||
a
|
||||
b
|
||||
### Run commands using --arg-sep
|
||||
echo a
|
||||
a
|
||||
echo b
|
||||
b
|
||||
### Change --arg-sep
|
||||
echo a
|
||||
a
|
||||
echo b
|
||||
b
|
||||
echo a
|
||||
a
|
||||
echo b
|
||||
b
|
||||
echo a
|
||||
a
|
||||
echo b
|
||||
b
|
||||
echo a
|
||||
a
|
||||
echo b
|
||||
b
|
||||
### Test stdin goes to first command only
|
||||
cat
|
||||
via cat
|
||||
echo b
|
||||
b
|
||||
cat
|
||||
via cat
|
||||
echo b
|
||||
b
|
||||
### Bug made 4 5 go before 1 2 3
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
### Bug made 3 go before 1 2
|
||||
1
|
||||
2
|
||||
3
|
||||
### Bug did not quote
|
||||
echo \>
|
||||
>
|
||||
echo \>
|
||||
>
|
||||
echo \> 2
|
||||
> 2
|
||||
> 2
|
||||
### Must not quote
|
||||
echo | wc -l
|
||||
1
|
||||
echo | wc -l
|
||||
1
|
||||
echo a b c | wc -w
|
||||
3
|
||||
echo a b c | wc -w
|
||||
3
|
||||
echo a b | wc -w
|
||||
2
|
||||
### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin
|
||||
### Test --load locally - should take >10s
|
||||
# This will run 10 processes in parallel for 10s
|
||||
OK
|
||||
### Test --load remote
|
||||
OK
|
||||
### Test --load read from a file - more than 3s
|
||||
# This will run 10 processes in parallel for 10s
|
||||
OK
|
||||
### Test --load read from a file - less than 10s
|
||||
# This will run 10 processes in parallel for 10s
|
||||
OK
|
||||
|
|
|
|||
Loading…
Reference in a new issue