tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixes bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded

Browse source
This commit is contained in:
Ole Tange 2014-11-07 11:05:24 +01:00
parent 7c33ff6850
commit ee6d07d49b
8 changed files with 36 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/release_new_version
View file

@ -234,10 +234,14 @@ Haiku of the month:
<<>>
A central piece of command generation was rewritten making this release alpha quality.
A central piece of command generation was rewritten making this release beta quality. As always it passes the testsuite, so most functionality clearly works.
New in this release:
* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close
* GNU Parallel was cited in: Exploring a multiprocessor design space to analyze the impact of using STT-RAM in the memory hierarchy http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf
* Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane.

17
src/parallel
View file

@ -771,7 +771,7 @@ sub get_options_from_array {
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20141023;
$Global::version = 20141107;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -2280,16 +2280,21 @@ sub parse_sshlogin {
}
}
$Global::minimal_command_line_length = 8_000_000;
for my $sshlogin_string (::uniq(@login)) {
for my $ncpu_sshlogin_string (::uniq(@login)) {
my $sshlogin = SSHLogin->new($ncpu_sshlogin_string);
my $sshlogin_string = $sshlogin->string();
if($Global::host{$sshlogin_string}) {
# This sshlogin has already been added:
# It is probably a host that has come back
# Set the max_jobs_running back to the original
$Global::host{$sshlogin_string}->set_max_jobs_running(
$Global::host{$sshlogin_string}->orig_max_jobs_running());
debug("run","Already seen $sshlogin_string\n");
if($sshlogin->{'ncpus'}) {
# If ncpus set by '#/' of the sshlogin, overwrite it:
$Global::host{$sshlogin_string}->set_ncpus($sshlogin->ncpus());
}
$Global::host{$sshlogin_string}->set_max_jobs_running(undef);
next;
}
my $sshlogin = SSHLogin->new($sshlogin_string);
if($sshlogin_string eq ":") {
$sshlogin->set_maxlength(Limits::Command::max_length());
} else {
@ -2298,7 +2303,7 @@ sub parse_sshlogin {
}
$Global::minimal_command_line_length =
::min($Global::minimal_command_line_length, $sshlogin->maxlength());
$Global::host{$sshlogin->string()} = $sshlogin;
$Global::host{$sshlogin_string} = $sshlogin;
}
# debug("start", "sshlogin: ", my_dump(%Global::host),"\n");
if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) {

2
src/parallel.pod
View file

@ -565,7 +565,7 @@ remote execution.
In Bash I<var> can also be a Bash function - just remember to B<export
-f> the function, see B<command>.
The variable '_' is special. It will copy all enviroment variables
The variable '_' is special. It will copy all environment variables
except for the ones mentioned in ~/.parallel/ignored_vars.
To copy Bash arrays you need an importer function, as Bash arrays

4
src/parallel_tutorial.1
View file

@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "PARALLEL_TUTORIAL 1"
.TH PARALLEL_TUTORIAL 1 "2014-09-23" "20140922" "parallel"
.TH PARALLEL_TUTORIAL 1 "2014-10-31" "20141022" "parallel"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
@ -1664,7 +1664,7 @@ Output:
.Ve
.PP
\&\s-1GNU\s0 Parallel can retry the command with \-\-retries. This is useful if a
command fails for unkown reasons now and then.
command fails for unknown reasons now and then.
.PP
.Vb 2
\& parallel \-k \-\-retries 3 \*(Aqecho tried {} >>/tmp/runs; echo completed {}; exit {}\*(Aq ::: 1 2 0

2
src/parallel_tutorial.html
View file

@ -1240,7 +1240,7 @@
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
echo 5; exit 5</code></pre>
<p>GNU Parallel can retry the command with --retries. This is useful if a command fails for unkown reasons now and then.</p>
<p>GNU Parallel can retry the command with --retries. This is useful if a command fails for unknown reasons now and then.</p>
<pre><code> parallel -k --retries 3 &#39;echo tried {} &gt;&gt;/tmp/runs; echo completed {}; exit {}&#39; ::: 1 2 0
cat /tmp/runs</code></pre>

2
src/parallel_tutorial.pod
View file

@ -1190,7 +1190,7 @@ Output:
echo 5; exit 5
GNU Parallel can retry the command with --retries. This is useful if a
command fails for unkown reasons now and then.
command fails for unknown reasons now and then.
parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
cat /tmp/runs

10
testsuite/tests-to-run/parallel-local-ssh2.sh
View file

@ -1,6 +1,14 @@
#!/bin/bash
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj8 -k -L1
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj7 -k -L1
echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded"
# Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed
# Thus must take at least 25 sec to run
echo -e '1/lo\n1/csh@lo\n1/tcsh@lo\n1/parallel@lo\n' > /tmp/parallel.bug43518;
parallel --delay 0.1 -N0 echo 1/: '>>' /tmp/parallel.bug43518 ::: {1..100} &
seq 30 | stdout /usr/bin/time -f %e parallel --slf /tmp/parallel.bug43518 'sleep {=$_=$_%3?0:10=}.{%}' |
perl -ne '$_ > 25 and print "OK\n"'
echo '### --filter-hosts --slf <()'
parallel --nonall --filter-hosts --slf <(echo localhost) echo OK

6
testsuite/wanted-results/parallel-local-ssh2
View file

@ -1,3 +1,9 @@
echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded"
### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded
# Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed
# Thus must take at least 25 sec to run
echo -e '1/lo\n1/csh@lo\n1/tcsh@lo\n1/parallel@lo\n' > /tmp/parallel.bug43518; parallel --delay 0.1 -N0 echo 1/: '>>' /tmp/parallel.bug43518 ::: {1..100} & seq 30 | stdout /usr/bin/time -f %e parallel --slf /tmp/parallel.bug43518 'sleep {=$_=$_%3?0:10=}.{%}' | perl -ne '$_ > 25 and print "OK\n"'
OK
echo '### --filter-hosts --slf <()'
### --filter-hosts --slf <()
parallel --nonall --filter-hosts --slf <(echo localhost) echo OK