tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-26 07:57:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Code cleanup.

Browse source
This commit is contained in:
Ole Tange 2015-12-14 01:23:34 +01:00
parent c1d01e800e
commit 2fcb6f8df6
2 changed files with 26 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/parallel
View file

@ -1160,7 +1160,7 @@ sub check_invalid_option_combinations {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20151123; $Global::version = 20151213;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -3012,25 +3012,31 @@ sub parallelized_host_filtering {
while (my ($host, $sshlogin) = each %Global::host) { while (my ($host, $sshlogin) = each %Global::host) {
if($host eq ":") { next } if($host eq ":") { next }
# The 'true' is used to get the $host out later # The 'true' is used to get the $host out later
my $sshcmd = "true $host; exec " . $sshlogin->sshcommand()." ".$sshlogin->serverlogin(); my $sshcmd = "true $host; exec " .$sshlogin->sshcommand()." ".
push(@cores, $host."\t".$sshcmd." -- ".$envvar." parallel --number-of-cores\n\0"); $sshlogin->serverlogin();
push(@cpus, $host."\t".$sshcmd." -- ".$envvar." parallel --number-of-cpus\n\0"); push(@cores, $host."\t".$sshcmd." -- ".$envvar.
push(@maxline, $host."\t".$sshcmd." -- ".$envvar." parallel --max-line-length-allowed\n\0"); " parallel --number-of-cores\n\0");
push(@cpus, $host."\t".$sshcmd." -- ".$envvar.
" parallel --number-of-cpus\n\0");
push(@maxline, $host."\t".$sshcmd." -- ".$envvar.
" parallel --max-line-length-allowed\n\0");
# 'echo' is used to get the best possible value for an ssh login time # 'echo' is used to get the best possible value for an ssh login time
push(@echo, $host."\t".$sshcmd." -- echo\n\0"); push(@echo, $host."\t".$sshcmd." -- echo\n\0");
} }
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh"); my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh");
print $fh @cores, @cpus, @maxline, @echo; print $fh @cores, @cpus, @maxline, @echo;
close $fh; close $fh;
# --timeout 5: Setting up an SSH connection and running a simple # --timeout 10: Setting up an SSH connection and running a simple
# command should never take > 5 sec. # command should never take > 10 sec.
# --delay 0.1: If multiple sshlogins use the same proxy the delay # --delay 0.1: If multiple sshlogins use the same proxy the delay
# will make it less likely to overload the ssh daemon. # will make it less likely to overload the ssh daemon.
# --retries 3: If the ssh daemon it overloaded, try 3 times # --retries 3: If the ssh daemon it overloaded, try 3 times
# -s 16000: Half of the max line on UnixWare # -s 16000: Half of the max line on UnixWare
# TODO sh -c wrapper to work in csh # TODO sh -c wrapper to work in csh
my $unlinkcmd = $Global::debug ? "true" : "rm $tmpfile"; my $unlinkcmd = $Global::debug ? "true" : "rm $tmpfile";
my $cmd = "($unlinkcmd; cat -) < $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null"; my $cmd = "($unlinkcmd; cat -) < $tmpfile | ".
"$0 -j0 --timeout 10 -s 16000 --joblog - --plain --delay 0.1 --retries 3 ".
"--tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null";
::debug("init", $cmd, "\n"); ::debug("init", $cmd, "\n");
my @out; my @out;
my $prepend = ""; my $prepend = "";
@ -4703,15 +4709,7 @@ sub loadavg {
::die_bug("loadavg_invalid_content: " . ::die_bug("loadavg_invalid_content: " .
$self->{'loadavg_file'} . "\n$load_out"); $self->{'loadavg_file'} . "\n$load_out");
} }
# Because of instant load average, it should not be delayed 10 secs
# The instant load does not give 2 R if there is only 1 cpu.
# ::debug("load", "Last update: ", $self->{'last_loadavg_update'});
# if(time - $self->{'last_loadavg_update'} > 10) {
# # last loadavg was started 10 seconds ago
# ::debug("load", time - $self->{'last_loadavg_update'}, " secs old: ",
# $self->{'loadavg_file'});
$update_loadavg_file = 1; $update_loadavg_file = 1;
# }
} else { } else {
::debug("load", "No loadavg file: ", $self->{'loadavg_file'}); ::debug("load", "No loadavg file: ", $self->{'loadavg_file'});
$self->{'loadavg'} = undef; $self->{'loadavg'} = undef;

11
src/parallel_design.pod
View file

@ -35,9 +35,8 @@ sense to wait for a job to finish. Instead the load average must be
checked again. Load average is not the only reason: --timeout has a checked again. Load average is not the only reason: --timeout has a
similar problem. similar problem.
To not burn up too up too much CPU GNU B<parallel> sleeps To not burn up too much CPU GNU B<parallel> sleeps exponentially
exponentially longer and longer if nothing happens, maxing out at 1 longer and longer if nothing happens, maxing out at 1 second.
second.
=head2 Shell compatibility =head2 Shell compatibility
@ -59,7 +58,7 @@ job from the queue will be run in that slot. But originally GNU
B<parallel> did not model job slots in the code. Job slots have been B<parallel> did not model job slots in the code. Job slots have been
added to make it possible to use {%} as a replacement string. added to make it possible to use {%} as a replacement string.
Job slots were added to the code in 20140522, but while the job While the job
sequence number can be computed in advance, the job slot can only be sequence number can be computed in advance, the job slot can only be
computed the moment a slot becomes available. So it has been computed the moment a slot becomes available. So it has been
implemented as a stack with lazy evaluation: Draw one from an empty implemented as a stack with lazy evaluation: Draw one from an empty
@ -185,6 +184,10 @@ ssh I<sln> I<shell quoted input>
Read about B<--protocol 30> in the section B<Rsync protocol version>. Read about B<--protocol 30> in the section B<Rsync protocol version>.
=item --transferfile I<file>
<<todo>>
=item --basefile =item --basefile
<<todo>> <<todo>>