Fixed bug #42983: include/exclude servers on-the-fly.

Fixed ln -s to relative path.
This commit is contained in:
Ole Tange 2014-08-24 17:19:17 +02:00
parent 82550070ee
commit 6e612565a9
7 changed files with 93 additions and 68 deletions

View file

@ -2,7 +2,7 @@
== alpha/beta/production ==
Update documentation:
Check that documentation is updated:
git diff last-release-commit
Unmodified beta since last version => production
@ -166,6 +166,14 @@ git tag -s -m "Released as $YYYYMMDD ('$TAG')" $YYYYMMDD
git push
== Update documentation ==
Update version number + 1
Unmodified beta since last version => production
Unmodified alpha since last version => beta
Modified => alpha
== Update Savannah ==
file:///home/tange/privat/parallel/doc/release_new_version
@ -207,7 +215,6 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Ludovic Courtès <ludo@gnu.org>, Markus Ammer <mkmm@gmx-topmail.de>,
Pavel Nuzhdin <pnzhdin@gmail.com>, Phil Sung <psung@alum.mit.edu>,
Michael Shigorin <mike@altlinux.org>,
Andrew McFague <amcfague@wgen.net>,
Chris Howey <howeyc@gmail.com>,
Fethican Coşkuner <fethicanc@gmail.com>,
Rogério Brito <rbrito@ime.usp.br>,

View file

@ -2,7 +2,7 @@ bin_SCRIPTS = parallel sql niceload
install-exec-hook:
rm $(DESTDIR)$(bindir)/sem || true
$(LN_S) $(DESTDIR)$(bindir)/parallel $(DESTDIR)$(bindir)/sem
$(LN_S) parallel $(DESTDIR)$(bindir)/sem
if DOCUMENTATION
man_MANS = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.1

View file

@ -525,7 +525,7 @@ uninstall-man: uninstall-man1
install-exec-hook:
rm $(DESTDIR)$(bindir)/sem || true
$(LN_S) $(DESTDIR)$(bindir)/parallel $(DESTDIR)$(bindir)/sem
$(LN_S) parallel $(DESTDIR)$(bindir)/sem
# Build documentation file if the tool to build exists.
# Otherwise: Use the distributed version

View file

@ -721,7 +721,7 @@ sub get_options_from_array {
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20140822;
$Global::version = 20140823;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -1647,19 +1647,19 @@ sub start_more_jobs {
if($mtime - $last_mtime{$actual_file} > 1) {
::debug("run","--sshloginfile $actual_file changed. reload\n");
$last_mtime{$actual_file} = $mtime;
# TODO reload $slf
my %sshlogin_before = %Global::host;
# Reload $slf
# Empty sshlogins
@Global::sshlogin = ();
for (values %Global::host) {
# Don't start new jobs on any host
# except the ones added back later
$_->set_max_jobs_running(0);
}
# This will set max_jobs_running on the SSHlogins
read_sshloginfile($actual_file);
parse_sshlogin();
$opt::filter_hosts and filter_hosts();
setup_basefile();
my @added = grep { not $sshlogin_before{$_} } keys %Global::host;
# my @removed = grep { not $sshlogin_before{$_} } keys sshlogin_before %Global::host;
#wrong my @back = grep { $Global::host{$_}->jobslots() == 0 }
keys %Global::host;
# map { $Global::host{$_}->reset_jobslots() } @back;
}
}
}
@ -2188,6 +2188,7 @@ sub parse_sshlogin {
# Split up -S sshlogin,sshlogin
for my $s (split /,/, $sshlogin) {
if ($s eq ".." or $s eq "-") {
# This may add to @Global::sshlogin - possibly bug
read_sshloginfile(expand_slf_shorthand($s));
} else {
push (@login, $s);
@ -2195,9 +2196,13 @@ sub parse_sshlogin {
}
}
$Global::minimal_command_line_length = 8_000_000;
for my $sshlogin_string (@login) {
for my $sshlogin_string (::uniq(@login)) {
if($Global::host{$sshlogin_string}) {
# This sshlogin has already been added: next
# This sshlogin has already been added:
# It is probably a host that has come back
# Set the max_jobs_running back to the original
$Global::host{$sshlogin_string}->set_max_jobs_running(
$Global::host{$sshlogin_string}->orig_max_jobs_running());
next;
}
my $sshlogin = SSHLogin->new($sshlogin_string);
@ -3213,8 +3218,6 @@ sub new {
'swap_activity_file' => $ENV{'HOME'} . "/.parallel/tmp/swap_activity-" .
$no_slash_string,
'swap_activity' => undef,
# What time was this object created?
'added' => time,
}, ref($class) || $class;
}
@ -3276,10 +3279,11 @@ sub set_max_jobs_running {
# max_jobs_running could be resat if -j is a changed file
$Global::max_jobs_running += $self->{'max_jobs_running'};
}
# Initialize orig to the first value that comes around
# Initialize orig to the first non-zero value that comes around
$self->{'orig_max_jobs_running'} ||= $self->{'max_jobs_running'};
}
sub swapping {
my $self = shift;
my $swapping = $self->swap_activity();
@ -3646,6 +3650,11 @@ sub max_jobs_running {
return $self->{'max_jobs_running'};
}
sub orig_max_jobs_running {
my $self = shift;
return $self->{'orig_max_jobs_running'};
}
sub compute_number_of_processes {
# Number of processes wanted and limited by system resources
# Returns:

View file

@ -97,7 +97,7 @@ B<exportf> to export and to set $SHELL to bash:
The command cannot contain the character \257 (macron: ¯).
=item B<{}> (alpha testing)
=item B<{}> (beta testing)
Input line. This replacement string will be replaced by a full line
read from the input source. The input source is normally stdin
@ -110,7 +110,7 @@ If the command line contains no replacement strings then B<{}> will be
appended to the command line.
=item B<{.}> (alpha testing)
=item B<{.}> (beta testing)
Input line without extension. This replacement string will be replaced
by the input with the extension removed. If the input line contains
@ -126,7 +126,7 @@ The replacement string B<{.}> can be changed with B<--er>.
To understand replacement strings see B<{}>.
=item B<{/}> (alpha testing)
=item B<{/}> (beta testing)
Basename of input line. This replacement string will be replaced by
the input with the directory part removed.
@ -137,7 +137,7 @@ B<--basenamereplace>.
To understand replacement strings see B<{}>.
=item B<{//}> (alpha testing)
=item B<{//}> (beta testing)
Dirname of input line. This replacement string will be replaced by the
dir of the input line. See B<dirname>(1).
@ -148,7 +148,7 @@ B<--dirnamereplace>.
To understand replacement strings see B<{}>.
=item B<{/.}> (alpha testing)
=item B<{/.}> (beta testing)
Basename of input line without extension. This replacement string will
be replaced by the input with the directory and extension part
@ -160,7 +160,7 @@ B<--basenameextensionreplace>.
To understand replacement strings see B<{}>.
=item B<{#}> (alpha testing)
=item B<{#}> (beta testing)
Sequence number of the job to run. This replacement string will be
replaced by the sequence number of the job being run. It contains the
@ -171,7 +171,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>.
To understand replacement strings see B<{}>.
=item B<{%}> (alpha testing)
=item B<{%}> (beta testing)
Job slot number. This replacement string will be replaced by the job's
slot number between 1 and number of jobs to run in parallel. There
@ -183,7 +183,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>.
To understand replacement strings see B<{}>.
=item B<{>I<n>B<}> (alpha testing)
=item B<{>I<n>B<}> (beta testing)
Argument from input source I<n> or the I<n>'th argument. This
positional replacement string will be replaced by the input from input
@ -194,7 +194,7 @@ I<n>'th last argument.
To understand replacement strings see B<{}>.
=item B<{>I<n>.B<}> (alpha testing)
=item B<{>I<n>.B<}> (beta testing)
Argument from input source I<n> or the I<n>'th argument without
extension. It is a combination of B<{>I<n>B<}> and B<{.}>.
@ -207,7 +207,7 @@ extension removed.
To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{>I<n>/B<}> (alpha testing)
=item B<{>I<n>/B<}> (beta testing)
Basename of argument from input source I<n> or the I<n>'th argument.
It is a combination of B<{>I<n>B<}> and B<{/}>.
@ -220,7 +220,7 @@ directory (if any) removed.
To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{>I<n>//B<}> (alpha testing)
=item B<{>I<n>//B<}> (beta testing)
Dirname of argument from input source I<n> or the I<n>'th argument.
It is a combination of B<{>I<n>B<}> and B<{//}>.
@ -232,7 +232,7 @@ the I<n>'th argument (when used with B<-N>). See B<dirname>(1).
To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{>I<n>/.B<}> (alpha testing)
=item B<{>I<n>/.B<}> (beta testing)
Basename of argument from input source I<n> or the I<n>'th argument
without extension. It is a combination of B<{>I<n>B<}>, B<{/}>, and
@ -246,7 +246,7 @@ directory (if any) and extension removed.
To understand positional replacement strings see B<{>I<n>B<}>.
=item B<{=>I<perl expression>B<=}> (alpha testing)
=item B<{=>I<perl expression>B<=}> (beta testing)
Replace with calculated I<perl expression>. B<$_> will contain the
same as B<{}>. After evaluating I<perl expression> B<$_> will be used
@ -259,7 +259,7 @@ The B<{=>I<perl expression>B<=}> must be given as a single string.
See also: B<--rpl> B<--parens>
=item B<{=>I<n> I<perl expression>B<=}> (alpha testing)
=item B<{=>I<n> I<perl expression>B<=}> (beta testing)
Positional equivalent to B<{= perl expression =}>. To understand
positional replacement strings see B<{>I<n>B<}>.
@ -321,9 +321,9 @@ Use NUL as delimiter. Normally input lines will end in \n
for processing arguments that may contain \n (newline).
=item B<--arg-file> I<input-file> (beta testing)
=item B<--arg-file> I<input-file>
=item B<-a> I<input-file> (beta testing)
=item B<-a> I<input-file>
Use I<input-file> as input source. If you use this option, stdin
(standard input) is given to the first process run. Otherwise, stdin
@ -360,7 +360,7 @@ string that is not in the command line.
See also: B<:::>.
=item B<--bar> (alpha testing)
=item B<--bar> (beta testing)
Show progress as a progress bar. In the bar is shown: % of jobs
completed, estimated seconds left, and number of jobs started.
@ -370,9 +370,9 @@ It is compatible with B<zenity>:
seq 1000 | parallel -j30 --bar '(echo {};sleep 0.1)' 2> >(zenity --progress --auto-kill) | wc
=item B<--basefile> I<file> (alpha testing)
=item B<--basefile> I<file> (beta testing)
=item B<--bf> I<file> (alpha testing)
=item B<--bf> I<file> (beta testing)
I<file> will be transferred to each sshlogin before a jobs is
started. It will be removed if B<--cleanup> is active. The file may be
@ -413,9 +413,9 @@ Print the BibTeX entry for GNU B<parallel> and disable citation
notice.
=item B<--block> I<size> (beta testing)
=item B<--block> I<size>
=item B<--block-size> I<size> (beta testing)
=item B<--block-size> I<size>
Size of block in bytes. The size can be postfixed with K, M, G, T, P,
k, m, g, t, or p which would multiply the size with 1024, 1048576,
@ -523,7 +523,7 @@ Use the replacement string I<replace-str> instead of B<{//}> for
dirname of input line.
=item B<-E> I<eof-str> (alpha testing)
=item B<-E> I<eof-str> (beta testing)
Set the end of file string to eof-str. If the end of file string
occurs as a line of input, the rest of the input is ignored. If
@ -546,9 +546,9 @@ the job may be scheduled on another computer or the local computer if
: is in the list.
=item B<--eof>[=I<eof-str>] (alpha testing)
=item B<--eof>[=I<eof-str>] (beta testing)
=item B<-e>[I<eof-str>] (alpha testing)
=item B<-e>[I<eof-str>] (beta testing)
This option is a synonym for the B<-E> option. Use B<-E> instead,
because it is POSIX compliant for B<xargs> while this option is not.
@ -556,7 +556,7 @@ If I<eof-str> is omitted, there is no end of file string. If neither
B<-E> nor B<-e> is used, no end of file string is used.
=item B<--env> I<var> (alpha testing)
=item B<--env> I<var> (beta testing)
Copy environment variable I<var>. This will copy I<var> to the
environment that the command is run in. This is especially useful for
@ -604,7 +604,7 @@ To copy the full environment use this function:
See also: B<--record-env>.
=item B<--eta> (beta testing)
=item B<--eta>
Show the estimated number of seconds before finishing. This forces GNU
B<parallel> to read all jobs before starting to find the number of
@ -674,9 +674,9 @@ See also: B<--line-buffer> B<--ungroup>
Print a summary of the options to GNU B<parallel> and exit.
=item B<--halt-on-error> I<val> (alpha testing)
=item B<--halt-on-error> I<val> (beta testing)
=item B<--halt> I<val> (alpha testing)
=item B<--halt> I<val> (beta testing)
How should GNU B<parallel> terminate if one of more jobs fail?
@ -726,9 +726,9 @@ If I<regexp> is a number, it will match that many lines.
Use the replacement string I<replace-str> instead of {}.
=item B<--replace>[=I<replace-str>] (alpha testing)
=item B<--replace>[=I<replace-str>] (beta testing)
=item B<-i>[I<replace-str>] (alpha testing)
=item B<-i>[I<replace-str>] (beta testing)
This option is a synonym for B<-I>I<replace-str> if I<replace-str> is
specified, and for B<-I>{} otherwise. This option is deprecated;
@ -789,13 +789,13 @@ If the evaluated number is less than 1 then 1 will be used. See also
B<--use-cpus-instead-of-cores>.
=item B<--jobs> I<N>% (alpha testing)
=item B<--jobs> I<N>% (beta testing)
=item B<-j> I<N>% (alpha testing)
=item B<-j> I<N>% (beta testing)
=item B<--max-procs> I<N>% (alpha testing)
=item B<--max-procs> I<N>% (beta testing)
=item B<-P> I<N>% (alpha testing)
=item B<-P> I<N>% (beta testing)
Multiply N% with the number of CPU cores. Run this many jobs in
parallel. See also B<--use-cpus-instead-of-cores>.
@ -887,9 +887,9 @@ the same syntax as B<--jobs>, so I<100%> for one per CPU is a valid
setting. Only difference is 0 which is interpreted as 0.01.
=item B<--controlmaster> (alpha testing)
=item B<--controlmaster> (beta testing)
=item B<-M> (alpha testing)
=item B<-M> (beta testing)
Use ssh's ControlMaster to make ssh connections faster. Useful if jobs
run remote and are very fast to run. This is disabled for sshlogins
@ -999,7 +999,7 @@ B<--files> is often used with B<--pipe>.
See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>.
=item B<--pipepart> (beta testing)
=item B<--pipepart>
Pipe parts of a physical file. B<--pipepart> works similar to
B<--pipe>, but is much faster. It has a few limitations:
@ -1024,7 +1024,7 @@ control on the command line (used by GNU B<parallel> internally when
called with B<--sshlogin>).
=item B<--plus> (alpha testing)
=item B<--plus> (beta testing)
Activate additional replacement strings: {+/} {+.} {+..} {+...} {..}
{...} {/..} {/...}. The idea being that '{+foo}' matches the opposite of
@ -1145,9 +1145,9 @@ B<,,>:
See also: B<--rpl> B<{= perl expression =}>
=item B<--profile> I<profilename> (alpha testing)
=item B<--profile> I<profilename> (beta testing)
=item B<-J> I<profilename> (alpha testing)
=item B<-J> I<profilename> (beta testing)
Use profile I<profilename> for options. This is useful if you want to
have multiple profiles. You could have one profile for running jobs in
@ -1320,7 +1320,7 @@ useful if some jobs fail for no apparent reason (such as network
failure).
=item B<--return> I<filename> (alpha testing)
=item B<--return> I<filename> (beta testing)
Transfer files from remote computers. B<--return> is used with
B<--sshlogin> when the arguments are files on the remote computers. When
@ -1368,7 +1368,7 @@ B<--keep-order> will not work with B<--round-robin> as it is
impossible to track which input block corresponds to which output.
=item B<--rpl> 'I<tag> I<perl expression>' (alpha testing)
=item B<--rpl> 'I<tag> I<perl expression>' (beta testing)
Use I<tag> as a replacement string for I<perl expression>. This makes
it possible to define your own replacement strings. GNU B<parallel>'s
@ -1549,9 +1549,9 @@ I<secs> seconds after starting each ssh. I<secs> can be less than 1
seconds.
=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (alpha testing)
=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (beta testing)
=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (alpha testing)
=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (beta testing)
Distribute jobs to remote computers. The jobs will be run on a list of
remote computers. GNU B<parallel> will determine the number of CPU
@ -1626,6 +1626,9 @@ The sshloginfile '.' is special, it read sshlogins from
The sshloginfile '-' is special, too, it read sshlogins from stdin
(standard input).
If the sshloginfile is changed it will be re-read when a job finishes
though at most once per second.
=item B<--slotreplace> I<replace-str>
@ -1674,7 +1677,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
setting $TMPDIR.
=item B<--tmux> (beta testing)
=item B<--tmux>
Use B<tmux> for output. Start a B<tmux> session and run each job in a
window in that session. No other output will be produced.
@ -1700,7 +1703,7 @@ Print the job to be run on stderr (standard error).
See also B<-v>, B<-p>.
=item B<--transfer> (alpha testing)
=item B<--transfer> (beta testing)
Transfer files to remote computers. B<--transfer> is used with
B<--sshlogin> when the arguments are files and should be transferred

View file

@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "PARALLEL_TUTORIAL 1"
.TH PARALLEL_TUTORIAL 1 "2014-07-26" "20140815" "parallel"
.TH PARALLEL_TUTORIAL 1 "2014-08-23" "20140822" "parallel"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l

View file

@ -2118,6 +2118,11 @@ If you like GNU Parallel:
=item *
(Re-)walk through the tutorial if you have not done so in the past year
(http://www.gnu.org/software/parallel/parallel_tutorial.html)
=item *
Give a demo at your local user group/team/colleagues
=item *
@ -2152,11 +2157,12 @@ If GNU Parallel saves you money:
=item *
(Have your company) donate to FSF https://my.fsf.org/donate/
(Have your company) donate to FSF or become a member
https://my.fsf.org/donate/
=back
(C) 20140622 Ole Tange GPLv3
(C) 2013,2014 Ole Tange, GPLv3
=cut