From 6e612565a990718bc716c8a4961b9a0e210010f0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 24 Aug 2014 17:19:17 +0200 Subject: [PATCH] Fixed bug #42983: include/exclude servers on-the-fly. Fixed ln -s to relative path. --- doc/release_new_version | 11 ++++- src/Makefile.am | 2 +- src/Makefile.in | 2 +- src/parallel | 39 +++++++++------- src/parallel.pod | 95 ++++++++++++++++++++------------------- src/parallel_tutorial.1 | 2 +- src/parallel_tutorial.pod | 10 ++++- 7 files changed, 93 insertions(+), 68 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 7143bf35..f0207950 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -2,7 +2,7 @@ == alpha/beta/production == -Update documentation: +Check that documentation is updated: git diff last-release-commit Unmodified beta since last version => production @@ -166,6 +166,14 @@ git tag -s -m "Released as $YYYYMMDD ('$TAG')" $YYYYMMDD git push + +== Update documentation == + +Update version number + 1 +Unmodified beta since last version => production +Unmodified alpha since last version => beta +Modified => alpha + == Update Savannah == file:///home/tange/privat/parallel/doc/release_new_version @@ -207,7 +215,6 @@ cc:Tim Cuthbertson , Ludovic Courtès , Markus Ammer , Pavel Nuzhdin , Phil Sung , Michael Shigorin , - Andrew McFague , Chris Howey , Fethican Coşkuner , Rogério Brito , diff --git a/src/Makefile.am b/src/Makefile.am index 52dcc7f2..f69604c9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,7 +2,7 @@ bin_SCRIPTS = parallel sql niceload install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true - $(LN_S) $(DESTDIR)$(bindir)/parallel $(DESTDIR)$(bindir)/sem + $(LN_S) parallel $(DESTDIR)$(bindir)/sem if DOCUMENTATION man_MANS = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.1 diff --git a/src/Makefile.in b/src/Makefile.in index dc45a1e7..ba2715f3 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -525,7 +525,7 @@ uninstall-man: uninstall-man1 install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true - $(LN_S) $(DESTDIR)$(bindir)/parallel $(DESTDIR)$(bindir)/sem + $(LN_S) parallel $(DESTDIR)$(bindir)/sem # Build documentation file if the tool to build exists. # Otherwise: Use the distributed version diff --git a/src/parallel b/src/parallel index b16534b2..62da04bb 100755 --- a/src/parallel +++ b/src/parallel @@ -721,7 +721,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20140822; + $Global::version = 20140823; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1647,19 +1647,19 @@ sub start_more_jobs { if($mtime - $last_mtime{$actual_file} > 1) { ::debug("run","--sshloginfile $actual_file changed. reload\n"); $last_mtime{$actual_file} = $mtime; - # TODO reload $slf - my %sshlogin_before = %Global::host; + # Reload $slf + # Empty sshlogins + @Global::sshlogin = (); + for (values %Global::host) { + # Don't start new jobs on any host + # except the ones added back later + $_->set_max_jobs_running(0); + } + # This will set max_jobs_running on the SSHlogins read_sshloginfile($actual_file); parse_sshlogin(); $opt::filter_hosts and filter_hosts(); setup_basefile(); - my @added = grep { not $sshlogin_before{$_} } keys %Global::host; -# my @removed = grep { not $sshlogin_before{$_} } keys sshlogin_before %Global::host; -#wrong my @back = grep { $Global::host{$_}->jobslots() == 0 } - keys %Global::host; -# map { $Global::host{$_}->reset_jobslots() } @back; - - } } } @@ -2188,6 +2188,7 @@ sub parse_sshlogin { # Split up -S sshlogin,sshlogin for my $s (split /,/, $sshlogin) { if ($s eq ".." or $s eq "-") { + # This may add to @Global::sshlogin - possibly bug read_sshloginfile(expand_slf_shorthand($s)); } else { push (@login, $s); @@ -2195,9 +2196,13 @@ sub parse_sshlogin { } } $Global::minimal_command_line_length = 8_000_000; - for my $sshlogin_string (@login) { + for my $sshlogin_string (::uniq(@login)) { if($Global::host{$sshlogin_string}) { - # This sshlogin has already been added: next + # This sshlogin has already been added: + # It is probably a host that has come back + # Set the max_jobs_running back to the original + $Global::host{$sshlogin_string}->set_max_jobs_running( + $Global::host{$sshlogin_string}->orig_max_jobs_running()); next; } my $sshlogin = SSHLogin->new($sshlogin_string); @@ -3213,8 +3218,6 @@ sub new { 'swap_activity_file' => $ENV{'HOME'} . "/.parallel/tmp/swap_activity-" . $no_slash_string, 'swap_activity' => undef, - # What time was this object created? - 'added' => time, }, ref($class) || $class; } @@ -3276,10 +3279,11 @@ sub set_max_jobs_running { # max_jobs_running could be resat if -j is a changed file $Global::max_jobs_running += $self->{'max_jobs_running'}; } - # Initialize orig to the first value that comes around + # Initialize orig to the first non-zero value that comes around $self->{'orig_max_jobs_running'} ||= $self->{'max_jobs_running'}; } + sub swapping { my $self = shift; my $swapping = $self->swap_activity(); @@ -3646,6 +3650,11 @@ sub max_jobs_running { return $self->{'max_jobs_running'}; } +sub orig_max_jobs_running { + my $self = shift; + return $self->{'orig_max_jobs_running'}; +} + sub compute_number_of_processes { # Number of processes wanted and limited by system resources # Returns: diff --git a/src/parallel.pod b/src/parallel.pod index 07236dd1..d3e24c41 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -97,7 +97,7 @@ B to export and to set $SHELL to bash: The command cannot contain the character \257 (macron: ¯). -=item B<{}> (alpha testing) +=item B<{}> (beta testing) Input line. This replacement string will be replaced by a full line read from the input source. The input source is normally stdin @@ -110,7 +110,7 @@ If the command line contains no replacement strings then B<{}> will be appended to the command line. -=item B<{.}> (alpha testing) +=item B<{.}> (beta testing) Input line without extension. This replacement string will be replaced by the input with the extension removed. If the input line contains @@ -126,7 +126,7 @@ The replacement string B<{.}> can be changed with B<--er>. To understand replacement strings see B<{}>. -=item B<{/}> (alpha testing) +=item B<{/}> (beta testing) Basename of input line. This replacement string will be replaced by the input with the directory part removed. @@ -137,7 +137,7 @@ B<--basenamereplace>. To understand replacement strings see B<{}>. -=item B<{//}> (alpha testing) +=item B<{//}> (beta testing) Dirname of input line. This replacement string will be replaced by the dir of the input line. See B(1). @@ -148,7 +148,7 @@ B<--dirnamereplace>. To understand replacement strings see B<{}>. -=item B<{/.}> (alpha testing) +=item B<{/.}> (beta testing) Basename of input line without extension. This replacement string will be replaced by the input with the directory and extension part @@ -160,7 +160,7 @@ B<--basenameextensionreplace>. To understand replacement strings see B<{}>. -=item B<{#}> (alpha testing) +=item B<{#}> (beta testing) Sequence number of the job to run. This replacement string will be replaced by the sequence number of the job being run. It contains the @@ -171,7 +171,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>. To understand replacement strings see B<{}>. -=item B<{%}> (alpha testing) +=item B<{%}> (beta testing) Job slot number. This replacement string will be replaced by the job's slot number between 1 and number of jobs to run in parallel. There @@ -183,7 +183,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>. To understand replacement strings see B<{}>. -=item B<{>IB<}> (alpha testing) +=item B<{>IB<}> (beta testing) Argument from input source I or the I'th argument. This positional replacement string will be replaced by the input from input @@ -194,7 +194,7 @@ I'th last argument. To understand replacement strings see B<{}>. -=item B<{>I.B<}> (alpha testing) +=item B<{>I.B<}> (beta testing) Argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}> and B<{.}>. @@ -207,7 +207,7 @@ extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/B<}> (alpha testing) +=item B<{>I/B<}> (beta testing) Basename of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{/}>. @@ -220,7 +220,7 @@ directory (if any) removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I//B<}> (alpha testing) +=item B<{>I//B<}> (beta testing) Dirname of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{//}>. @@ -232,7 +232,7 @@ the I'th argument (when used with B<-N>). See B(1). To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/.B<}> (alpha testing) +=item B<{>I/.B<}> (beta testing) Basename of argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}>, B<{/}>, and @@ -246,7 +246,7 @@ directory (if any) and extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{=>IB<=}> (alpha testing) +=item B<{=>IB<=}> (beta testing) Replace with calculated I. B<$_> will contain the same as B<{}>. After evaluating I B<$_> will be used @@ -259,7 +259,7 @@ The B<{=>IB<=}> must be given as a single string. See also: B<--rpl> B<--parens> -=item B<{=>I IB<=}> (alpha testing) +=item B<{=>I IB<=}> (beta testing) Positional equivalent to B<{= perl expression =}>. To understand positional replacement strings see B<{>IB<}>. @@ -321,9 +321,9 @@ Use NUL as delimiter. Normally input lines will end in \n for processing arguments that may contain \n (newline). -=item B<--arg-file> I (beta testing) +=item B<--arg-file> I -=item B<-a> I (beta testing) +=item B<-a> I Use I as input source. If you use this option, stdin (standard input) is given to the first process run. Otherwise, stdin @@ -360,7 +360,7 @@ string that is not in the command line. See also: B<:::>. -=item B<--bar> (alpha testing) +=item B<--bar> (beta testing) Show progress as a progress bar. In the bar is shown: % of jobs completed, estimated seconds left, and number of jobs started. @@ -370,9 +370,9 @@ It is compatible with B: seq 1000 | parallel -j30 --bar '(echo {};sleep 0.1)' 2> >(zenity --progress --auto-kill) | wc -=item B<--basefile> I (alpha testing) +=item B<--basefile> I (beta testing) -=item B<--bf> I (alpha testing) +=item B<--bf> I (beta testing) I will be transferred to each sshlogin before a jobs is started. It will be removed if B<--cleanup> is active. The file may be @@ -413,9 +413,9 @@ Print the BibTeX entry for GNU B and disable citation notice. -=item B<--block> I (beta testing) +=item B<--block> I -=item B<--block-size> I (beta testing) +=item B<--block-size> I Size of block in bytes. The size can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply the size with 1024, 1048576, @@ -523,7 +523,7 @@ Use the replacement string I instead of B<{//}> for dirname of input line. -=item B<-E> I (alpha testing) +=item B<-E> I (beta testing) Set the end of file string to eof-str. If the end of file string occurs as a line of input, the rest of the input is ignored. If @@ -546,9 +546,9 @@ the job may be scheduled on another computer or the local computer if : is in the list. -=item B<--eof>[=I] (alpha testing) +=item B<--eof>[=I] (beta testing) -=item B<-e>[I] (alpha testing) +=item B<-e>[I] (beta testing) This option is a synonym for the B<-E> option. Use B<-E> instead, because it is POSIX compliant for B while this option is not. @@ -556,7 +556,7 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I (alpha testing) +=item B<--env> I (beta testing) Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for @@ -604,7 +604,7 @@ To copy the full environment use this function: See also: B<--record-env>. -=item B<--eta> (beta testing) +=item B<--eta> Show the estimated number of seconds before finishing. This forces GNU B to read all jobs before starting to find the number of @@ -674,9 +674,9 @@ See also: B<--line-buffer> B<--ungroup> Print a summary of the options to GNU B and exit. -=item B<--halt-on-error> I (alpha testing) +=item B<--halt-on-error> I (beta testing) -=item B<--halt> I (alpha testing) +=item B<--halt> I (beta testing) How should GNU B terminate if one of more jobs fail? @@ -726,9 +726,9 @@ If I is a number, it will match that many lines. Use the replacement string I instead of {}. -=item B<--replace>[=I] (alpha testing) +=item B<--replace>[=I] (beta testing) -=item B<-i>[I] (alpha testing) +=item B<-i>[I] (beta testing) This option is a synonym for B<-I>I if I is specified, and for B<-I>{} otherwise. This option is deprecated; @@ -789,13 +789,13 @@ If the evaluated number is less than 1 then 1 will be used. See also B<--use-cpus-instead-of-cores>. -=item B<--jobs> I% (alpha testing) +=item B<--jobs> I% (beta testing) -=item B<-j> I% (alpha testing) +=item B<-j> I% (beta testing) -=item B<--max-procs> I% (alpha testing) +=item B<--max-procs> I% (beta testing) -=item B<-P> I% (alpha testing) +=item B<-P> I% (beta testing) Multiply N% with the number of CPU cores. Run this many jobs in parallel. See also B<--use-cpus-instead-of-cores>. @@ -887,9 +887,9 @@ the same syntax as B<--jobs>, so I<100%> for one per CPU is a valid setting. Only difference is 0 which is interpreted as 0.01. -=item B<--controlmaster> (alpha testing) +=item B<--controlmaster> (beta testing) -=item B<-M> (alpha testing) +=item B<-M> (beta testing) Use ssh's ControlMaster to make ssh connections faster. Useful if jobs run remote and are very fast to run. This is disabled for sshlogins @@ -999,7 +999,7 @@ B<--files> is often used with B<--pipe>. See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>. -=item B<--pipepart> (beta testing) +=item B<--pipepart> Pipe parts of a physical file. B<--pipepart> works similar to B<--pipe>, but is much faster. It has a few limitations: @@ -1024,7 +1024,7 @@ control on the command line (used by GNU B internally when called with B<--sshlogin>). -=item B<--plus> (alpha testing) +=item B<--plus> (beta testing) Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} {...} {/..} {/...}. The idea being that '{+foo}' matches the opposite of @@ -1145,9 +1145,9 @@ B<,,>: See also: B<--rpl> B<{= perl expression =}> -=item B<--profile> I (alpha testing) +=item B<--profile> I (beta testing) -=item B<-J> I (alpha testing) +=item B<-J> I (beta testing) Use profile I for options. This is useful if you want to have multiple profiles. You could have one profile for running jobs in @@ -1320,7 +1320,7 @@ useful if some jobs fail for no apparent reason (such as network failure). -=item B<--return> I (alpha testing) +=item B<--return> I (beta testing) Transfer files from remote computers. B<--return> is used with B<--sshlogin> when the arguments are files on the remote computers. When @@ -1368,7 +1368,7 @@ B<--keep-order> will not work with B<--round-robin> as it is impossible to track which input block corresponds to which output. -=item B<--rpl> 'I I' (alpha testing) +=item B<--rpl> 'I I' (beta testing) Use I as a replacement string for I. This makes it possible to define your own replacement strings. GNU B's @@ -1549,9 +1549,9 @@ I seconds after starting each ssh. I can be less than 1 seconds. -=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (alpha testing) +=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (beta testing) -=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (alpha testing) +=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (beta testing) Distribute jobs to remote computers. The jobs will be run on a list of remote computers. GNU B will determine the number of CPU @@ -1626,6 +1626,9 @@ The sshloginfile '.' is special, it read sshlogins from The sshloginfile '-' is special, too, it read sshlogins from stdin (standard input). +If the sshloginfile is changed it will be re-read when a job finishes +though at most once per second. + =item B<--slotreplace> I @@ -1674,7 +1677,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--tmux> (beta testing) +=item B<--tmux> Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. @@ -1700,7 +1703,7 @@ Print the job to be run on stderr (standard error). See also B<-v>, B<-p>. -=item B<--transfer> (alpha testing) +=item B<--transfer> (beta testing) Transfer files to remote computers. B<--transfer> is used with B<--sshlogin> when the arguments are files and should be transferred diff --git a/src/parallel_tutorial.1 b/src/parallel_tutorial.1 index 5eee0547..48434205 100644 --- a/src/parallel_tutorial.1 +++ b/src/parallel_tutorial.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL_TUTORIAL 1" -.TH PARALLEL_TUTORIAL 1 "2014-07-26" "20140815" "parallel" +.TH PARALLEL_TUTORIAL 1 "2014-08-23" "20140822" "parallel" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 3fe35a03..ac87b0da 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -2118,6 +2118,11 @@ If you like GNU Parallel: =item * +(Re-)walk through the tutorial if you have not done so in the past year +(http://www.gnu.org/software/parallel/parallel_tutorial.html) + +=item * + Give a demo at your local user group/team/colleagues =item * @@ -2152,11 +2157,12 @@ If GNU Parallel saves you money: =item * -(Have your company) donate to FSF https://my.fsf.org/donate/ +(Have your company) donate to FSF or become a member +https://my.fsf.org/donate/ =back -(C) 20140622 Ole Tange GPLv3 +(C) 2013,2014 Ole Tange, GPLv3 =cut