tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 22:17:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #50719: Adjust estimate time upwards less frequently.

Browse source 
parallel: Introduced $PARALLEL_HOME instead of ~/.parallel.
Makefile: Added monitorman.
This commit is contained in:
Ole Tange 2017-04-04 23:17:29 +02:00
parent 70e29c7b5b
commit 685018f532
6 changed files with 60 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Makefile.am
View file

@ -115,4 +115,8 @@ alphaupload:
pushd; \ pushd; \
sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD)
monitorman:
# If man page changed: open new pdfman
inotifywait -qmre MOVED_TO -e CLOSE_WRITE --format %w%f . | parallel -uj1 'echo {=/\.pod$$/ or skip()=};make -j && sudo make install; pdfman {/.} &'
EXTRA_DIST = CITATION CREDITS EXTRA_DIST = CITATION CREDITS

4
Makefile.in
View file

@ -863,6 +863,10 @@ alphaupload:
pushd; \ pushd; \
sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD)
monitorman:
# If man page changed: open new pdfman
inotifywait -qmre MOVED_TO -e CLOSE_WRITE --format %w%f . | parallel -uj1 'echo {=/\.pod$$/ or skip()=};make -j && sudo make install; pdfman {/.} &'
# Tell versions [3.59,3.63) of GNU make to not export all variables. # Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded. # Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT: .NOEXPORT:

8
doc/release_new_version
View file

@ -193,9 +193,9 @@ file:///home/tange/privat/parallel/doc/release_new_version
from:tange@gnu.org from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
Subject: GNU Parallel 20170422 ('') released <<[stable]>> Subject: GNU Parallel 20170422 ('Санкт-Петербу́рг') released <<[stable]>>
GNU Parallel 20170422 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20170422 ('Санкт-Петербу́рг') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -215,9 +215,11 @@ http://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-017-1593-0
https://128.84.21.199/pdf/1703.09026.pdf https://128.84.21.199/pdf/1703.09026.pdf
https://wiki.ncsa.illinois.edu/display/ROGER/Using+GNU+Parallel
https://github.com/lucascbeyeler/zmbackup https://github.com/lucascbeyeler/zmbackup
http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0174575#references http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0174575
* <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>> * <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>

75
src/parallel
View file

@ -1438,6 +1438,7 @@ sub init_globals {
::warning("\$HOME not set. Using /tmp."); ::warning("\$HOME not set. Using /tmp.");
$ENV{HOME} = "/tmp"; $ENV{HOME} = "/tmp";
} }
$ENV{'PARALLEL_HOME'} ||= $ENV{'HOME'} . "/.parallel";
} }
sub parse_halt { sub parse_halt {
@ -1607,9 +1608,9 @@ sub parse_semaphore {
} }
sub record_env { sub record_env {
# Record current %ENV-keys in ~/.parallel/ignored_vars # Record current %ENV-keys in $PARALLEL_HOME/ignored_vars
# Returns: N/A # Returns: N/A
my $ignore_filename = $ENV{'HOME'} . "/.parallel/ignored_vars"; my $ignore_filename = $ENV{'PARALLEL_HOME'} . "/ignored_vars";
if(open(my $vars_fh, ">", $ignore_filename)) { if(open(my $vars_fh, ">", $ignore_filename)) {
print $vars_fh map { $_,"\n" } keys %ENV; print $vars_fh map { $_,"\n" } keys %ENV;
} else { } else {
@ -1857,10 +1858,10 @@ sub read_options {
my @ARGV_profile = (); my @ARGV_profile = ();
my @ARGV_env = (); my @ARGV_env = ();
if(not $opt::plain) { if(not $opt::plain) {
# Add options from .parallel/config and other profiles # Add options from $PARALLEL_HOME/config and other profiles
my @config_profiles = ( my @config_profiles = (
"/etc/parallel/config", "/etc/parallel/config",
$ENV{'HOME'}."/.parallel/config", $ENV{'PARALLEL_HOME'}."/config",
$ENV{'HOME'}."/.parallelrc"); $ENV{'HOME'}."/.parallelrc");
my @profiles = @config_profiles; my @profiles = @config_profiles;
if(@opt::profile) { if(@opt::profile) {
@ -1870,7 +1871,7 @@ sub read_options {
if(-r $profile) { if(-r $profile) {
push @profiles, $profile; push @profiles, $profile;
} else { } else {
push @profiles, $ENV{'HOME'}."/.parallel/".$profile; push @profiles, $ENV{'PARALLEL_HOME'}."/".$profile;
} }
} }
} }
@ -2846,7 +2847,8 @@ sub progress {
} }
{ {
my ($total, $first_completed, $smoothed_avg_time);
my ($total, $first_completed, $smoothed_avg_time, $last_eta);
sub compute_eta { sub compute_eta {
# Calculate important numbers for ETA # Calculate important numbers for ETA
@ -2872,6 +2874,12 @@ sub progress {
$smoothed_avg_time = (1 - $pctcomplete) * $smoothed_avg_time + $smoothed_avg_time = (1 - $pctcomplete) * $smoothed_avg_time +
$pctcomplete * $avgtime; $pctcomplete * $avgtime;
my $eta = int($left * $smoothed_avg_time); my $eta = int($left * $smoothed_avg_time);
if($eta*0.90 < $last_eta and $last_eta < $eta) {
# Eta jumped less that 10% up: Keep the last eta instead
$eta = $last_eta;
} else {
$last_eta = $eta;
}
return($total, $completed, $left, $pctcomplete, $avgtime, $eta); return($total, $completed, $left, $pctcomplete, $avgtime, $eta);
} }
} }
@ -3041,16 +3049,16 @@ sub expand_slf_shorthand {
if($file eq "-") { if($file eq "-") {
# skip: It is stdin # skip: It is stdin
} elsif($file eq "..") { } elsif($file eq "..") {
$file = $ENV{'HOME'}."/.parallel/sshloginfile"; $file = $ENV{'PARALLEL_HOME'}."/sshloginfile";
} elsif($file eq ".") { } elsif($file eq ".") {
$file = "/etc/parallel/sshloginfile"; $file = "/etc/parallel/sshloginfile";
} elsif(not -r $file) { } elsif(not -r $file) {
if(not -r $ENV{'HOME'}."/.parallel/".$file) { if(not -r $ENV{'PARALLEL_HOME'}."/".$file) {
# Try prepending ~/.parallel # Try prepending $PARALLEL_HOME
::error("Cannot open $file."); ::error("Cannot open $file.");
::wait_and_exit(255); ::wait_and_exit(255);
} else { } else {
$file = $ENV{'HOME'}."/.parallel/".$file; $file = $ENV{'PARALLEL_HOME'}."/".$file;
} }
} }
return $file; return $file;
@ -3948,7 +3956,7 @@ sub usage {
sub citation_notice { sub citation_notice {
# if --will-cite or --plain: do nothing # if --will-cite or --plain: do nothing
# if stderr redirected: do nothing # if stderr redirected: do nothing
# if ~/.parallel/will-cite: do nothing # if $PARALLEL_HOME/will-cite: do nothing
# else: print citation notice to stderr # else: print citation notice to stderr
if($opt::willcite if($opt::willcite
or or
@ -3956,7 +3964,7 @@ sub citation_notice {
or or
not -t $Global::original_stderr not -t $Global::original_stderr
or or
-e $ENV{'HOME'}."/.parallel/will-cite") { -e $ENV{'PARALLEL_HOME'}."/will-cite") {
# skip # skip
} else { } else {
::status ::status
@ -4063,15 +4071,15 @@ sub citation {
"", "",
"" ""
); );
while(not -e $ENV{'HOME'}."/.parallel/will-cite") { while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") {
print "\nType: 'will cite' and press enter.\n> "; print "\nType: 'will cite' and press enter.\n> ";
my $input = <STDIN>; my $input = <STDIN>;
if(not defined $input) { if(not defined $input) {
exit(255); exit(255);
} }
if($input =~ /will cite/i) { if($input =~ /will cite/i) {
mkdir $ENV{'HOME'}."/.parallel"; mkdir $ENV{'PARALLEL_HOME'};
if(open (my $fh, ">", $ENV{'HOME'}."/.parallel/will-cite")) { if(open (my $fh, ">", $ENV{'PARALLEL_HOME'}."/will-cite")) {
close $fh; close $fh;
::status( ::status(
"", "",
@ -4784,12 +4792,12 @@ sub new {
'control_path' => undef, 'control_path' => undef,
'time_to_login' => undef, 'time_to_login' => undef,
'last_login_at' => undef, 'last_login_at' => undef,
'loadavg_file' => $ENV{'HOME'} . "/.parallel/tmp/loadavg-" . 'loadavg_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" .
$no_slash_string, $no_slash_string . "/loadavg",
'loadavg' => undef, 'loadavg' => undef,
'last_loadavg_update' => 0, 'last_loadavg_update' => 0,
'swap_activity_file' => $ENV{'HOME'} . "/.parallel/tmp/swap_activity-" . 'swap_activity_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" .
$no_slash_string, $no_slash_string . "/swap_activity",
'swap_activity' => undef, 'swap_activity' => undef,
}, ref($class) || $class; }, ref($class) || $class;
} }
@ -5007,8 +5015,8 @@ sub swap_activity {
if($update_swap_activity_file) { if($update_swap_activity_file) {
::debug("swap", "Updating swap_activity file ", $self->{'swap_activity_file'}); ::debug("swap", "Updating swap_activity file ", $self->{'swap_activity_file'});
$self->{'last_swap_activity_update'} = time; $self->{'last_swap_activity_update'} = time;
-e $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel"; my $dir = ::dirname($self->{'swap_activity_file'});
-e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp"; -d $dir or eval { File::Path::mkpath($dir); };
my $swap_activity; my $swap_activity;
$swap_activity = swapactivityscript(); $swap_activity = swapactivityscript();
if($self->{'string'} ne ":") { if($self->{'string'} ne ":") {
@ -5297,10 +5305,9 @@ sub loadavg {
if($update_loadavg_file) { if($update_loadavg_file) {
::debug("load", "Updating loadavg file", $self->{'loadavg_file'}, "\n"); ::debug("load", "Updating loadavg file", $self->{'loadavg_file'}, "\n");
$self->{'last_loadavg_update'} = time; $self->{'last_loadavg_update'} = time;
-w $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel" my $dir = ::dirname($self->{'swap_activity_file'});
or ::die_bug("Cannot write to $ENV{'HOME'}/.parallel"); -d $dir or eval { File::Path::mkpath($dir); };
-w $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp" -w $dir or ::die_bug("Cannot write to $dir");
or ::die_bug("Cannot write to $ENV{'HOME'}/.parallel/tmp");
my $cmd = ""; my $cmd = "";
if($self->{'string'} ne ":") { if($self->{'string'} ne ":") {
$cmd = $self->sshcommand() . " " . $self->serverlogin() . " " . $cmd = $self->sshcommand() . " " . $self->serverlogin() . " " .
@ -7495,7 +7502,7 @@ sub sshlogin_wrap {
if(grep { /^_$/ } @vars) { if(grep { /^_$/ } @vars) {
# --env _ # --env _
# Include all vars that are not in a clean environment # Include all vars that are not in a clean environment
if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) { if(open(my $vars_fh, "<", $ENV{'PARALLEL_HOME'} . "/ignored_vars")) {
my @ignore = <$vars_fh>; my @ignore = <$vars_fh>;
chomp @ignore; chomp @ignore;
my %ignore; my %ignore;
@ -9806,7 +9813,8 @@ sub max_length {
# number of chars on the longest command line allowed # number of chars on the longest command line allowed
if(not $Limits::Command::line_max_len) { if(not $Limits::Command::line_max_len) {
# Disk cache of max command line length # Disk cache of max command line length
my $len_cache = $ENV{'HOME'} . "/.parallel/tmp/linelen-" . ::hostname(); my $len_cache = $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" . ::hostname() .
"/linelen";
my $cached_limit; my $cached_limit;
if(-e $len_cache) { if(-e $len_cache) {
open(my $fh, "<", $len_cache) || ::die_bug("Cannot read $len_cache"); open(my $fh, "<", $len_cache) || ::die_bug("Cannot read $len_cache");
@ -9815,8 +9823,8 @@ sub max_length {
} else { } else {
$cached_limit = real_max_length(); $cached_limit = real_max_length();
# If $HOME is write protected: Do not fail # If $HOME is write protected: Do not fail
mkdir($ENV{'HOME'} . "/.parallel"); my $dir = ::dirname($len_cache);
mkdir($ENV{'HOME'} . "/.parallel/tmp"); -d $dir or eval { File::Path::mkpath($dir); };
open(my $fh, ">", $len_cache); open(my $fh, ">", $len_cache);
print $fh $cached_limit; print $fh $cached_limit;
close $fh; close $fh;
@ -11013,8 +11021,8 @@ package Semaphore;
# If a process dies without releasing the semaphore the next process # If a process dies without releasing the semaphore the next process
# that needs that entry will clean up dead semaphores # that needs that entry will clean up dead semaphores
# #
# The semaphores are stored in ~/.parallel/semaphores/id-<name> Each # The semaphores are stored in $PARALLEL_HOME/semaphores/id-<name> Each
# file in ~/.parallel/semaphores/id-<name>/ is the process ID of the # file in $PARALLEL_HOME/semaphores/id-<name>/ is the process ID of the
# process holding the entry. If the process dies, the entry can be # process holding the entry. If the process dies, the entry can be
# taken by another process. # taken by another process.
@ -11024,11 +11032,10 @@ sub new {
my $count = shift; my $count = shift;
$id =~ s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex $id =~ s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex
$id = "id-".$id; # To distinguish it from a process id $id = "id-".$id; # To distinguish it from a process id
my $parallel_dir = $ENV{'HOME'}."/.parallel"; my $parallel_locks = $ENV{'PARALLEL_HOME'}."/semaphores";
-d $parallel_dir or ::mkdir_or_die($parallel_dir);
my $parallel_locks = $parallel_dir."/semaphores";
-d $parallel_locks or ::mkdir_or_die($parallel_locks); -d $parallel_locks or ::mkdir_or_die($parallel_locks);
my $lockdir = "$parallel_locks/$id"; my $lockdir = "$parallel_locks/$id";
my $lockfile = $lockdir.".lock"; my $lockfile = $lockdir.".lock";
if($count < 1) { ::die_bug("semaphore-count: $count"); } if($count < 1) { ::die_bug("semaphore-count: $count"); }
return bless { return bless {

5
src/parallel.pod
View file

@ -4113,6 +4113,11 @@ the currently running jobs are finished before exiting.
=over 9 =over 9
=item $PARALLEL_HOME
Dir where GNU B<parallel> stores config files, semaphores, and caches
information between invocations. Default: $HOME/.parallel
=item $PARALLEL_PID =item $PARALLEL_PID
The environment variable $PARALLEL_PID is set by GNU B<parallel> and The environment variable $PARALLEL_PID is set by GNU B<parallel> and

1
testsuite/wanted-results/parallel-tutorial
View file

@ -947,6 +947,7 @@ MOZ_NO_REMOTE
ORACLE_HOME ORACLE_HOME
ORACLE_SID ORACLE_SID
PARALLEL PARALLEL
PARALLEL_HOME
PARALLEL_PID PARALLEL_PID
PARALLEL_SEQ PARALLEL_SEQ
PARALLEL_TMP PARALLEL_TMP