mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 20:57:53 +00:00
parallel: $XDG_*
This commit is contained in:
Ole Tange
parent
4548ebd756
commit
c63881c4cc
|
|
@ -131,6 +131,7 @@ git diff
|
|||
YYYYMMDD=`yyyymmdd`
|
||||
TAG=MyTag
|
||||
echo "Released as $YYYYMMDD ('$TAG')." | grep MyTag && (STOP;STOP;STOP)
|
||||
echo "$TAG" | grep ' ' && (STOP;STOP;STOP)
|
||||
|
||||
git commit -a -m "Released as $YYYYMMDD ('$TAG')"
|
||||
git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $TAG
|
||||
|
|
@ -198,26 +199,27 @@ to:parallel@gnu.org, bug-parallel@gnu.org
|
|||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
|
||||
Subject: GNU Parallel 20170922 ('Mexico City') released <<[stable]>>
|
||||
Subject: GNU Parallel 20171022 ('Peter Madsen/Catalonia/Las Vegas') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20170922 ('Mexico City') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
GNU Parallel 20171022 ('Peter Madsen') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
|
||||
<<No new functionality was introduced so this is a good candidate for a stable release.>>
|
||||
|
||||
Haiku of the month:
|
||||
|
||||
--limit can
|
||||
limit jobs dynamic'ly
|
||||
given a command
|
||||
<<>>
|
||||
--ole-tange
|
||||
|
||||
New in this release:
|
||||
|
||||
* Use '--limit myprog' to make a dynamic job limit. Just return 0 to spawn another job, 1 to not spawn another job, and 2 to kill the youngest job.
|
||||
*
|
||||
https://www.turing.ac.uk/events/masterclass-cluster-computing/
|
||||
|
||||
* PARALLEL_RSYNC_OPTS and --rsync-opts sets the options for rsync (Default: -rlDzR).
|
||||
https://openwetware.org/wiki/Butlin:Unix_for_Bioinformatics_-_advanced_tutorial
|
||||
|
||||
* Download all of humble bundle books in parallel https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel
|
||||
https://veer66.wordpress.com/2017/09/02/use-multi-core/
|
||||
|
||||
https://gist.github.com/drmalex07/de8ff1379a560f6d84a0d863e60ae378
|
||||
|
||||
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
|
||||
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ function env_parallel
|
|||
s/\n/\001/g'
|
||||
end;
|
||||
)
|
||||
# If --record-env: exit
|
||||
perl -e 'exit grep { /^--record-env$/ } @ARGV' -- $argv; and parallel $argv;
|
||||
set _parallel_exit_CODE $status
|
||||
set -e PARALLEL_ENV
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20170922;
|
||||
$Global::version = 20170923;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
68
src/parallel
68
src/parallel
|
|
@ -1381,7 +1381,7 @@ sub check_invalid_option_combinations {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20170922;
|
||||
$Global::version = 20170923;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -1458,7 +1458,30 @@ sub init_globals {
|
|||
::warning("\$HOME not set. Using /tmp.");
|
||||
$ENV{HOME} = "/tmp";
|
||||
}
|
||||
$ENV{'PARALLEL_HOME'} ||= $ENV{'HOME'} . "/.parallel";
|
||||
# no warnings to allow for undefined $XDG_*
|
||||
no warnings 'uninitialized';
|
||||
# $xdg_config_home is needed to make env_parallel.fish stop complaining
|
||||
my $xdg_config_home = $ENV{'XDG_CONFIG_HOME'};
|
||||
# config_dirs = $PARALLEL_HOME, $XDG_CONFIG_HOME/parallel,
|
||||
# $(each XDG_CONFIG_DIRS)/parallel, $HOME/.parallel
|
||||
# Keep only dirs that exist
|
||||
@Global::config_dirs =
|
||||
(grep { -d $_ }
|
||||
$ENV{'PARALLEL_HOME'},
|
||||
(map { "$_/parallel" }
|
||||
$xdg_config_home,
|
||||
split /:/, $ENV{'XDG_CONFIG_DIRS'}),
|
||||
$ENV{'HOME'} . "/.parallel");
|
||||
# Use first dir as config dir
|
||||
$Global::config_dir = $Global::config_dirs[0] ||
|
||||
$ENV{'HOME'} . "/.parallel";
|
||||
# cache_dirs = $PARALLEL_HOME, $XDG_CACHE_HOME/parallel,
|
||||
# Keep only dirs that exist
|
||||
@Global::cache_dirs =
|
||||
(grep { -d $_ }
|
||||
$ENV{'PARALLEL_HOME'}, $ENV{'XDG_CACHE_HOME'}."/parallel");
|
||||
$Global::cache_dir = $Global::cache_dirs[0] ||
|
||||
$ENV{'HOME'} . "/.parallel";
|
||||
}
|
||||
|
||||
sub parse_halt {
|
||||
|
|
@ -1633,7 +1656,7 @@ sub parse_semaphore {
|
|||
sub record_env {
|
||||
# Record current %ENV-keys in $PARALLEL_HOME/ignored_vars
|
||||
# Returns: N/A
|
||||
my $ignore_filename = $ENV{'PARALLEL_HOME'} . "/ignored_vars";
|
||||
my $ignore_filename = $Global::config_dir . "/ignored_vars";
|
||||
if(open(my $vars_fh, ">", $ignore_filename)) {
|
||||
print $vars_fh map { $_,"\n" } keys %ENV;
|
||||
} else {
|
||||
|
|
@ -1905,18 +1928,16 @@ sub read_options {
|
|||
# Add options from $PARALLEL_HOME/config and other profiles
|
||||
my @config_profiles = (
|
||||
"/etc/parallel/config",
|
||||
$ENV{'PARALLEL_HOME'}."/config",
|
||||
(map { "$_/config" } @Global::config_dirs),
|
||||
$ENV{'HOME'}."/.parallelrc");
|
||||
my @profiles = @config_profiles;
|
||||
if(@opt::profile) {
|
||||
# --profile overrides default profiles
|
||||
@profiles = ();
|
||||
for my $profile (@opt::profile) {
|
||||
if(-r $profile) {
|
||||
push @profiles, $profile;
|
||||
} else {
|
||||
push @profiles, $ENV{'PARALLEL_HOME'}."/".$profile;
|
||||
}
|
||||
# Look for the $profile in . and @Global::config_dirs
|
||||
push @profiles, grep { -r $_ }
|
||||
map { "$_/$profile" } ".", @Global::config_dirs;
|
||||
}
|
||||
}
|
||||
for my $profile (@profiles) {
|
||||
|
|
@ -3116,16 +3137,19 @@ sub expand_slf_shorthand {
|
|||
if($file eq "-") {
|
||||
# skip: It is stdin
|
||||
} elsif($file eq "..") {
|
||||
$file = $ENV{'PARALLEL_HOME'}."/sshloginfile";
|
||||
$file = $Global::config_dir."/sshloginfile";
|
||||
} elsif($file eq ".") {
|
||||
$file = "/etc/parallel/sshloginfile";
|
||||
} elsif(not -r $file) {
|
||||
if(not -r $ENV{'PARALLEL_HOME'}."/".$file) {
|
||||
for(@Global::config_dirs) {
|
||||
if(not -r $_."/".$file) {
|
||||
# Try prepending $PARALLEL_HOME
|
||||
::error("Cannot open $file.");
|
||||
::wait_and_exit(255);
|
||||
} else {
|
||||
$file = $ENV{'PARALLEL_HOME'}."/".$file;
|
||||
} else {
|
||||
$file = $_."/".$file;
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $file;
|
||||
|
|
@ -4049,7 +4073,7 @@ sub citation_notice {
|
|||
or
|
||||
not -t $Global::original_stderr
|
||||
or
|
||||
-e $ENV{'PARALLEL_HOME'}."/will-cite") {
|
||||
grep { -e "$_/will-cite" } @Global::config_dirs) {
|
||||
# skip
|
||||
} else {
|
||||
::status
|
||||
|
|
@ -4158,15 +4182,15 @@ sub citation {
|
|||
"mentioned in the release notes of next version of GNU Parallel.",
|
||||
""
|
||||
);
|
||||
while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") {
|
||||
while(not grep { -e "$_/will-cite" } @Global::config_dirs) {
|
||||
print "\nType: 'will cite' and press enter.\n> ";
|
||||
my $input = <STDIN>;
|
||||
if(not defined $input) {
|
||||
exit(255);
|
||||
}
|
||||
if($input =~ /will cite/i) {
|
||||
mkdir $ENV{'PARALLEL_HOME'};
|
||||
if(open (my $fh, ">", $ENV{'PARALLEL_HOME'}."/will-cite")) {
|
||||
mkdir $Global::config_dir;
|
||||
if(open (my $fh, ">", $Global::config_dir."/will-cite")) {
|
||||
close $fh;
|
||||
::status(
|
||||
"",
|
||||
|
|
@ -4917,11 +4941,11 @@ sub new {
|
|||
'control_path' => undef,
|
||||
'time_to_login' => undef,
|
||||
'last_login_at' => undef,
|
||||
'loadavg_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" .
|
||||
'loadavg_file' => $Global::cache_dir . "/tmp/sshlogin/" .
|
||||
$no_slash_string . "/loadavg",
|
||||
'loadavg' => undef,
|
||||
'last_loadavg_update' => 0,
|
||||
'swap_activity_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" .
|
||||
'swap_activity_file' => $Global::cache_dir . "/tmp/sshlogin/" .
|
||||
$no_slash_string . "/swap_activity",
|
||||
'swap_activity' => undef,
|
||||
}, ref($class) || $class;
|
||||
|
|
@ -7762,7 +7786,7 @@ sub sshlogin_wrap {
|
|||
local $/ = "\n";
|
||||
# --env _
|
||||
# Include all vars that are not in a clean environment
|
||||
if(open(my $vars_fh, "<", $ENV{'PARALLEL_HOME'} . "/ignored_vars")) {
|
||||
if(open(my $vars_fh, "<", $Global::config_dir . "/ignored_vars")) {
|
||||
my @ignore = <$vars_fh>;
|
||||
chomp @ignore;
|
||||
my %ignore;
|
||||
|
|
@ -10113,7 +10137,7 @@ sub max_length {
|
|||
# number of chars on the longest command line allowed
|
||||
if(not $Limits::Command::line_max_len) {
|
||||
# Disk cache of max command line length
|
||||
my $len_cache = $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" . ::hostname() .
|
||||
my $len_cache = $Global::cache_dir . "/tmp/sshlogin/" . ::hostname() .
|
||||
"/linelen";
|
||||
my $cached_limit;
|
||||
if(-e $len_cache) {
|
||||
|
|
@ -11348,7 +11372,7 @@ sub new {
|
|||
my $count = shift;
|
||||
$id =~ s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex
|
||||
$id = "id-".$id; # To distinguish it from a process id
|
||||
my $parallel_locks = $ENV{'PARALLEL_HOME'}."/semaphores";
|
||||
my $parallel_locks = $Global::cache_dir . "/semaphores";
|
||||
-d $parallel_locks or ::mkdir_or_die($parallel_locks);
|
||||
my $lockdir = "$parallel_locks/$id";
|
||||
|
||||
|
|
|
|||
|
|
@ -1042,7 +1042,7 @@ B<-l 0> is an alias for B<-l 1>.
|
|||
Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
|
||||
|
||||
|
||||
=item B<--limit> "I<command> I<args>" (alpha testing)
|
||||
=item B<--limit> "I<command> I<args>" (beta testing)
|
||||
|
||||
Dynamic job limit. Before starting a new job run I<command> with
|
||||
I<args>. The exit value of I<command> determines what GNU B<parallel>
|
||||
|
|
@ -1085,9 +1085,9 @@ Similar to B<--memfree>.
|
|||
=back
|
||||
|
||||
|
||||
=item B<--line-buffer> (beta testing)
|
||||
=item B<--line-buffer>
|
||||
|
||||
=item B<--lb> (beta testing)
|
||||
=item B<--lb>
|
||||
|
||||
Buffer output on line basis. B<--group> will keep the output together
|
||||
for a whole job. B<--ungroup> allows output to mixup with half a line
|
||||
|
|
@ -1915,7 +1915,7 @@ You can even use multiple matches:
|
|||
See also: B<{= perl expression =}> B<--parens>
|
||||
|
||||
|
||||
=item B<--rsync-opts> I<options> (alpha testing)
|
||||
=item B<--rsync-opts> I<options> (beta testing)
|
||||
|
||||
Options to pass on to B<rsync>. Setting B<--rsync-opts> takes
|
||||
precedence over setting the environment variable $PARALLEL_RSYNC_OPTS.
|
||||
|
|
@ -2188,7 +2188,8 @@ B<ssh-copy-id>, and B<sshpass> may help with that).
|
|||
The sshlogin ':' is special, it means 'no ssh' and will therefore run
|
||||
on the local computer.
|
||||
|
||||
The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile
|
||||
The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile or
|
||||
$XDG_CONFIG_HOME/parallel/sshloginfile
|
||||
|
||||
The sshlogin '-' is special, too, it read sshlogins from stdin
|
||||
(standard input).
|
||||
|
|
|
|||
|
|
@ -1288,6 +1288,18 @@ GNU B<parallel> can be used as a poor-man's version of ClusterSSH:
|
|||
|
||||
B<parallel --nonall -S server-a,server-b do_stuff foo bar>
|
||||
|
||||
=head2 coshell
|
||||
|
||||
B<coshell> only accepts full commands on standard input. Any quoting
|
||||
needs to be done by the user.
|
||||
|
||||
Commands are run in B<sh> so any B<bash>/B<tcsh>/B<zsh> specific
|
||||
syntax will not work.
|
||||
|
||||
Output can be buffered by using B<-d>. Output is buffered in memory,
|
||||
so big output can cause swapping and therefore be terrible slow or
|
||||
even cause out of memory.
|
||||
|
||||
|
||||
=head2 Todo
|
||||
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20170922;
|
||||
$Global::version = 20170923;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -62,9 +62,9 @@ par_linebuffer_matters_compress_tag() {
|
|||
|
||||
testfunc() {
|
||||
linebuffer="$1"
|
||||
# Sleep 2 sec to give time to linebuffer-print the first part
|
||||
# Sleep 3 sec to give time to linebuffer-print the first part
|
||||
parallel -j0 $linebuffer --compress --tag \
|
||||
"shuf $randomfile; sleep 2; shuf $randomfile; true" ::: {0..10} |
|
||||
"shuf $randomfile; sleep 3; shuf $randomfile; true" ::: {0..10} |
|
||||
perl -ne '/^(\S+)\t/ and print "$1\n"' | uniq | sort
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -349,13 +349,13 @@ echo '{##} bug #45841: Replacement string for total no of jobs'
|
|||
3 3
|
||||
parallel -k -S 8/: -X --plus echo {#} {##} ::: {1..15}
|
||||
1 15 15
|
||||
2 15 15
|
||||
3 15 15
|
||||
4 15 15
|
||||
5 15 15
|
||||
6 15 15
|
||||
7 15 15
|
||||
8 15
|
||||
2 14 14
|
||||
3 14 14
|
||||
4 14 14
|
||||
5 14 14
|
||||
6 14 14
|
||||
7 14 14
|
||||
8 14
|
||||
echo '**'
|
||||
**
|
||||
echo 'bug #47002: --tagstring with -d \n\n'
|
||||
|
|
|
|||
Loading…
Reference in a new issue