parallel: $XDG_*

doc/release_new_version

@@ -131,6 +131,7 @@ git diff
 YYYYMMDD=`yyyymmdd`
 TAG=MyTag
 echo "Released as $YYYYMMDD ('$TAG')." | grep MyTag && (STOP;STOP;STOP)
+echo "$TAG" | grep ' ' && (STOP;STOP;STOP)
 
 git commit -a -m "Released as $YYYYMMDD ('$TAG')"
 git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $TAG
@@ -198,26 +199,27 @@ to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
 
-Subject: GNU Parallel 20170922 ('Mexico City') released <<[stable]>>
+Subject: GNU Parallel 20171022 ('Peter Madsen/Catalonia/Las Vegas') released <<[stable]>>
 
-GNU Parallel 20170922 ('Mexico City') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20171022 ('Peter Madsen') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
 Haiku of the month:
 
-  --limit can
+  <<>>
-  limit jobs dynamic'ly
-  given a command
     --ole-tange
 
 New in this release:
 
-* Use '--limit myprog' to make a dynamic job limit. Just return 0 to spawn another job, 1 to not spawn another job, and 2 to kill the youngest job.
+*
+https://www.turing.ac.uk/events/masterclass-cluster-computing/
 
-* PARALLEL_RSYNC_OPTS and --rsync-opts sets the options for rsync (Default: -rlDzR).
+https://openwetware.org/wiki/Butlin:Unix_for_Bioinformatics_-_advanced_tutorial
 
-* Download all of humble bundle books in parallel https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel
+https://veer66.wordpress.com/2017/09/02/use-multi-core/
+
+https://gist.github.com/drmalex07/de8ff1379a560f6d84a0d863e60ae378
 
 <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
 

src/env_parallel.fish

@@ -168,6 +168,7 @@ function env_parallel
           s/\n/\001/g'
     end;
     )
+  # If --record-env: exit
   perl -e 'exit grep { /^--record-env$/ } @ARGV' -- $argv; and parallel $argv;
   set _parallel_exit_CODE $status
   set -e PARALLEL_ENV

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20170922;
+$Global::version = 20170923;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -1381,7 +1381,7 @@ sub check_invalid_option_combinations {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20170922;
+    $Global::version = 20170923;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -1458,7 +1458,30 @@ sub init_globals {
 	::warning("\$HOME not set. Using /tmp.");
 	$ENV{HOME} = "/tmp";
     }
-    $ENV{'PARALLEL_HOME'} ||= $ENV{'HOME'} . "/.parallel";
+    # no warnings to allow for undefined $XDG_*
+    no warnings 'uninitialized';
+    # $xdg_config_home is needed to make env_parallel.fish stop complaining
+    my $xdg_config_home = $ENV{'XDG_CONFIG_HOME'};
+    # config_dirs = $PARALLEL_HOME, $XDG_CONFIG_HOME/parallel,
+    #   $(each XDG_CONFIG_DIRS)/parallel, $HOME/.parallel
+    # Keep only dirs that exist
+    @Global::config_dirs =
+	(grep { -d $_ }
+	 $ENV{'PARALLEL_HOME'},
+	 (map { "$_/parallel" }
+	  $xdg_config_home,
+	  split /:/, $ENV{'XDG_CONFIG_DIRS'}),
+	 $ENV{'HOME'} . "/.parallel");
+    # Use first dir as config dir
+    $Global::config_dir = $Global::config_dirs[0] ||
+	$ENV{'HOME'} . "/.parallel";
+    # cache_dirs = $PARALLEL_HOME, $XDG_CACHE_HOME/parallel,
+    # Keep only dirs that exist
+    @Global::cache_dirs =
+	(grep { -d $_ }
+	 $ENV{'PARALLEL_HOME'}, $ENV{'XDG_CACHE_HOME'}."/parallel");
+    $Global::cache_dir = $Global::cache_dirs[0] ||
+	$ENV{'HOME'} . "/.parallel";
 }
 
 sub parse_halt {
@@ -1633,7 +1656,7 @@ sub parse_semaphore {
 sub record_env {
     # Record current %ENV-keys in $PARALLEL_HOME/ignored_vars
     # Returns: N/A
-    my $ignore_filename = $ENV{'PARALLEL_HOME'} . "/ignored_vars";
+    my $ignore_filename = $Global::config_dir . "/ignored_vars";
     if(open(my $vars_fh, ">", $ignore_filename)) {
 	print $vars_fh map { $_,"\n" } keys %ENV;
     } else {
@@ -1905,18 +1928,16 @@ sub read_options {
 	# Add options from $PARALLEL_HOME/config and other profiles
 	my @config_profiles = (
 	    "/etc/parallel/config",
-	    $ENV{'PARALLEL_HOME'}."/config",
+	    (map { "$_/config" } @Global::config_dirs),
 	    $ENV{'HOME'}."/.parallelrc");
 	my @profiles = @config_profiles;
 	if(@opt::profile) {
 	    # --profile overrides default profiles
 	    @profiles = ();
 	    for my $profile (@opt::profile) {
-		if(-r $profile) {
+		# Look for the $profile in . and @Global::config_dirs
-		    push @profiles, $profile;
+		push @profiles, grep { -r $_ }
-		} else {
+		    map { "$_/$profile" } ".", @Global::config_dirs;
-		    push @profiles, $ENV{'PARALLEL_HOME'}."/".$profile;
-		}
 	    }
 	}
 	for my $profile (@profiles) {
@@ -3116,16 +3137,19 @@ sub expand_slf_shorthand {
     if($file eq "-") {
 	# skip: It is stdin
     } elsif($file eq "..") {
-        $file = $ENV{'PARALLEL_HOME'}."/sshloginfile";
+        $file = $Global::config_dir."/sshloginfile";
     } elsif($file eq ".") {
         $file = "/etc/parallel/sshloginfile";
     } elsif(not -r $file) {
-	if(not -r $ENV{'PARALLEL_HOME'}."/".$file) {
+	for(@Global::config_dirs) {
+	    if(not -r $_."/".$file) {
 		# Try prepending $PARALLEL_HOME
 		::error("Cannot open $file.");
 		::wait_and_exit(255);
-	} else {
+	    } else {
-	    $file = $ENV{'PARALLEL_HOME'}."/".$file;
+		$file = $_."/".$file;
+		last;
+	    }
 	}
     }
     return $file;
@@ -4049,7 +4073,7 @@ sub citation_notice {
        or
        not -t $Global::original_stderr
        or
-       -e $ENV{'PARALLEL_HOME'}."/will-cite") {
+       grep { -e "$_/will-cite" } @Global::config_dirs) {
 	# skip
     } else {
 	::status
@@ -4158,15 +4182,15 @@ sub citation {
 	  "mentioned in the release notes of next version of GNU Parallel.",
 	  ""
         );
-    while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") {
+    while(not grep { -e "$_/will-cite" } @Global::config_dirs) {
 	print "\nType: 'will cite' and press enter.\n> ";
 	my $input = <STDIN>;
 	if(not defined $input) {
 	    exit(255);
 	}
 	if($input =~ /will cite/i) {
-	    mkdir $ENV{'PARALLEL_HOME'};
+	    mkdir $Global::config_dir;
-	    if(open (my $fh, ">", $ENV{'PARALLEL_HOME'}."/will-cite")) {
+	    if(open (my $fh, ">", $Global::config_dir."/will-cite")) {
 		close $fh;
 		::status(
 		    "",
@@ -4917,11 +4941,11 @@ sub new {
         'control_path' => undef,
 	'time_to_login' => undef,
 	'last_login_at' => undef,
-        'loadavg_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" .
+        'loadavg_file' => $Global::cache_dir . "/tmp/sshlogin/" .
 	    $no_slash_string . "/loadavg",
         'loadavg' => undef,
 	'last_loadavg_update' => 0,
-        'swap_activity_file' => $ENV{'PARALLEL_HOME'}  . "/tmp/sshlogin/" .
+        'swap_activity_file' => $Global::cache_dir . "/tmp/sshlogin/" .
             $no_slash_string . "/swap_activity",
         'swap_activity' => undef,
     }, ref($class) || $class;
@@ -7762,7 +7786,7 @@ sub sshlogin_wrap {
 	    local $/ = "\n";
 	    # --env _
 	    # Include all vars that are not in a clean environment
-	    if(open(my $vars_fh, "<", $ENV{'PARALLEL_HOME'} . "/ignored_vars")) {
+	    if(open(my $vars_fh, "<", $Global::config_dir . "/ignored_vars")) {
 		my @ignore = <$vars_fh>;
 		chomp @ignore;
 		my %ignore;
@@ -10113,7 +10137,7 @@ sub max_length {
     #   number of chars on the longest command line allowed
     if(not $Limits::Command::line_max_len) {
 	# Disk cache of max command line length
-	my $len_cache = $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" . ::hostname() .
+	my $len_cache = $Global::cache_dir . "/tmp/sshlogin/" . ::hostname() .
 	    "/linelen";
 	my $cached_limit;
 	if(-e $len_cache) {
@@ -11348,7 +11372,7 @@ sub new {
     my $count = shift;
     $id =~ s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex
     $id = "id-".$id; # To distinguish it from a process id
-    my $parallel_locks = $ENV{'PARALLEL_HOME'}."/semaphores";
+    my $parallel_locks = $Global::cache_dir . "/semaphores";
     -d $parallel_locks or ::mkdir_or_die($parallel_locks);
     my $lockdir = "$parallel_locks/$id";
 

src/parallel.pod

@@ -1042,7 +1042,7 @@ B<-l 0> is an alias for B<-l 1>.
 Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
 
 
-=item B<--limit> "I<command> I<args>" (alpha testing)
+=item B<--limit> "I<command> I<args>" (beta testing)
 
 Dynamic job limit. Before starting a new job run I<command> with
 I<args>. The exit value of I<command> determines what GNU B<parallel>
@@ -1085,9 +1085,9 @@ Similar to B<--memfree>.
 =back
 
 
-=item B<--line-buffer> (beta testing)
+=item B<--line-buffer>
 
-=item B<--lb> (beta testing)
+=item B<--lb>
 
 Buffer output on line basis. B<--group> will keep the output together
 for a whole job. B<--ungroup> allows output to mixup with half a line
@@ -1915,7 +1915,7 @@ You can even use multiple matches:
 See also: B<{= perl expression =}> B<--parens>
 
 
-=item B<--rsync-opts> I<options> (alpha testing)
+=item B<--rsync-opts> I<options> (beta testing)
 
 Options to pass on to B<rsync>. Setting B<--rsync-opts> takes
 precedence over setting the environment variable $PARALLEL_RSYNC_OPTS.
@@ -2188,7 +2188,8 @@ B<ssh-copy-id>, and B<sshpass> may help with that).
 The sshlogin ':' is special, it means 'no ssh' and will therefore run
 on the local computer.
 
-The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile
+The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile or
+$XDG_CONFIG_HOME/parallel/sshloginfile
 
 The sshlogin '-' is special, too, it read sshlogins from stdin
 (standard input).

src/parallel_alternatives.pod

@@ -1288,6 +1288,18 @@ GNU B<parallel> can be used as a poor-man's version of ClusterSSH:
 
 B<parallel --nonall -S server-a,server-b do_stuff foo bar>
 
+=head2 coshell
+
+B<coshell> only accepts full commands on standard input. Any quoting
+needs to be done by the user.
+
+Commands are run in B<sh> so any B<bash>/B<tcsh>/B<zsh> specific
+syntax will not work.
+
+Output can be buffered by using B<-d>. Output is buffered in memory,
+so big output can cause swapping and therefore be terrible slow or
+even cause out of memory.
+
 
 =head2 Todo
 

src/sql

@@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20170922;
+    $Global::version = 20170923;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/parallel-local-30s.sh

@@ -62,9 +62,9 @@ par_linebuffer_matters_compress_tag() {
 
     testfunc() {
 	linebuffer="$1"
-	# Sleep 2 sec to give time to linebuffer-print the first part
+	# Sleep 3 sec to give time to linebuffer-print the first part
 	parallel -j0 $linebuffer --compress --tag \
-		 "shuf $randomfile; sleep 2; shuf $randomfile; true" ::: {0..10} |
+		 "shuf $randomfile; sleep 3; shuf $randomfile; true" ::: {0..10} |
 	    perl -ne '/^(\S+)\t/ and print "$1\n"' | uniq | sort
     }
 

testsuite/wanted-results/parallel-local-0.3s

@@ -349,13 +349,13 @@ echo '{##} bug #45841: Replacement string for total no of jobs'
 3 3
   parallel -k -S 8/: -X --plus echo {#} {##} ::: {1..15}
 1 15 15
-2 15 15
+2 14 14
-3 15 15
+3 14 14
-4 15 15
+4 14 14
-5 15 15
+5 14 14
-6 15 15
+6 14 14
-7 15 15
+7 14 14
-8 15
+8 14
 echo '**'
 **
 echo 'bug #47002: --tagstring with -d \n\n'