From c63881c4cc7247175133586b273bb8374aabd5a2 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 10 Oct 2017 00:33:43 +0200 Subject: [PATCH] parallel: $XDG_* --- doc/release_new_version | 18 +++--- src/env_parallel.fish | 1 + src/niceload | 2 +- src/parallel | 68 +++++++++++++------- src/parallel.pod | 11 ++-- src/parallel_alternatives.pod | 12 ++++ src/sql | 2 +- testsuite/tests-to-run/parallel-local-30s.sh | 4 +- testsuite/wanted-results/parallel-local-0.3s | 14 ++-- 9 files changed, 86 insertions(+), 46 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 85d7aca5..375e6aa9 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -131,6 +131,7 @@ git diff YYYYMMDD=`yyyymmdd` TAG=MyTag echo "Released as $YYYYMMDD ('$TAG')." | grep MyTag && (STOP;STOP;STOP) +echo "$TAG" | grep ' ' && (STOP;STOP;STOP) git commit -a -m "Released as $YYYYMMDD ('$TAG')" git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $TAG @@ -198,26 +199,27 @@ to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20170922 ('Mexico City') released <<[stable]>> +Subject: GNU Parallel 20171022 ('Peter Madsen/Catalonia/Las Vegas') released <<[stable]>> -GNU Parallel 20170922 ('Mexico City') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20171022 ('Peter Madsen') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> Haiku of the month: - --limit can - limit jobs dynamic'ly - given a command + <<>> --ole-tange New in this release: -* Use '--limit myprog' to make a dynamic job limit. Just return 0 to spawn another job, 1 to not spawn another job, and 2 to kill the youngest job. +* +https://www.turing.ac.uk/events/masterclass-cluster-computing/ -* PARALLEL_RSYNC_OPTS and --rsync-opts sets the options for rsync (Default: -rlDzR). +https://openwetware.org/wiki/Butlin:Unix_for_Bioinformatics_-_advanced_tutorial -* Download all of humble bundle books in parallel https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel +https://veer66.wordpress.com/2017/09/02/use-multi-core/ + +https://gist.github.com/drmalex07/de8ff1379a560f6d84a0d863e60ae378 <> diff --git a/src/env_parallel.fish b/src/env_parallel.fish index 63d3fe21..deffb1d7 100755 --- a/src/env_parallel.fish +++ b/src/env_parallel.fish @@ -168,6 +168,7 @@ function env_parallel s/\n/\001/g' end; ) + # If --record-env: exit perl -e 'exit grep { /^--record-env$/ } @ARGV' -- $argv; and parallel $argv; set _parallel_exit_CODE $status set -e PARALLEL_ENV diff --git a/src/niceload b/src/niceload index 90ba741f..76a69643 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20170922; +$Global::version = 20170923; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index d93ecaa5..59cf10e8 100755 --- a/src/parallel +++ b/src/parallel @@ -1381,7 +1381,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170922; + $Global::version = 20170923; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1458,7 +1458,30 @@ sub init_globals { ::warning("\$HOME not set. Using /tmp."); $ENV{HOME} = "/tmp"; } - $ENV{'PARALLEL_HOME'} ||= $ENV{'HOME'} . "/.parallel"; + # no warnings to allow for undefined $XDG_* + no warnings 'uninitialized'; + # $xdg_config_home is needed to make env_parallel.fish stop complaining + my $xdg_config_home = $ENV{'XDG_CONFIG_HOME'}; + # config_dirs = $PARALLEL_HOME, $XDG_CONFIG_HOME/parallel, + # $(each XDG_CONFIG_DIRS)/parallel, $HOME/.parallel + # Keep only dirs that exist + @Global::config_dirs = + (grep { -d $_ } + $ENV{'PARALLEL_HOME'}, + (map { "$_/parallel" } + $xdg_config_home, + split /:/, $ENV{'XDG_CONFIG_DIRS'}), + $ENV{'HOME'} . "/.parallel"); + # Use first dir as config dir + $Global::config_dir = $Global::config_dirs[0] || + $ENV{'HOME'} . "/.parallel"; + # cache_dirs = $PARALLEL_HOME, $XDG_CACHE_HOME/parallel, + # Keep only dirs that exist + @Global::cache_dirs = + (grep { -d $_ } + $ENV{'PARALLEL_HOME'}, $ENV{'XDG_CACHE_HOME'}."/parallel"); + $Global::cache_dir = $Global::cache_dirs[0] || + $ENV{'HOME'} . "/.parallel"; } sub parse_halt { @@ -1633,7 +1656,7 @@ sub parse_semaphore { sub record_env { # Record current %ENV-keys in $PARALLEL_HOME/ignored_vars # Returns: N/A - my $ignore_filename = $ENV{'PARALLEL_HOME'} . "/ignored_vars"; + my $ignore_filename = $Global::config_dir . "/ignored_vars"; if(open(my $vars_fh, ">", $ignore_filename)) { print $vars_fh map { $_,"\n" } keys %ENV; } else { @@ -1905,18 +1928,16 @@ sub read_options { # Add options from $PARALLEL_HOME/config and other profiles my @config_profiles = ( "/etc/parallel/config", - $ENV{'PARALLEL_HOME'}."/config", + (map { "$_/config" } @Global::config_dirs), $ENV{'HOME'}."/.parallelrc"); my @profiles = @config_profiles; if(@opt::profile) { # --profile overrides default profiles @profiles = (); for my $profile (@opt::profile) { - if(-r $profile) { - push @profiles, $profile; - } else { - push @profiles, $ENV{'PARALLEL_HOME'}."/".$profile; - } + # Look for the $profile in . and @Global::config_dirs + push @profiles, grep { -r $_ } + map { "$_/$profile" } ".", @Global::config_dirs; } } for my $profile (@profiles) { @@ -3116,16 +3137,19 @@ sub expand_slf_shorthand { if($file eq "-") { # skip: It is stdin } elsif($file eq "..") { - $file = $ENV{'PARALLEL_HOME'}."/sshloginfile"; + $file = $Global::config_dir."/sshloginfile"; } elsif($file eq ".") { $file = "/etc/parallel/sshloginfile"; } elsif(not -r $file) { - if(not -r $ENV{'PARALLEL_HOME'}."/".$file) { + for(@Global::config_dirs) { + if(not -r $_."/".$file) { # Try prepending $PARALLEL_HOME ::error("Cannot open $file."); ::wait_and_exit(255); - } else { - $file = $ENV{'PARALLEL_HOME'}."/".$file; + } else { + $file = $_."/".$file; + last; + } } } return $file; @@ -4049,7 +4073,7 @@ sub citation_notice { or not -t $Global::original_stderr or - -e $ENV{'PARALLEL_HOME'}."/will-cite") { + grep { -e "$_/will-cite" } @Global::config_dirs) { # skip } else { ::status @@ -4158,15 +4182,15 @@ sub citation { "mentioned in the release notes of next version of GNU Parallel.", "" ); - while(not -e $ENV{'PARALLEL_HOME'}."/will-cite") { + while(not grep { -e "$_/will-cite" } @Global::config_dirs) { print "\nType: 'will cite' and press enter.\n> "; my $input = ; if(not defined $input) { exit(255); } if($input =~ /will cite/i) { - mkdir $ENV{'PARALLEL_HOME'}; - if(open (my $fh, ">", $ENV{'PARALLEL_HOME'}."/will-cite")) { + mkdir $Global::config_dir; + if(open (my $fh, ">", $Global::config_dir."/will-cite")) { close $fh; ::status( "", @@ -4917,11 +4941,11 @@ sub new { 'control_path' => undef, 'time_to_login' => undef, 'last_login_at' => undef, - 'loadavg_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" . + 'loadavg_file' => $Global::cache_dir . "/tmp/sshlogin/" . $no_slash_string . "/loadavg", 'loadavg' => undef, 'last_loadavg_update' => 0, - 'swap_activity_file' => $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" . + 'swap_activity_file' => $Global::cache_dir . "/tmp/sshlogin/" . $no_slash_string . "/swap_activity", 'swap_activity' => undef, }, ref($class) || $class; @@ -7762,7 +7786,7 @@ sub sshlogin_wrap { local $/ = "\n"; # --env _ # Include all vars that are not in a clean environment - if(open(my $vars_fh, "<", $ENV{'PARALLEL_HOME'} . "/ignored_vars")) { + if(open(my $vars_fh, "<", $Global::config_dir . "/ignored_vars")) { my @ignore = <$vars_fh>; chomp @ignore; my %ignore; @@ -10113,7 +10137,7 @@ sub max_length { # number of chars on the longest command line allowed if(not $Limits::Command::line_max_len) { # Disk cache of max command line length - my $len_cache = $ENV{'PARALLEL_HOME'} . "/tmp/sshlogin/" . ::hostname() . + my $len_cache = $Global::cache_dir . "/tmp/sshlogin/" . ::hostname() . "/linelen"; my $cached_limit; if(-e $len_cache) { @@ -11348,7 +11372,7 @@ sub new { my $count = shift; $id =~ s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex $id = "id-".$id; # To distinguish it from a process id - my $parallel_locks = $ENV{'PARALLEL_HOME'}."/semaphores"; + my $parallel_locks = $Global::cache_dir . "/semaphores"; -d $parallel_locks or ::mkdir_or_die($parallel_locks); my $lockdir = "$parallel_locks/$id"; diff --git a/src/parallel.pod b/src/parallel.pod index 27aa12e6..67c07b43 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1042,7 +1042,7 @@ B<-l 0> is an alias for B<-l 1>. Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set. -=item B<--limit> "I I" (alpha testing) +=item B<--limit> "I I" (beta testing) Dynamic job limit. Before starting a new job run I with I. The exit value of I determines what GNU B @@ -1085,9 +1085,9 @@ Similar to B<--memfree>. =back -=item B<--line-buffer> (beta testing) +=item B<--line-buffer> -=item B<--lb> (beta testing) +=item B<--lb> Buffer output on line basis. B<--group> will keep the output together for a whole job. B<--ungroup> allows output to mixup with half a line @@ -1915,7 +1915,7 @@ You can even use multiple matches: See also: B<{= perl expression =}> B<--parens> -=item B<--rsync-opts> I (alpha testing) +=item B<--rsync-opts> I (beta testing) Options to pass on to B. Setting B<--rsync-opts> takes precedence over setting the environment variable $PARALLEL_RSYNC_OPTS. @@ -2188,7 +2188,8 @@ B, and B may help with that). The sshlogin ':' is special, it means 'no ssh' and will therefore run on the local computer. -The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile +The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile or +$XDG_CONFIG_HOME/parallel/sshloginfile The sshlogin '-' is special, too, it read sshlogins from stdin (standard input). diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index aeffff66..ac51ed26 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -1288,6 +1288,18 @@ GNU B can be used as a poor-man's version of ClusterSSH: B +=head2 coshell + +B only accepts full commands on standard input. Any quoting +needs to be done by the user. + +Commands are run in B so any B/B/B specific +syntax will not work. + +Output can be buffered by using B<-d>. Output is buffered in memory, +so big output can cause swapping and therefore be terrible slow or +even cause out of memory. + =head2 Todo diff --git a/src/sql b/src/sql index 0153cd33..34a7d54c 100755 --- a/src/sql +++ b/src/sql @@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20170922; + $Global::version = 20170923; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 4454edf7..029a420b 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -62,9 +62,9 @@ par_linebuffer_matters_compress_tag() { testfunc() { linebuffer="$1" - # Sleep 2 sec to give time to linebuffer-print the first part + # Sleep 3 sec to give time to linebuffer-print the first part parallel -j0 $linebuffer --compress --tag \ - "shuf $randomfile; sleep 2; shuf $randomfile; true" ::: {0..10} | + "shuf $randomfile; sleep 3; shuf $randomfile; true" ::: {0..10} | perl -ne '/^(\S+)\t/ and print "$1\n"' | uniq | sort } diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 595e1f69..7ba462fc 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -349,13 +349,13 @@ echo '{##} bug #45841: Replacement string for total no of jobs' 3 3 parallel -k -S 8/: -X --plus echo {#} {##} ::: {1..15} 1 15 15 -2 15 15 -3 15 15 -4 15 15 -5 15 15 -6 15 15 -7 15 15 -8 15 +2 14 14 +3 14 14 +4 14 14 +5 14 14 +6 14 14 +7 14 14 +8 14 echo '**' ** echo 'bug #47002: --tagstring with -d \n\n'