Fixed bug #57364: Race condition creating len cache file.

NEWS

@@ -1,3 +1,17 @@
+20191222
+
+* GNU Parallel course in Copenhagen
+  https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-med-ole-tange/
+
+* GNU Parallel course in Århus
+  https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/
+
+* GNU Parallel is used in
+  https://github.com/JeffersonLab/rfw_tsf_extractor
+
+* Bug fixes and man page updates.
+
+
 20191122
 
 * GNU Parallel was presented at Driving IT. Slides:

README

@@ -57,11 +57,11 @@ document.
 
 Full installation of GNU Parallel is as simple as:
 
-    wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2
+    wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2
-    wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2.sig
+    wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2.sig
-    gpg parallel-20191122.tar.bz2.sig
+    gpg parallel-20191222.tar.bz2.sig
-    bzip2 -dc parallel-20191122.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20191222.tar.bz2 | tar xvf -
-    cd parallel-20191122
+    cd parallel-20191222
     ./configure && make && sudo make install
 
 
@@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as:
 If you are not root you can add ~/bin to your path and install in
 ~/bin and ~/share:
 
-    wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2
+    wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2
-    wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2.sig
+    wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2.sig
-    gpg parallel-20191122.tar.bz2.sig
+    gpg parallel-20191222.tar.bz2.sig
-    bzip2 -dc parallel-20191122.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20191222.tar.bz2 | tar xvf -
-    cd parallel-20191122
+    cd parallel-20191222
     ./configure --prefix=$HOME && make && make install
 
 Or if your system lacks 'make' you can simply copy src/parallel

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for parallel 20191122.
+# Generated by GNU Autoconf 2.69 for parallel 20191222.
 #
 # Report bugs to <bug-parallel@gnu.org>.
 #
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='parallel'
 PACKAGE_TARNAME='parallel'
-PACKAGE_VERSION='20191122'
+PACKAGE_VERSION='20191222'
-PACKAGE_STRING='parallel 20191122'
+PACKAGE_STRING='parallel 20191222'
 PACKAGE_BUGREPORT='bug-parallel@gnu.org'
 PACKAGE_URL=''
 
@@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures parallel 20191122 to adapt to many kinds of systems.
+\`configure' configures parallel 20191222 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1281,7 +1281,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of parallel 20191122:";;
+     short | recursive ) echo "Configuration of parallel 20191222:";;
    esac
   cat <<\_ACEOF
 
@@ -1357,7 +1357,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-parallel configure 20191122
+parallel configure 20191222
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by parallel $as_me 20191122, which was
+It was created by parallel $as_me 20191222, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2237,7 +2237,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='parallel'
- VERSION='20191122'
+ VERSION='20191222'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by parallel $as_me 20191122, which was
+This file was extended by parallel $as_me 20191222, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2942,7 +2942,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-parallel config.status 20191122
+parallel config.status 20191222
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20191122], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20191222], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/haikus

@@ -1,5 +1,11 @@
 Quote of the month:
 
+  GNU parallel all the way!
+    -- David Manouchehri @DaveManouchehri@twitter
+
+  I found GNU Parallel and it's awesome.
+    -- Teddy Choi @TeddyJChoi@twitter
+
   Well anyway, It was blazingly fast and astonished by performance. guess I'll never use xargs.
     -- (Not) Akaming @_Akamig@twitter
 

doc/release_new_version

@@ -209,9 +209,9 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20191122 ('Quantum Supremacy') released <<[stable]>>
+Subject: GNU Parallel 20191222 ('Impeachment') released <<[stable]>>
 
-GNU Parallel 20191122 ('Quantum Supremacy') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20191222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -224,14 +224,15 @@ Quote of the month:
 
 New in this release:
 
-* GNU Parallel was presented at Driving IT. Slides: cloud.prosa.dk/s/drivingit
+* GNU Parallel course in Copenhagen https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-med-ole-tange/
 
-* Restarting supervisord processes in parallel https://blog.naderman.de/2019/11/14/restarting-supervisord-processes-in-parallel/
+* GNU Parallel course in Århus https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/
 
-* Using GNU Parallel to Create Archives Faster https://www.reddit.com/r/DataHoarder/comments/dsgmhc/parallel_archiving_techniques/
+* GNU Parallel is used in https://github.com/JeffersonLab/rfw_tsf_extractor
 
 * Bug fixes and man page updates.
 
+
 Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
 
 GNU Parallel - For people who live life in the parallel lane.

src/niceload

@@ -23,7 +23,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20191122;
+$Global::version = 20191222;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -949,30 +949,18 @@ sub spreadstdin() {
     my $blocksize = $Global::blocksize;
     my $in = *STDIN;
     my $header = find_header(\$buf,$in);
-    my $timeout = 3;
     while(1) {
       my $anything_written = 0;
       my $buflen = length $buf;
       my $readsize = ($buflen < $blocksize) ? $blocksize-$buflen : $blocksize;
       # If $buf < $blocksize, append so it is $blocksize long after reading.
       # Otherwise append a full $blocksize
-      local $SIG{ALRM} = sub {
-	  ::set_fh_non_blocking($in);
-	  read($in,substr($buf,$buflen,0),$readsize);
-	  ::set_fh_blocking($in);
-#	  warn("ee $buf");
-	  alarm $timeout;
-      };
-    #  alarm $timeout;
       if(not read($in,substr($buf,$buflen,0),$readsize)) {
-#	  warn("ff");
 	  # End-of-file
 	  $chunk_number != 1 and last;
 	  # Force the while-loop once if everything was read by header reading
 	  $one_time_through++ and last;
       }
- #     warn("yy");
-  #    alarm 0;
       if($opt::r) {
 	  # Remove empty lines
 	  $buf =~ s/^\s*\n//gm;
@@ -2083,7 +2071,7 @@ sub check_invalid_option_combinations() {
 
 sub init_globals() {
     # Defaults:
-    $Global::version = 20191023;
+    $Global::version = 20191222;
     $Global::progname = 'parallel';
     $::name = "GNU Parallel";
     $Global::infinity = 2**31;
@@ -7966,29 +7954,6 @@ sub free_slot($) {
     }
 }
 
-{
-    my $which_sh;
-    
-    sub startshell($) {
-	my $self = shift;
-
-	# Shell must support 'exec >foo 2>bar'
-	# Using sh always will cause functions to not be exported
-	# Could perl be used instead?
-	$which_sh = $Global::cshell ? "/bin/sh" : $Global::shell;
-	my ($pid, $stdin);
-	if($pid = open($stdin, "|-", $which_sh)) {
-	    my $fileno = fileno($stdin);
-	    # Assume we get pid, fileno from spawner child
-	    $self->{'pid'} = $pid;
-	    open(my $stdin_fh, ">&=$fileno") or die ($fileno);
-	    $self->set_fh(0,"w",$stdin_fh);
-	} else {
-	    die();
-	}
-    }
-}
-    
 sub openoutputfiles($) {
     # Open files for STDOUT and STDERR
     # Set file handles in $self->fh
@@ -7999,21 +7964,20 @@ sub openoutputfiles($) {
        ($opt::keeporder or $opt::files or $opt::results or
 	$opt::compress or $opt::compress_program or
 	$opt::decompress_program)) {
+	# Do not save to files: Use non-blocking pipe
 	my ($outfhr, $errfhr);
-	$outname = ::tmpfifo();
+	pipe($outfhr, $outfhw) || die;
-	$errname = ::tmpfifo();
+	pipe($errfhr, $errfhw) || die;
-	open($outfhr,"+<",$outname);
-	open($errfhr,"+<",$errname);
-	$self->set_fh(1,'r',$outfhr);
-	$self->set_fh(2,'r',$errfhr);
-	open($outfhw,"+>",$outname);
-	open($errfhw,"+>",$errname);
 	$self->set_fh(1,'w',$outfhw);
 	$self->set_fh(2,'w',$errfhw);
+	$self->set_fh(1,'r',$outfhr);
+	$self->set_fh(2,'r',$errfhr);
 	# Make it possible to read non-blocking from the pipe
 	for my $fdno (1,2) {
 	    ::set_fh_non_blocking($self->fh($fdno,'r'));
 	}
+	# Return immediately because we do not need setting filenames
+	return;
     } elsif($opt::results and not $Global::csvsep) {
 	my $out = $self->{'commandline'}->results_out();
 	my $seqname;
@@ -8073,8 +8037,8 @@ sub openoutputfiles($) {
 	}
     } else {
 	# --ungroup
-#	open($outfhw,">&",$Global::fd{1}) || die;
+	open($outfhw,">&",$Global::fd{1}) || die;
-#	open($errfhw,">&",$Global::fd{2}) || die;
+	open($errfhw,">&",$Global::fd{2}) || die;
 	# File name must be empty as it will otherwise be printed
 	$outname = "";
 	$errname = "";
@@ -8082,31 +8046,15 @@ sub openoutputfiles($) {
 	$self->set_fh(2,"unlink",$errname);
     }
     # Set writing FD
-#    $self->set_fh(1,'w',$outfhw);
+    $self->set_fh(1,'w',$outfhw);
-#    $self->set_fh(2,'w',$errfhw);
+    $self->set_fh(2,'w',$errfhw);
     $self->set_fh(1,'name',$outname);
     $self->set_fh(2,'name',$errname);
-    
     if($opt::compress) {
 	$self->filter_through_compress();
     } elsif(not $opt::ungroup) {
 	$self->grouped();
     }
-    my $in = $self->fh(0,'w');
-    if($outname) {
-	syswrite($in,"exec >$outname\n");
-	# Must be unlinked by worker child
-	my $n = $self->fh(1,"unlink");
-	if(-e $n) { syswrite($in,"rm $n\n"); }
-    } elsif($errname) {
-	syswrite($in,"exec 2>$errname\n");
-	# Must be unlinked by worker child
-	my $n = $self->fh(2,"unlink");
-	if(-e $n) { syswrite($in,"rm $n\n"); }
-    }
-#    close $outfhw;
-#    close $errfhw;
-    
     if($opt::linebuffer) {
 	# Make it possible to read non-blocking from
 	# the buffer files
@@ -8166,7 +8114,7 @@ sub grouped($) {
 	    ::die_bug("fdr: Cannot open ".$self->fh($fdno,'name'));
 	$self->set_fh($fdno,'r',$fdr);
 	# Unlink if not debugging
-#	$Global::debug or ::rm($self->fh($fdno,"unlink"));
+	$Global::debug or ::rm($self->fh($fdno,"unlink"));
     }
 }
 
@@ -9330,23 +9278,15 @@ sub start($) {
 	open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
 	open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
 	# The eval is needed to catch exception from open3
-#	eval {
+	eval {
-#	    if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
+	    if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
-#		# Each child gets its own process group to make it safe to killall
+		# Each child gets its own process group to make it safe to killall
-#		eval{ setpgrp(0,0) };
+		eval{ setpgrp(0,0) };
-#		eval{ setpriority(0,0,$opt::nice) };
+		eval{ setpriority(0,0,$opt::nice) };
-#		exec($Global::shell,"-c",$command)
+		exec($Global::shell,"-c",$command)
-#		    || ::die_bug("open3-$stdin_fh $command");
+		    || ::die_bug("open3-$stdin_fh $command");
-#	    }
+	    }
-#	};
+	};
-#	my ($in,$out,$err);
-#	if($pid = open($in, "|-", "/bin/sh")) {
-#	    $fileno = fileno($in);
-#	}
-	# $pid $fileno
-#	$stdin_fh = $self->fh("w",0)
-#	print $stdin_fh $command;
-	
 	return $pid;
     }
 
@@ -9426,15 +9366,18 @@ sub start($) {
 	eval $redefine_eval;
     }
 
-    sub _open3_setpgrp {
+    sub open3_setpgrp {
 	my $setgprp_cache = $Global::cache_dir . "/tmp/sshlogin/" .
 	    ::hostname() . "/setpgrp_func";
-	if(-e $setgprp_cache) {
+	sub read_cache() {
+	    -e $setgprp_cache || return 0;
 	    local $/ = undef;
-	    open(my $fh, "<", $setgprp_cache) || die;
+	    open(my $fh, "<", $setgprp_cache) || return 0;
-	    eval <$fh> || die;
+	    eval <$fh> || return 0;
 	    close $fh;
-	} else {
+	    return 1;
+	}
+       	if(not read_cache()) {
 	    redefine_open3_setpgrp($setgprp_cache);
 	}
 	# The sub is now redefined. Call it
@@ -9455,47 +9398,55 @@ sub start($) {
 	# $job->skip() was called
 	$command = "true";
     }
-    $ENV{'PARALLEL_SEQ'} = $job->seq();
-    $ENV{'PARALLEL_PID'} = $$;
-    $ENV{'PARALLEL_TMP'} = ::tmpname("par");
-    $job->startshell();
     $job->openoutputfiles();
     $job->print_verbose_dryrun();
     # Call slot to store the slot value
     $job->slot();
+    my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w"));
     if($opt::dryrun or $opt::sqlmaster) { $command = "true"; }
+    $ENV{'PARALLEL_SEQ'} = $job->seq();
+    $ENV{'PARALLEL_PID'} = $$;
+    $ENV{'PARALLEL_TMP'} = ::tmpname("par");
     $job->add_rm($ENV{'PARALLEL_TMP'});
     ::debug("run", $Global::total_running, " processes . Starting (",
 	    $job->seq(), "): $command\n");
-
-    my ($stdin_fh) = $job->fh(0,"w");
-    if ($opt::tty and -c "/dev/tty" and
-	open(my $devtty_fh, "<", "/dev/tty")) {
-	# Give /dev/tty to the command if no one else is using it
-	close $devtty_fh;
-	syswrite($stdin_fh,"exec < /dev/tty\n");
-    }
-    my @setpgrp = ('exec perl','-e',
-		   ::Q("eval\{setpgrp\}\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
-		       "exec '$Global::shell', '-c', \@ARGV"));
-    syswrite($stdin_fh,"@setpgrp ".::Q($command)."\n");
-#    print $stdin_fh "@setpgrp ",::Q($command),"\n";
-    ::debug("run", "Run: $command\n");
     if($opt::pipe) {
+	my ($stdin_fh) = ::gensym();
+	$pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command);
 	if($opt::roundrobin and not $opt::keeporder) {
 	    # --keep-order will make sure the order will be reproducible
 	    ::set_fh_non_blocking($stdin_fh);
 	}
+	$job->set_fh(0,"w",$stdin_fh);
 	if($opt::tee or $opt::shard or $opt::bin) { $job->set_virgin(0); }
+    } elsif ($opt::tty and -c "/dev/tty" and
+	     open(my $devtty_fh, "<", "/dev/tty")) {
+	# Give /dev/tty to the command if no one else is using it
+	# The eval is needed to catch exception from open3
+	local (*IN,*OUT,*ERR);
+	open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
+	open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
+	*IN = $devtty_fh;
+	# The eval is needed to catch exception from open3
+	my @wrap = ('perl','-e',
+			    "eval\{setpriority\(0,0,$opt::nice\)\}\;".
+			    "exec '$Global::shell', '-c', \@ARGV");
+	eval {
+	    $pid = ::open3("<&IN", ">&OUT", ">&ERR", @wrap, $command)
+		|| ::die_bug("open3-/dev/tty");
+	    1;
+	};
+	close $devtty_fh;
+	$job->set_virgin(0);
     } else {
-	# Close stdin if not pipe input
+	$pid = open3_setpgrp(::gensym(),$stdout_fh,$stderr_fh,$command);
-	close $stdin_fh;
 	$job->set_virgin(0);
     }
-    if($job->{'pid'}) {
+    if($pid) {
 	# A job was started
 	$Global::total_running++;
 	$Global::total_started++;
+	$job->set_pid($pid);
 	$job->set_starttime();
 	$Global::running{$job->pid()} = $job;
 	if($opt::timeout) {
@@ -10065,7 +10016,7 @@ sub print_normal($) {
     my $self = shift;
     my ($fdno,$in_fh,$out_fd) = @_;
     my $buf;
-    #close $self->fh($fdno,"w");
+    close $self->fh($fdno,"w");
     if($? and $opt::compress) {
 	::error($opt::compress_program." failed.");
 	$self->set_exitstatus(255);

src/sql

@@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20191122;
+    $Global::version = 20191222;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/parallel-local-30s.sh

@@ -4,6 +4,19 @@
 # Each should be taking 30-100s and be possible to run in parallel
 # I.e.: No race conditions, no logins
 
+par_bug57364() {
+    echo '### bug #57364: Race condition creating len cache file.'
+    j=32
+    set -e
+    for i in $(seq 1 50); do
+        # Clear cache.
+        rm -rf "${HOME}/.parallel/tmp"
+        # Try to launch multiple parallel simultaneously.
+        seq $j |
+            xargs -P $j -n 1 parallel true $i :::
+    done 2>&1
+}
+
 par_sighup() {
     echo '### Test SIGHUP'
     parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &

testsuite/wanted-results/parallel-local-30s

@@ -1,3 +1,4 @@
+par_bug57364	### bug #57364: Race condition creating len cache file.
 par_keeporder_roundrobin	bug #50081: --keep-order --round-robin should give predictable results
 par_keeporder_roundrobin	OK
 par_linebuffer_files	bug #48658: --linebuffer --files