From a785e66b6c7d3c6f445c86ca294b79026baef147 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 22 Dec 2019 15:41:03 +0100 Subject: [PATCH] Fixed bug #57364: Race condition creating len cache file. --- Makefile.in | 8 +- NEWS | 14 ++ README | 20 +-- configure | 20 +-- configure.ac | 2 +- doc/haikus | 6 + doc/release_new_version | 11 +- src/niceload | 2 +- src/parallel | 169 +++++++------------ src/sql | 2 +- testsuite/tests-to-run/parallel-local-30s.sh | 13 ++ testsuite/wanted-results/parallel-local-30s | 1 + 12 files changed, 127 insertions(+), 141 deletions(-) diff --git a/Makefile.in b/Makefile.in index 7b84287a..538c1ea4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -767,8 +767,8 @@ upload: (echo '#!/bin/bash'; \ echo; \ echo "# To check the signature run:"; \ - echo "# echo | gpg"; \ - echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \ echo; \ echo "echo | gpg 2>/dev/null"; \ echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ @@ -780,8 +780,8 @@ upload: (echo '#!/bin/bash'; \ echo; \ echo "# To check the signature run:"; \ - echo "# echo | gpg"; \ - echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \ echo; \ echo "echo | gpg 2>/dev/null"; \ echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ diff --git a/NEWS b/NEWS index 7eee4f75..da8edc1a 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,17 @@ +20191222 + +* GNU Parallel course in Copenhagen + https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-med-ole-tange/ + +* GNU Parallel course in Århus + https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/ + +* GNU Parallel is used in + https://github.com/JeffersonLab/rfw_tsf_extractor + +* Bug fixes and man page updates. + + 20191122 * GNU Parallel was presented at Driving IT. Slides: diff --git a/README b/README index e358fb20..21ebe960 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2.sig - gpg parallel-20191122.tar.bz2.sig - bzip2 -dc parallel-20191122.tar.bz2 | tar xvf - - cd parallel-20191122 + wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2.sig + gpg parallel-20191222.tar.bz2.sig + bzip2 -dc parallel-20191222.tar.bz2 | tar xvf - + cd parallel-20191222 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2.sig - gpg parallel-20191122.tar.bz2.sig - bzip2 -dc parallel-20191122.tar.bz2 | tar xvf - - cd parallel-20191122 + wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2.sig + gpg parallel-20191222.tar.bz2.sig + bzip2 -dc parallel-20191222.tar.bz2 | tar xvf - + cd parallel-20191222 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index af9b1f2e..812b7eba 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20191122. +# Generated by GNU Autoconf 2.69 for parallel 20191222. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20191122' -PACKAGE_STRING='parallel 20191122' +PACKAGE_VERSION='20191222' +PACKAGE_STRING='parallel 20191222' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20191122 to adapt to many kinds of systems. +\`configure' configures parallel 20191222 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20191122:";; + short | recursive ) echo "Configuration of parallel 20191222:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20191122 +parallel configure 20191222 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20191122, which was +It was created by parallel $as_me 20191222, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20191122' + VERSION='20191222' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20191122, which was +This file was extended by parallel $as_me 20191222, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20191122 +parallel config.status 20191222 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index eb94221c..cd0a89df 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20191122], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20191222], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 965937bd..1c83aa96 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,5 +1,11 @@ Quote of the month: + GNU parallel all the way! + -- David Manouchehri @DaveManouchehri@twitter + + I found GNU Parallel and it's awesome. + -- Teddy Choi @TeddyJChoi@twitter + Well anyway, It was blazingly fast and astonished by performance. guess I'll never use xargs. -- (Not) Akaming @_Akamig@twitter diff --git a/doc/release_new_version b/doc/release_new_version index b603b6e1..68cdb901 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -209,9 +209,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20191122 ('Quantum Supremacy') released <<[stable]>> +Subject: GNU Parallel 20191222 ('Impeachment') released <<[stable]>> -GNU Parallel 20191122 ('Quantum Supremacy') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20191222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -224,14 +224,15 @@ Quote of the month: New in this release: -* GNU Parallel was presented at Driving IT. Slides: cloud.prosa.dk/s/drivingit +* GNU Parallel course in Copenhagen https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-med-ole-tange/ -* Restarting supervisord processes in parallel https://blog.naderman.de/2019/11/14/restarting-supervisord-processes-in-parallel/ +* GNU Parallel course in Århus https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/ -* Using GNU Parallel to Create Archives Faster https://www.reddit.com/r/DataHoarder/comments/dsgmhc/parallel_archiving_techniques/ +* GNU Parallel is used in https://github.com/JeffersonLab/rfw_tsf_extractor * Bug fixes and man page updates. + Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html GNU Parallel - For people who live life in the parallel lane. diff --git a/src/niceload b/src/niceload index ca7cd156..012e9f9b 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20191122; +$Global::version = 20191222; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index f2c8c4e6..48977448 100755 --- a/src/parallel +++ b/src/parallel @@ -949,30 +949,18 @@ sub spreadstdin() { my $blocksize = $Global::blocksize; my $in = *STDIN; my $header = find_header(\$buf,$in); - my $timeout = 3; while(1) { my $anything_written = 0; my $buflen = length $buf; my $readsize = ($buflen < $blocksize) ? $blocksize-$buflen : $blocksize; # If $buf < $blocksize, append so it is $blocksize long after reading. # Otherwise append a full $blocksize - local $SIG{ALRM} = sub { - ::set_fh_non_blocking($in); - read($in,substr($buf,$buflen,0),$readsize); - ::set_fh_blocking($in); -# warn("ee $buf"); - alarm $timeout; - }; - # alarm $timeout; if(not read($in,substr($buf,$buflen,0),$readsize)) { -# warn("ff"); # End-of-file $chunk_number != 1 and last; # Force the while-loop once if everything was read by header reading $one_time_through++ and last; } - # warn("yy"); - # alarm 0; if($opt::r) { # Remove empty lines $buf =~ s/^\s*\n//gm; @@ -2083,7 +2071,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20191023; + $Global::version = 20191222; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -7966,29 +7954,6 @@ sub free_slot($) { } } -{ - my $which_sh; - - sub startshell($) { - my $self = shift; - - # Shell must support 'exec >foo 2>bar' - # Using sh always will cause functions to not be exported - # Could perl be used instead? - $which_sh = $Global::cshell ? "/bin/sh" : $Global::shell; - my ($pid, $stdin); - if($pid = open($stdin, "|-", $which_sh)) { - my $fileno = fileno($stdin); - # Assume we get pid, fileno from spawner child - $self->{'pid'} = $pid; - open(my $stdin_fh, ">&=$fileno") or die ($fileno); - $self->set_fh(0,"w",$stdin_fh); - } else { - die(); - } - } -} - sub openoutputfiles($) { # Open files for STDOUT and STDERR # Set file handles in $self->fh @@ -7999,21 +7964,20 @@ sub openoutputfiles($) { ($opt::keeporder or $opt::files or $opt::results or $opt::compress or $opt::compress_program or $opt::decompress_program)) { + # Do not save to files: Use non-blocking pipe my ($outfhr, $errfhr); - $outname = ::tmpfifo(); - $errname = ::tmpfifo(); - open($outfhr,"+<",$outname); - open($errfhr,"+<",$errname); - $self->set_fh(1,'r',$outfhr); - $self->set_fh(2,'r',$errfhr); - open($outfhw,"+>",$outname); - open($errfhw,"+>",$errname); + pipe($outfhr, $outfhw) || die; + pipe($errfhr, $errfhw) || die; $self->set_fh(1,'w',$outfhw); $self->set_fh(2,'w',$errfhw); + $self->set_fh(1,'r',$outfhr); + $self->set_fh(2,'r',$errfhr); # Make it possible to read non-blocking from the pipe for my $fdno (1,2) { ::set_fh_non_blocking($self->fh($fdno,'r')); } + # Return immediately because we do not need setting filenames + return; } elsif($opt::results and not $Global::csvsep) { my $out = $self->{'commandline'}->results_out(); my $seqname; @@ -8073,8 +8037,8 @@ sub openoutputfiles($) { } } else { # --ungroup -# open($outfhw,">&",$Global::fd{1}) || die; -# open($errfhw,">&",$Global::fd{2}) || die; + open($outfhw,">&",$Global::fd{1}) || die; + open($errfhw,">&",$Global::fd{2}) || die; # File name must be empty as it will otherwise be printed $outname = ""; $errname = ""; @@ -8082,31 +8046,15 @@ sub openoutputfiles($) { $self->set_fh(2,"unlink",$errname); } # Set writing FD -# $self->set_fh(1,'w',$outfhw); -# $self->set_fh(2,'w',$errfhw); + $self->set_fh(1,'w',$outfhw); + $self->set_fh(2,'w',$errfhw); $self->set_fh(1,'name',$outname); $self->set_fh(2,'name',$errname); - if($opt::compress) { $self->filter_through_compress(); } elsif(not $opt::ungroup) { $self->grouped(); } - my $in = $self->fh(0,'w'); - if($outname) { - syswrite($in,"exec >$outname\n"); - # Must be unlinked by worker child - my $n = $self->fh(1,"unlink"); - if(-e $n) { syswrite($in,"rm $n\n"); } - } elsif($errname) { - syswrite($in,"exec 2>$errname\n"); - # Must be unlinked by worker child - my $n = $self->fh(2,"unlink"); - if(-e $n) { syswrite($in,"rm $n\n"); } - } -# close $outfhw; -# close $errfhw; - if($opt::linebuffer) { # Make it possible to read non-blocking from # the buffer files @@ -8166,7 +8114,7 @@ sub grouped($) { ::die_bug("fdr: Cannot open ".$self->fh($fdno,'name')); $self->set_fh($fdno,'r',$fdr); # Unlink if not debugging -# $Global::debug or ::rm($self->fh($fdno,"unlink")); + $Global::debug or ::rm($self->fh($fdno,"unlink")); } } @@ -9330,23 +9278,15 @@ sub start($) { open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!"); open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!"); # The eval is needed to catch exception from open3 -# eval { -# if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) { -# # Each child gets its own process group to make it safe to killall -# eval{ setpgrp(0,0) }; -# eval{ setpriority(0,0,$opt::nice) }; -# exec($Global::shell,"-c",$command) -# || ::die_bug("open3-$stdin_fh $command"); -# } -# }; -# my ($in,$out,$err); -# if($pid = open($in, "|-", "/bin/sh")) { -# $fileno = fileno($in); -# } - # $pid $fileno -# $stdin_fh = $self->fh("w",0) -# print $stdin_fh $command; - + eval { + if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) { + # Each child gets its own process group to make it safe to killall + eval{ setpgrp(0,0) }; + eval{ setpriority(0,0,$opt::nice) }; + exec($Global::shell,"-c",$command) + || ::die_bug("open3-$stdin_fh $command"); + } + }; return $pid; } @@ -9426,15 +9366,18 @@ sub start($) { eval $redefine_eval; } - sub _open3_setpgrp { + sub open3_setpgrp { my $setgprp_cache = $Global::cache_dir . "/tmp/sshlogin/" . ::hostname() . "/setpgrp_func"; - if(-e $setgprp_cache) { + sub read_cache() { + -e $setgprp_cache || return 0; local $/ = undef; - open(my $fh, "<", $setgprp_cache) || die; - eval <$fh> || die; + open(my $fh, "<", $setgprp_cache) || return 0; + eval <$fh> || return 0; close $fh; - } else { + return 1; + } + if(not read_cache()) { redefine_open3_setpgrp($setgprp_cache); } # The sub is now redefined. Call it @@ -9455,47 +9398,55 @@ sub start($) { # $job->skip() was called $command = "true"; } - $ENV{'PARALLEL_SEQ'} = $job->seq(); - $ENV{'PARALLEL_PID'} = $$; - $ENV{'PARALLEL_TMP'} = ::tmpname("par"); - $job->startshell(); $job->openoutputfiles(); $job->print_verbose_dryrun(); # Call slot to store the slot value $job->slot(); + my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w")); if($opt::dryrun or $opt::sqlmaster) { $command = "true"; } + $ENV{'PARALLEL_SEQ'} = $job->seq(); + $ENV{'PARALLEL_PID'} = $$; + $ENV{'PARALLEL_TMP'} = ::tmpname("par"); $job->add_rm($ENV{'PARALLEL_TMP'}); ::debug("run", $Global::total_running, " processes . Starting (", $job->seq(), "): $command\n"); - - my ($stdin_fh) = $job->fh(0,"w"); - if ($opt::tty and -c "/dev/tty" and - open(my $devtty_fh, "<", "/dev/tty")) { - # Give /dev/tty to the command if no one else is using it - close $devtty_fh; - syswrite($stdin_fh,"exec < /dev/tty\n"); - } - my @setpgrp = ('exec perl','-e', - ::Q("eval\{setpgrp\}\;eval\{setpriority\(0,0,$opt::nice\)\}\;". - "exec '$Global::shell', '-c', \@ARGV")); - syswrite($stdin_fh,"@setpgrp ".::Q($command)."\n"); -# print $stdin_fh "@setpgrp ",::Q($command),"\n"; - ::debug("run", "Run: $command\n"); if($opt::pipe) { + my ($stdin_fh) = ::gensym(); + $pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command); if($opt::roundrobin and not $opt::keeporder) { # --keep-order will make sure the order will be reproducible ::set_fh_non_blocking($stdin_fh); } + $job->set_fh(0,"w",$stdin_fh); if($opt::tee or $opt::shard or $opt::bin) { $job->set_virgin(0); } + } elsif ($opt::tty and -c "/dev/tty" and + open(my $devtty_fh, "<", "/dev/tty")) { + # Give /dev/tty to the command if no one else is using it + # The eval is needed to catch exception from open3 + local (*IN,*OUT,*ERR); + open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!"); + open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!"); + *IN = $devtty_fh; + # The eval is needed to catch exception from open3 + my @wrap = ('perl','-e', + "eval\{setpriority\(0,0,$opt::nice\)\}\;". + "exec '$Global::shell', '-c', \@ARGV"); + eval { + $pid = ::open3("<&IN", ">&OUT", ">&ERR", @wrap, $command) + || ::die_bug("open3-/dev/tty"); + 1; + }; + close $devtty_fh; + $job->set_virgin(0); } else { - # Close stdin if not pipe input - close $stdin_fh; + $pid = open3_setpgrp(::gensym(),$stdout_fh,$stderr_fh,$command); $job->set_virgin(0); } - if($job->{'pid'}) { + if($pid) { # A job was started $Global::total_running++; $Global::total_started++; + $job->set_pid($pid); $job->set_starttime(); $Global::running{$job->pid()} = $job; if($opt::timeout) { @@ -10065,7 +10016,7 @@ sub print_normal($) { my $self = shift; my ($fdno,$in_fh,$out_fd) = @_; my $buf; - #close $self->fh($fdno,"w"); + close $self->fh($fdno,"w"); if($? and $opt::compress) { ::error($opt::compress_program." failed."); $self->set_exitstatus(255); diff --git a/src/sql b/src/sql index 0762c977..bccbec6f 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20191122; + $Global::version = 20191222; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 7e4fd02d..37eedabd 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -4,6 +4,19 @@ # Each should be taking 30-100s and be possible to run in parallel # I.e.: No race conditions, no logins +par_bug57364() { + echo '### bug #57364: Race condition creating len cache file.' + j=32 + set -e + for i in $(seq 1 50); do + # Clear cache. + rm -rf "${HOME}/.parallel/tmp" + # Try to launch multiple parallel simultaneously. + seq $j | + xargs -P $j -n 1 parallel true $i ::: + done 2>&1 +} + par_sighup() { echo '### Test SIGHUP' parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 & diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index b694058d..157e4134 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -1,3 +1,4 @@ +par_bug57364 ### bug #57364: Race condition creating len cache file. par_keeporder_roundrobin bug #50081: --keep-order --round-robin should give predictable results par_keeporder_roundrobin OK par_linebuffer_files bug #48658: --linebuffer --files