tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-10-18 04:49:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #57364: Race condition creating len cache file.

Browse source
This commit is contained in:
Ole Tange 2019-12-22 15:41:03 +01:00
parent f1e58af040
commit a785e66b6c
12 changed files with 127 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Makefile.in
View file

@ -767,8 +767,8 @@ upload:
(echo '#!/bin/bash'; \
echo; \
echo "# To check the signature run:"; \
echo "# echo | gpg"; \
echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \
echo "# echo | gpg"; \
echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \
echo; \
echo "echo | gpg 2>/dev/null"; \
echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \
@ -780,8 +780,8 @@ upload:
(echo '#!/bin/bash'; \
echo; \
echo "# To check the signature run:"; \
echo "# echo | gpg"; \
echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \
echo "# echo | gpg"; \
echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \
echo; \
echo "echo | gpg 2>/dev/null"; \
echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \

14
NEWS
View file

@ -1,3 +1,17 @@
20191222
* GNU Parallel course in Copenhagen
https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-med-ole-tange/
* GNU Parallel course in Århus
https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/
* GNU Parallel is used in
https://github.com/JeffersonLab/rfw_tsf_extractor
* Bug fixes and man page updates.
20191122
* GNU Parallel was presented at Driving IT. Slides:

20
README
View file

@ -57,11 +57,11 @@ document.
Full installation of GNU Parallel is as simple as:
wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2
wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2.sig
gpg parallel-20191122.tar.bz2.sig
bzip2 -dc parallel-20191122.tar.bz2 | tar xvf -
cd parallel-20191122
wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2
wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2.sig
gpg parallel-20191222.tar.bz2.sig
bzip2 -dc parallel-20191222.tar.bz2 | tar xvf -
cd parallel-20191222
./configure && make && sudo make install
@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as:
If you are not root you can add ~/bin to your path and install in
~/bin and ~/share:
wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2
wget https://ftpmirror.gnu.org/parallel/parallel-20191122.tar.bz2.sig
gpg parallel-20191122.tar.bz2.sig
bzip2 -dc parallel-20191122.tar.bz2 | tar xvf -
cd parallel-20191122
wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2
wget https://ftpmirror.gnu.org/parallel/parallel-20191222.tar.bz2.sig
gpg parallel-20191222.tar.bz2.sig
bzip2 -dc parallel-20191222.tar.bz2 | tar xvf -
cd parallel-20191222
./configure --prefix=$HOME && make && make install
Or if your system lacks 'make' you can simply copy src/parallel

20
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for parallel 20191122.
# Generated by GNU Autoconf 2.69 for parallel 20191222.
#
# Report bugs to <bug-parallel@gnu.org>.
#
@ -579,8 +579,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20191122'
PACKAGE_STRING='parallel 20191122'
PACKAGE_VERSION='20191222'
PACKAGE_STRING='parallel 20191222'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20191122 to adapt to many kinds of systems.
\`configure' configures parallel 20191222 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1281,7 +1281,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20191122:";;
short | recursive ) echo "Configuration of parallel 20191222:";;
esac
cat <<\_ACEOF
@ -1357,7 +1357,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20191122
parallel configure 20191222
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20191122, which was
It was created by parallel $as_me 20191222, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -2237,7 +2237,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20191122'
VERSION='20191222'
cat >>confdefs.h <<_ACEOF
@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20191122, which was
This file was extended by parallel $as_me 20191222, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2942,7 +2942,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20191122
parallel config.status 20191222
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

2
configure.ac
View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20191122], [bug-parallel@gnu.org])
AC_INIT([parallel], [20191222], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([

6
doc/haikus
View file

@ -1,5 +1,11 @@
Quote of the month:
GNU parallel all the way!
-- David Manouchehri @DaveManouchehri@twitter
I found GNU Parallel and it's awesome.
-- Teddy Choi @TeddyJChoi@twitter
Well anyway, It was blazingly fast and astonished by performance. guess I'll never use xargs.
-- (Not) Akaming @_Akamig@twitter

11
doc/release_new_version
View file

@ -209,9 +209,9 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20191122 ('Quantum Supremacy') released <<[stable]>>
Subject: GNU Parallel 20191222 ('Impeachment') released <<[stable]>>
GNU Parallel 20191122 ('Quantum Supremacy') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20191222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -224,14 +224,15 @@ Quote of the month:
New in this release:
* GNU Parallel was presented at Driving IT. Slides: cloud.prosa.dk/s/drivingit
* GNU Parallel course in Copenhagen https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-med-ole-tange/
* Restarting supervisord processes in parallel https://blog.naderman.de/2019/11/14/restarting-supervisord-processes-in-parallel/
* GNU Parallel course in Århus https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/
* Using GNU Parallel to Create Archives Faster https://www.reddit.com/r/DataHoarder/comments/dsgmhc/parallel_archiving_techniques/
* GNU Parallel is used in https://github.com/JeffersonLab/rfw_tsf_extractor
* Bug fixes and man page updates.
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
GNU Parallel - For people who live life in the parallel lane.

2
src/niceload
View file

@ -23,7 +23,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20191122;
$Global::version = 20191222;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($opt::version) {

169
src/parallel
View file

@ -949,30 +949,18 @@ sub spreadstdin() {
my $blocksize = $Global::blocksize;
my $in = *STDIN;
my $header = find_header(\$buf,$in);
my $timeout = 3;
while(1) {
my $anything_written = 0;
my $buflen = length $buf;
my $readsize = ($buflen < $blocksize) ? $blocksize-$buflen : $blocksize;
# If $buf < $blocksize, append so it is $blocksize long after reading.
# Otherwise append a full $blocksize
local $SIG{ALRM} = sub {
::set_fh_non_blocking($in);
read($in,substr($buf,$buflen,0),$readsize);
::set_fh_blocking($in);
# warn("ee $buf");
alarm $timeout;
};
# alarm $timeout;
if(not read($in,substr($buf,$buflen,0),$readsize)) {
# warn("ff");
# End-of-file
$chunk_number != 1 and last;
# Force the while-loop once if everything was read by header reading
$one_time_through++ and last;
}
# warn("yy");
# alarm 0;
if($opt::r) {
# Remove empty lines
$buf =~ s/^\s*\n//gm;
@ -2083,7 +2071,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20191023;
$Global::version = 20191222;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -7966,29 +7954,6 @@ sub free_slot($) {
}
}
{
my $which_sh;
sub startshell($) {
my $self = shift;
# Shell must support 'exec >foo 2>bar'
# Using sh always will cause functions to not be exported
# Could perl be used instead?
$which_sh = $Global::cshell ? "/bin/sh" : $Global::shell;
my ($pid, $stdin);
if($pid = open($stdin, "|-", $which_sh)) {
my $fileno = fileno($stdin);
# Assume we get pid, fileno from spawner child
$self->{'pid'} = $pid;
open(my $stdin_fh, ">&=$fileno") or die ($fileno);
$self->set_fh(0,"w",$stdin_fh);
} else {
die();
}
}
}
sub openoutputfiles($) {
# Open files for STDOUT and STDERR
# Set file handles in $self->fh
@ -7999,21 +7964,20 @@ sub openoutputfiles($) {
($opt::keeporder or $opt::files or $opt::results or
$opt::compress or $opt::compress_program or
$opt::decompress_program)) {
# Do not save to files: Use non-blocking pipe
my ($outfhr, $errfhr);
$outname = ::tmpfifo();
$errname = ::tmpfifo();
open($outfhr,"+<",$outname);
open($errfhr,"+<",$errname);
$self->set_fh(1,'r',$outfhr);
$self->set_fh(2,'r',$errfhr);
open($outfhw,"+>",$outname);
open($errfhw,"+>",$errname);
pipe($outfhr, $outfhw) || die;
pipe($errfhr, $errfhw) || die;
$self->set_fh(1,'w',$outfhw);
$self->set_fh(2,'w',$errfhw);
$self->set_fh(1,'r',$outfhr);
$self->set_fh(2,'r',$errfhr);
# Make it possible to read non-blocking from the pipe
for my $fdno (1,2) {
::set_fh_non_blocking($self->fh($fdno,'r'));
}
# Return immediately because we do not need setting filenames
return;
} elsif($opt::results and not $Global::csvsep) {
my $out = $self->{'commandline'}->results_out();
my $seqname;
@ -8073,8 +8037,8 @@ sub openoutputfiles($) {
}
} else {
# --ungroup
# open($outfhw,">&",$Global::fd{1}) || die;
# open($errfhw,">&",$Global::fd{2}) || die;
open($outfhw,">&",$Global::fd{1}) || die;
open($errfhw,">&",$Global::fd{2}) || die;
# File name must be empty as it will otherwise be printed
$outname = "";
$errname = "";
@ -8082,31 +8046,15 @@ sub openoutputfiles($) {
$self->set_fh(2,"unlink",$errname);
}
# Set writing FD
# $self->set_fh(1,'w',$outfhw);
# $self->set_fh(2,'w',$errfhw);
$self->set_fh(1,'w',$outfhw);
$self->set_fh(2,'w',$errfhw);
$self->set_fh(1,'name',$outname);
$self->set_fh(2,'name',$errname);
if($opt::compress) {
$self->filter_through_compress();
} elsif(not $opt::ungroup) {
$self->grouped();
}
my $in = $self->fh(0,'w');
if($outname) {
syswrite($in,"exec >$outname\n");
# Must be unlinked by worker child
my $n = $self->fh(1,"unlink");
if(-e $n) { syswrite($in,"rm $n\n"); }
} elsif($errname) {
syswrite($in,"exec 2>$errname\n");
# Must be unlinked by worker child
my $n = $self->fh(2,"unlink");
if(-e $n) { syswrite($in,"rm $n\n"); }
}
# close $outfhw;
# close $errfhw;
if($opt::linebuffer) {
# Make it possible to read non-blocking from
# the buffer files
@ -8166,7 +8114,7 @@ sub grouped($) {
::die_bug("fdr: Cannot open ".$self->fh($fdno,'name'));
$self->set_fh($fdno,'r',$fdr);
# Unlink if not debugging
# $Global::debug or ::rm($self->fh($fdno,"unlink"));
$Global::debug or ::rm($self->fh($fdno,"unlink"));
}
}
@ -9330,23 +9278,15 @@ sub start($) {
open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
# The eval is needed to catch exception from open3
# eval {
# if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
# # Each child gets its own process group to make it safe to killall
# eval{ setpgrp(0,0) };
# eval{ setpriority(0,0,$opt::nice) };
# exec($Global::shell,"-c",$command)
# || ::die_bug("open3-$stdin_fh $command");
# }
# };
# my ($in,$out,$err);
# if($pid = open($in, "|-", "/bin/sh")) {
# $fileno = fileno($in);
# }
# $pid $fileno
# $stdin_fh = $self->fh("w",0)
# print $stdin_fh $command;
eval {
if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
# Each child gets its own process group to make it safe to killall
eval{ setpgrp(0,0) };
eval{ setpriority(0,0,$opt::nice) };
exec($Global::shell,"-c",$command)
|| ::die_bug("open3-$stdin_fh $command");
}
};
return $pid;
}
@ -9426,15 +9366,18 @@ sub start($) {
eval $redefine_eval;
}
sub _open3_setpgrp {
sub open3_setpgrp {
my $setgprp_cache = $Global::cache_dir . "/tmp/sshlogin/" .
::hostname() . "/setpgrp_func";
if(-e $setgprp_cache) {
sub read_cache() {
-e $setgprp_cache || return 0;
local $/ = undef;
open(my $fh, "<", $setgprp_cache) || die;
eval <$fh> || die;
open(my $fh, "<", $setgprp_cache) || return 0;
eval <$fh> || return 0;
close $fh;
} else {
return 1;
}
if(not read_cache()) {
redefine_open3_setpgrp($setgprp_cache);
}
# The sub is now redefined. Call it
@ -9455,47 +9398,55 @@ sub start($) {
# $job->skip() was called
$command = "true";
}
$ENV{'PARALLEL_SEQ'} = $job->seq();
$ENV{'PARALLEL_PID'} = $$;
$ENV{'PARALLEL_TMP'} = ::tmpname("par");
$job->startshell();
$job->openoutputfiles();
$job->print_verbose_dryrun();
# Call slot to store the slot value
$job->slot();
my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w"));
if($opt::dryrun or $opt::sqlmaster) { $command = "true"; }
$ENV{'PARALLEL_SEQ'} = $job->seq();
$ENV{'PARALLEL_PID'} = $$;
$ENV{'PARALLEL_TMP'} = ::tmpname("par");
$job->add_rm($ENV{'PARALLEL_TMP'});
::debug("run", $Global::total_running, " processes . Starting (",
$job->seq(), "): $command\n");
my ($stdin_fh) = $job->fh(0,"w");
if ($opt::tty and -c "/dev/tty" and
open(my $devtty_fh, "<", "/dev/tty")) {
# Give /dev/tty to the command if no one else is using it
close $devtty_fh;
syswrite($stdin_fh,"exec < /dev/tty\n");
}
my @setpgrp = ('exec perl','-e',
::Q("eval\{setpgrp\}\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
"exec '$Global::shell', '-c', \@ARGV"));
syswrite($stdin_fh,"@setpgrp ".::Q($command)."\n");
# print $stdin_fh "@setpgrp ",::Q($command),"\n";
::debug("run", "Run: $command\n");
if($opt::pipe) {
my ($stdin_fh) = ::gensym();
$pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command);
if($opt::roundrobin and not $opt::keeporder) {
# --keep-order will make sure the order will be reproducible
::set_fh_non_blocking($stdin_fh);
}
$job->set_fh(0,"w",$stdin_fh);
if($opt::tee or $opt::shard or $opt::bin) { $job->set_virgin(0); }
} elsif ($opt::tty and -c "/dev/tty" and
open(my $devtty_fh, "<", "/dev/tty")) {
# Give /dev/tty to the command if no one else is using it
# The eval is needed to catch exception from open3
local (*IN,*OUT,*ERR);
open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
*IN = $devtty_fh;
# The eval is needed to catch exception from open3
my @wrap = ('perl','-e',
"eval\{setpriority\(0,0,$opt::nice\)\}\;".
"exec '$Global::shell', '-c', \@ARGV");
eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", @wrap, $command)
|| ::die_bug("open3-/dev/tty");
1;
};
close $devtty_fh;
$job->set_virgin(0);
} else {
# Close stdin if not pipe input
close $stdin_fh;
$pid = open3_setpgrp(::gensym(),$stdout_fh,$stderr_fh,$command);
$job->set_virgin(0);
}
if($job->{'pid'}) {
if($pid) {
# A job was started
$Global::total_running++;
$Global::total_started++;
$job->set_pid($pid);
$job->set_starttime();
$Global::running{$job->pid()} = $job;
if($opt::timeout) {
@ -10065,7 +10016,7 @@ sub print_normal($) {
my $self = shift;
my ($fdno,$in_fh,$out_fd) = @_;
my $buf;
#close $self->fh($fdno,"w");
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::compress_program." failed.");
$self->set_exitstatus(255);

2
src/sql
View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20191122;
$Global::version = 20191222;
$Global::progname = 'sql';
# This must be done first as this may exec myself

13
testsuite/tests-to-run/parallel-local-30s.sh
View file

@ -4,6 +4,19 @@
# Each should be taking 30-100s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_bug57364() {
echo '### bug #57364: Race condition creating len cache file.'
j=32
set -e
for i in $(seq 1 50); do
# Clear cache.
rm -rf "${HOME}/.parallel/tmp"
# Try to launch multiple parallel simultaneously.
seq $j |
xargs -P $j -n 1 parallel true $i :::
done 2>&1
}
par_sighup() {
echo '### Test SIGHUP'
parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &

1
testsuite/wanted-results/parallel-local-30s
View file

@ -1,3 +1,4 @@
par_bug57364 ### bug #57364: Race condition creating len cache file.
par_keeporder_roundrobin bug #50081: --keep-order --round-robin should give predictable results
par_keeporder_roundrobin OK
par_linebuffer_files bug #48658: --linebuffer --files