tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-26 07:57:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Minimize tmpfile race condition in --cat/--fifo.

Browse source
This commit is contained in:
Ole Tange 2015-04-19 02:55:17 +02:00
parent b9be3f78ba
commit d8a1dc1880
11 changed files with 122 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
src/parallel
View file

@ -1518,6 +1518,7 @@ sub read_args_from_command_line {
sub cleanup { sub cleanup {
# Returns: N/A # Returns: N/A
unlink keys %Global::unlink; unlink keys %Global::unlink;
map { rmdir $_ } keys %Global::unlink;
if(@opt::basefile) { cleanup_basefile(); } if(@opt::basefile) { cleanup_basefile(); }
} }
@ -3319,6 +3320,45 @@ sub tmpfile {
return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_); return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_);
} }
{
my($tmpdir,$fifo);
sub tmpname {
# Select a name that does not exist
# Do not create the file as that may cause problems
# if you ssh to localhost (or a shared file system) under a different name
my $name = shift;
my($tmpname);
do {
$tmpname = $ENV{'TMPDIR'}."/".$name.int(rand(9999));
} while($Global::unlink{$tmpname}++ or -e $tmpname);
# $Global::unlink{$tmpdir} ||= 1;
return $tmpname;
}
sub rmtmpname {
# Clean up mktmpname
my $tmpname = shift;
delete $Global::unlink{$tmpname};
unlink $tmpname;
rmdir $tmpdir;
}
sub tmpfifo {
# Securely make a fifo by securely making a dir with a fifo in it
use POSIX qw(mkfifo);
my $tmpfifo = tmpname("fifo",@_);
mkfifo($tmpfifo,0600);
return $tmpfifo;
}
sub rmtmpfifo {
# Clean up mktmpfifo
rmtmpname(@_);
}
}
sub uniq { sub uniq {
# Remove duplicates and return unique values # Remove duplicates and return unique values
return keys %{{ map { $_ => 1 } @_ }}; return keys %{{ map { $_ => 1 } @_ }};
@ -3506,8 +3546,8 @@ sub which {
# @full_path = full paths to @programs. Nothing if not found # @full_path = full paths to @programs. Nothing if not found
my @which; my @which;
for my $prg (@_) { for my $prg (@_) {
push @which, grep { not -d $_ and -x $_ } push(@which, grep { not -d $_ and -x $_ }
map { $_."/".$prg } split(":",$ENV{'PATH'}); map { $_."/".$prg } split(":",$ENV{'PATH'}));
} }
return @which; return @which;
} }
@ -6879,10 +6919,16 @@ sub print_dryrun_and_verbose {
my $self = shift; my $self = shift;
my $actual_command = shift; my $actual_command = shift;
# Temporary file name. Used for fifo to communicate exit val # Temporary file name. Used for fifo to communicate exit val
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx"); my ($fh, $tmpfifo) = ::tmpfile(SUFFIX => ".tmx");
$Global::unlink{$tmpfile} = 1; $Global::unlink{$tmpfifo} = 1;
close $fh; close $fh;
unlink $tmpfile; unlink $tmpfifo;
# # FIFO for communicating exit val
# my $tmpfifo = ::tmpfifo();
if(length($tmpfifo) >=100) {
::error("tmux does not support sockets with path > 100\n");
::wait_and_exit(255);
}
my $visual_command = $self->replaced(); my $visual_command = $self->replaced();
my $title = $visual_command; my $title = $visual_command;
if($visual_command =~ /\0/) { if($visual_command =~ /\0/) {
@ -6920,6 +6966,7 @@ sub print_dryrun_and_verbose {
my $tmux; my $tmux;
$ENV{'TMUX'} ||= "tmux"; $ENV{'TMUX'} ||= "tmux";
if(not $tmuxsocket) { if(not $tmuxsocket) {
# $tmuxsocket = ::tmpname("tmux");
(undef, $tmuxsocket) = ::tmpfile(SUFFIX => ".tms"); (undef, $tmuxsocket) = ::tmpfile(SUFFIX => ".tms");
$Global::unlink{$tmuxsocket} = 1; $Global::unlink{$tmuxsocket} = 1;
unlink $tmuxsocket; unlink $tmuxsocket;
@ -6933,20 +6980,20 @@ sub print_dryrun_and_verbose {
$Limits::Command::line_max_len, " tot ", $Limits::Command::line_max_len, " tot ",
$l_tot, "\n"); $l_tot, "\n");
return "mkfifo $tmpfile; $tmux ". return "mkfifo $tmpfifo; $tmux ".
# Run in tmux # Run in tmux
::shell_quote_scalar ::shell_quote_scalar
( (
"(".$actual_command.');'. "(".$actual_command.');'.
# The triple print is needed - otherwise the testsuite fails # The triple print is needed - otherwise the testsuite fails
q[ perl -e 'while($t++<3){ print $ARGV[0],"\n" }' $?h/$status >> ].$tmpfile."&". q[ perl -e 'while($t++<3){ print $ARGV[0],"\n" }' $?h/$status >> ].$tmpfifo."&".
"echo $title; echo \007Job finished at: `date`;sleep 10" "echo $title; echo \007Job finished at: `date`;sleep 10"
). ).
# Run outside tmux # Run outside tmux
# Read a / separated line: 0h/2 for csh, 2/0 for bash. # Read a / separated line: 0h/2 for csh, 2/0 for bash.
# If csh the first will be 0h, so use the second as exit value. # If csh the first will be 0h, so use the second as exit value.
# Otherwise just use the first value as exit value. # Otherwise just use the first value as exit value.
q{; exec perl -e '$/="/";$_=<>;$c=<>;unlink $ARGV; /(\d+)h/ and exit($1);exit$c' }.$tmpfile; q{; exec perl -e '$/="/";$_=<>;$c=<>;unlink $ARGV; /(\d+)h/ and exit($1);exit$c' }.$tmpfifo;
} }
} }
@ -7493,15 +7540,16 @@ sub populate {
my $next_arg; my $next_arg;
my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
if($opt::cat or $opt::fifo) { if($opt::cat) {
# Generate a tempfile name that will be used as {} # Generate a tempfile name that will be used as {}
my($outfh,$name) = ::tmpfile(SUFFIX => ".pip"); $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
close $outfh; unget([Arg->new(::tmpname("cat"))]);
# Unlink is needed if: ssh otheruser@localhost }
unlink $name; if($opt::fifo) {
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget([Arg->new($name)]); # Generate a tempfile name that will be used as {}
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
unget([Arg->new(::tmpname("fifo"))]);
} }
while (not $self->{'arg_queue'}->empty()) { while (not $self->{'arg_queue'}->empty()) {
$next_arg = $self->{'arg_queue'}->get(); $next_arg = $self->{'arg_queue'}->get();
if(not defined $next_arg) { if(not defined $next_arg) {

3
testsuite/Start.sh
View file

@ -34,6 +34,9 @@ run_test() {
} }
export -f run_test export -f run_test
# Create a monitor script
echo forever pstree -lp $$ >/tmp/monitor
chmod 755 /tmp/monitor
date date
mkdir -p actual-results mkdir -p actual-results
ls -t tests-to-run/*${1}*.sh | grep -v ${2} | ls -t tests-to-run/*${1}*.sh | grep -v ${2} |

6
testsuite/tests-to-run/parallel-local-ssh2.sh
View file

@ -41,9 +41,9 @@ echo '2bug #43358: shellshock breaks exporting functions using --env'
echo '### bug #42999: --pipepart with remote does not work' echo '### bug #42999: --pipepart with remote does not work'
seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999;
parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc;
parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:' ;
parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:' ;
rm /tmp/bug42999 rm /tmp/bug42999
echo '### --cat gives incorrect exit value in csh' echo '### --cat gives incorrect exit value in csh'

6
testsuite/tests-to-run/parallel-local22.sh
View file

@ -46,8 +46,8 @@ echo '### bug #42041: Implement $PARALLEL_JOBSLOT'
echo '### bug #42363: --pipepart and --fifo/--cat does not work' echo '### bug #42363: --pipepart and --fifo/--cat does not work'
seq 100 > /tmp/bug42363; seq 100 > /tmp/bug42363;
parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:';
parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:';
rm /tmp/bug42363 rm /tmp/bug42363
echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile' echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile'
@ -120,7 +120,7 @@ echo '### added transfersize/returnsize to local jobs'
echo '### --tmux test - check termination' echo '### --tmux test - check termination'
perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' | perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' |
stdout parallel --tmux echo {} :::: - ::: a b | stdout parallel --tmux echo {} :::: - ::: a b |
perl -pe 's:tmp.par.*tms:tmp/parXXXXX.tms:; s/\d+/0/g' perl -pe 's:/tmp/par......tms:/tmp/parXXXXX.tms:;'
EOF EOF

3
testsuite/tests-to-run/parallel-local7.sh Normal file → Executable file
View file

@ -41,8 +41,9 @@ echo '### tmux1.8'
seq 51 100 | TMUX=tmux1.8 par_tmux seq 51 100 | TMUX=tmux1.8 par_tmux
seq 101 150 | TMUX=tmux1.8 par_tmux seq 101 150 | TMUX=tmux1.8 par_tmux
seq 151 200 | TMUX=tmux1.8 par_tmux seq 151 200 | TMUX=tmux1.8 par_tmux
seq 201 233 | TMUX=tmux1.8 par_tmux seq 201 232 | TMUX=tmux1.8 par_tmux
echo '### tmux1.8 fails' echo '### tmux1.8 fails'
echo 233 | TMUX=tmux1.8 par_tmux
echo 234 | TMUX=tmux1.8 par_tmux echo 234 | TMUX=tmux1.8 par_tmux
echo 235 | TMUX=tmux1.8 par_tmux echo 235 | TMUX=tmux1.8 par_tmux
echo 236 | TMUX=tmux1.8 par_tmux echo 236 | TMUX=tmux1.8 par_tmux

4
testsuite/tests-to-run/parallel-tutorial.sh
View file

@ -22,7 +22,9 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
' | ' |
stdout parallel -j7 -vd'\n\n' | stdout parallel -j7 -vd'\n\n' |
perl -pe '$|=1; perl -pe '$|=1;
# --files and --tmux # --tmux
s:/tmp/........../t\d+:/tmp/tmuxtmp:;
# --files
s:/tmp/par......(...):/tmp/parXXXXX.$1:; s:/tmp/par......(...):/tmp/parXXXXX.$1:;
# --eta --progress # --eta --progress
s/ETA.*//g; s/local:.*//g; s/ETA.*//g; s/local:.*//g;

38
testsuite/wanted-results/parallel-local-ssh2
View file

@ -44,7 +44,7 @@ shellshock-hardened to non-shellshock-hardened
Function non-shellshock-hardened Function non-shellshock-hardened
echo '### bug #42999: --pipepart with remote does not work' echo '### bug #42999: --pipepart with remote does not work'
### bug #42999: --pipepart with remote does not work ### bug #42999: --pipepart with remote does not work
seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; rm /tmp/bug42999 seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:' ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:' ; rm /tmp/bug42999
14 14 33 14 14 33
11 11 33 11 11 33
11 11 33 11 11 33
@ -54,24 +54,24 @@ echo '### bug #42999: --pipepart with remote does not work'
11 11 33 11 11 33
11 11 33 11 11 33
9 9 28 9 9 28
14 14 33 /tmp/XXXX 14 14 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
9 9 28 /tmp/XXXX 9 9 28 /tmp/fifo0000
14 14 33 /tmp/XXXX 14 14 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
9 9 28 /tmp/XXXX 9 9 28 /tmp/cat0000
echo '### --cat gives incorrect exit value in csh' echo '### --cat gives incorrect exit value in csh'
### --cat gives incorrect exit value in csh ### --cat gives incorrect exit value in csh
echo false | parallel --pipe --cat -Scsh@lo 'cat {}; false' ; echo $?; echo false | parallel --pipe --cat -Stcsh@lo 'cat {}; false' ; echo $?; echo true | parallel --pipe --cat -Scsh@lo 'cat {}; true' ; echo $?; echo true | parallel --pipe --cat -Stcsh@lo 'cat {}; true' ; echo $?; echo '### --cat and --fifo exit value in bash' echo false | parallel --pipe --cat -Scsh@lo 'cat {}; false' ; echo $?; echo false | parallel --pipe --cat -Stcsh@lo 'cat {}; false' ; echo $?; echo true | parallel --pipe --cat -Scsh@lo 'cat {}; true' ; echo $?; echo true | parallel --pipe --cat -Stcsh@lo 'cat {}; true' ; echo $?; echo '### --cat and --fifo exit value in bash'

1
testsuite/wanted-results/parallel-local157
View file

@ -0,0 +1 @@
TEST

40
testsuite/wanted-results/parallel-local22
View file

@ -43,25 +43,25 @@ echo '### bug #42041: Implement $PARALLEL_JOBSLOT'
2 2
echo '### bug #42363: --pipepart and --fifo/--cat does not work' echo '### bug #42363: --pipepart and --fifo/--cat does not work'
### bug #42363: --pipepart and --fifo/--cat does not work ### bug #42363: --pipepart and --fifo/--cat does not work
seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; rm /tmp/bug42363 seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:'; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:'; rm /tmp/bug42363
14 14 33 /tmp/XXXX 14 14 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/fifo0000
9 9 28 /tmp/XXXX 9 9 28 /tmp/fifo0000
14 14 33 /tmp/XXXX 14 14 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
11 11 33 /tmp/XXXX 11 11 33 /tmp/cat0000
9 9 28 /tmp/XXXX 9 9 28 /tmp/cat0000
echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile' echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile'
### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile ### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile
parallel --pipepart -a /etc/passwd -L 1 should not be run parallel --pipepart -a /etc/passwd -L 1 should not be run
@ -353,5 +353,5 @@ Send Receive Exitval
Send Receive Exitval Send Receive Exitval
echo '### --tmux test - check termination' echo '### --tmux test - check termination'
### --tmux test - check termination ### --tmux test - check termination
perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' | stdout parallel --tmux echo {} :::: - ::: a b | perl -pe 's:tmp.par.*tms:tmp/parXXXXX.tms:; s/\d+/0/g' perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' | stdout parallel --tmux echo {} :::: - ::: a b | perl -pe 's:/tmp/par......tms:/tmp/parXXXXX.tms:;'
See output with: tmux -S /tmp/parXXXXX.tms attach See output with: tmux -S /tmp/parXXXXX.tms attach

5
testsuite/wanted-results/parallel-local7
View file

@ -82,11 +82,14 @@ See output with: tmux1.8 -S /tmp/parXXXXX.tms attach
seq 151 200 | TMUX=tmux1.8 par_tmux seq 151 200 | TMUX=tmux1.8 par_tmux
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach See output with: tmux1.8 -S /tmp/parXXXXX.tms attach
0 0
seq 201 233 | TMUX=tmux1.8 par_tmux seq 201 232 | TMUX=tmux1.8 par_tmux
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach See output with: tmux1.8 -S /tmp/parXXXXX.tms attach
0 0
echo '### tmux1.8 fails' echo '### tmux1.8 fails'
### tmux1.8 fails ### tmux1.8 fails
echo 233 | TMUX=tmux1.8 par_tmux
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach
0
echo 234 | TMUX=tmux1.8 par_tmux echo 234 | TMUX=tmux1.8 par_tmux
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach See output with: tmux1.8 -S /tmp/parXXXXX.tms attach
command too long command too long

2
testsuite/wanted-results/parallel-tutorial
View file

@ -406,7 +406,7 @@ B D
parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}' seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}'
See output with: tmux -S /tmp/parXXXXX.tms attach See output with: tmux -S /tmp/parXXXXX.tmux attach
tmux -S /tmp/parXXXXX.tms attach tmux -S /tmp/parXXXXX.tms attach
no sessions no sessions
parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3 parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3