From d8a1dc18804873ebedb7162cd85876d928404931 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 19 Apr 2015 02:55:17 +0200 Subject: [PATCH] parallel: Minimize tmpfile race condition in --cat/--fifo. --- src/parallel | 78 +++++++++++++++---- testsuite/Start.sh | 3 + testsuite/tests-to-run/parallel-local-ssh2.sh | 6 +- testsuite/tests-to-run/parallel-local22.sh | 6 +- testsuite/tests-to-run/parallel-local7.sh | 3 +- testsuite/tests-to-run/parallel-tutorial.sh | 4 +- testsuite/wanted-results/parallel-local-ssh2 | 38 ++++----- testsuite/wanted-results/parallel-local157 | 1 + testsuite/wanted-results/parallel-local22 | 40 +++++----- testsuite/wanted-results/parallel-local7 | 5 +- testsuite/wanted-results/parallel-tutorial | 2 +- 11 files changed, 122 insertions(+), 64 deletions(-) mode change 100644 => 100755 testsuite/tests-to-run/parallel-local7.sh diff --git a/src/parallel b/src/parallel index 8b740a89..44f2d80f 100755 --- a/src/parallel +++ b/src/parallel @@ -1518,6 +1518,7 @@ sub read_args_from_command_line { sub cleanup { # Returns: N/A unlink keys %Global::unlink; + map { rmdir $_ } keys %Global::unlink; if(@opt::basefile) { cleanup_basefile(); } } @@ -3319,6 +3320,45 @@ sub tmpfile { return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_); } + +{ + my($tmpdir,$fifo); + + sub tmpname { + # Select a name that does not exist + # Do not create the file as that may cause problems + # if you ssh to localhost (or a shared file system) under a different name + my $name = shift; + my($tmpname); + do { + $tmpname = $ENV{'TMPDIR'}."/".$name.int(rand(9999)); + } while($Global::unlink{$tmpname}++ or -e $tmpname); +# $Global::unlink{$tmpdir} ||= 1; + return $tmpname; + } + + sub rmtmpname { + # Clean up mktmpname + my $tmpname = shift; + delete $Global::unlink{$tmpname}; + unlink $tmpname; + rmdir $tmpdir; + } + + sub tmpfifo { + # Securely make a fifo by securely making a dir with a fifo in it + use POSIX qw(mkfifo); + my $tmpfifo = tmpname("fifo",@_); + mkfifo($tmpfifo,0600); + return $tmpfifo; + } + + sub rmtmpfifo { + # Clean up mktmpfifo + rmtmpname(@_); + } +} + sub uniq { # Remove duplicates and return unique values return keys %{{ map { $_ => 1 } @_ }}; @@ -3506,8 +3546,8 @@ sub which { # @full_path = full paths to @programs. Nothing if not found my @which; for my $prg (@_) { - push @which, grep { not -d $_ and -x $_ } - map { $_."/".$prg } split(":",$ENV{'PATH'}); + push(@which, grep { not -d $_ and -x $_ } + map { $_."/".$prg } split(":",$ENV{'PATH'})); } return @which; } @@ -6879,10 +6919,16 @@ sub print_dryrun_and_verbose { my $self = shift; my $actual_command = shift; # Temporary file name. Used for fifo to communicate exit val - my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx"); - $Global::unlink{$tmpfile} = 1; + my ($fh, $tmpfifo) = ::tmpfile(SUFFIX => ".tmx"); + $Global::unlink{$tmpfifo} = 1; close $fh; - unlink $tmpfile; + unlink $tmpfifo; +# # FIFO for communicating exit val +# my $tmpfifo = ::tmpfifo(); + if(length($tmpfifo) >=100) { + ::error("tmux does not support sockets with path > 100\n"); + ::wait_and_exit(255); + } my $visual_command = $self->replaced(); my $title = $visual_command; if($visual_command =~ /\0/) { @@ -6920,6 +6966,7 @@ sub print_dryrun_and_verbose { my $tmux; $ENV{'TMUX'} ||= "tmux"; if(not $tmuxsocket) { +# $tmuxsocket = ::tmpname("tmux"); (undef, $tmuxsocket) = ::tmpfile(SUFFIX => ".tms"); $Global::unlink{$tmuxsocket} = 1; unlink $tmuxsocket; @@ -6933,20 +6980,20 @@ sub print_dryrun_and_verbose { $Limits::Command::line_max_len, " tot ", $l_tot, "\n"); - return "mkfifo $tmpfile; $tmux ". + return "mkfifo $tmpfifo; $tmux ". # Run in tmux ::shell_quote_scalar ( "(".$actual_command.');'. # The triple print is needed - otherwise the testsuite fails - q[ perl -e 'while($t++<3){ print $ARGV[0],"\n" }' $?h/$status >> ].$tmpfile."&". + q[ perl -e 'while($t++<3){ print $ARGV[0],"\n" }' $?h/$status >> ].$tmpfifo."&". "echo $title; echo \007Job finished at: `date`;sleep 10" ). # Run outside tmux # Read a / separated line: 0h/2 for csh, 2/0 for bash. # If csh the first will be 0h, so use the second as exit value. # Otherwise just use the first value as exit value. - q{; exec perl -e '$/="/";$_=<>;$c=<>;unlink $ARGV; /(\d+)h/ and exit($1);exit$c' }.$tmpfile; + q{; exec perl -e '$/="/";$_=<>;$c=<>;unlink $ARGV; /(\d+)h/ and exit($1);exit$c' }.$tmpfifo; } } @@ -7493,15 +7540,16 @@ sub populate { my $next_arg; my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); - if($opt::cat or $opt::fifo) { + if($opt::cat) { # Generate a tempfile name that will be used as {} - my($outfh,$name) = ::tmpfile(SUFFIX => ".pip"); - close $outfh; - # Unlink is needed if: ssh otheruser@localhost - unlink $name; - $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget([Arg->new($name)]); + $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}-> + unget([Arg->new(::tmpname("cat"))]); + } + if($opt::fifo) { + # Generate a tempfile name that will be used as {} + $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}-> + unget([Arg->new(::tmpname("fifo"))]); } - while (not $self->{'arg_queue'}->empty()) { $next_arg = $self->{'arg_queue'}->get(); if(not defined $next_arg) { diff --git a/testsuite/Start.sh b/testsuite/Start.sh index 2b249f2c..17696eed 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -34,6 +34,9 @@ run_test() { } export -f run_test +# Create a monitor script +echo forever pstree -lp $$ >/tmp/monitor +chmod 755 /tmp/monitor date mkdir -p actual-results ls -t tests-to-run/*${1}*.sh | grep -v ${2} | diff --git a/testsuite/tests-to-run/parallel-local-ssh2.sh b/testsuite/tests-to-run/parallel-local-ssh2.sh index d49e957e..679ac853 100644 --- a/testsuite/tests-to-run/parallel-local-ssh2.sh +++ b/testsuite/tests-to-run/parallel-local-ssh2.sh @@ -41,9 +41,9 @@ echo '2bug #43358: shellshock breaks exporting functions using --env' echo '### bug #42999: --pipepart with remote does not work' seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; - parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; - parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; - parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; + parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc; + parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:' ; + parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:' ; rm /tmp/bug42999 echo '### --cat gives incorrect exit value in csh' diff --git a/testsuite/tests-to-run/parallel-local22.sh b/testsuite/tests-to-run/parallel-local22.sh index 6932ba86..01f69adc 100755 --- a/testsuite/tests-to-run/parallel-local22.sh +++ b/testsuite/tests-to-run/parallel-local22.sh @@ -46,8 +46,8 @@ echo '### bug #42041: Implement $PARALLEL_JOBSLOT' echo '### bug #42363: --pipepart and --fifo/--cat does not work' seq 100 > /tmp/bug42363; - parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; - parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; + parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:'; + parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:'; rm /tmp/bug42363 echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile' @@ -120,7 +120,7 @@ echo '### added transfersize/returnsize to local jobs' echo '### --tmux test - check termination' perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' | stdout parallel --tmux echo {} :::: - ::: a b | - perl -pe 's:tmp.par.*tms:tmp/parXXXXX.tms:; s/\d+/0/g' + perl -pe 's:/tmp/par......tms:/tmp/parXXXXX.tms:;' EOF diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh old mode 100644 new mode 100755 index 1ba72e8d..4d2f28ef --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -41,8 +41,9 @@ echo '### tmux1.8' seq 51 100 | TMUX=tmux1.8 par_tmux seq 101 150 | TMUX=tmux1.8 par_tmux seq 151 200 | TMUX=tmux1.8 par_tmux - seq 201 233 | TMUX=tmux1.8 par_tmux + seq 201 232 | TMUX=tmux1.8 par_tmux echo '### tmux1.8 fails' + echo 233 | TMUX=tmux1.8 par_tmux echo 234 | TMUX=tmux1.8 par_tmux echo 235 | TMUX=tmux1.8 par_tmux echo 236 | TMUX=tmux1.8 par_tmux diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 0ae97091..3f447838 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -22,7 +22,9 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ ' | stdout parallel -j7 -vd'\n\n' | perl -pe '$|=1; - # --files and --tmux + # --tmux + s:/tmp/........../t\d+:/tmp/tmuxtmp:; + # --files s:/tmp/par......(...):/tmp/parXXXXX.$1:; # --eta --progress s/ETA.*//g; s/local:.*//g; diff --git a/testsuite/wanted-results/parallel-local-ssh2 b/testsuite/wanted-results/parallel-local-ssh2 index aa5b7ebc..b23ff294 100644 --- a/testsuite/wanted-results/parallel-local-ssh2 +++ b/testsuite/wanted-results/parallel-local-ssh2 @@ -44,7 +44,7 @@ shellshock-hardened to non-shellshock-hardened Function non-shellshock-hardened echo '### bug #42999: --pipepart with remote does not work' ### bug #42999: --pipepart with remote does not work - seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; rm /tmp/bug42999 + seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:' ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:' ; rm /tmp/bug42999 14 14 33 11 11 33 11 11 33 @@ -54,24 +54,24 @@ echo '### bug #42999: --pipepart with remote does not work' 11 11 33 11 11 33 9 9 28 - 14 14 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 9 9 28 /tmp/XXXX -14 14 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX - 9 9 28 /tmp/XXXX + 14 14 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 9 9 28 /tmp/fifo0000 +14 14 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 + 9 9 28 /tmp/cat0000 echo '### --cat gives incorrect exit value in csh' ### --cat gives incorrect exit value in csh echo false | parallel --pipe --cat -Scsh@lo 'cat {}; false' ; echo $?; echo false | parallel --pipe --cat -Stcsh@lo 'cat {}; false' ; echo $?; echo true | parallel --pipe --cat -Scsh@lo 'cat {}; true' ; echo $?; echo true | parallel --pipe --cat -Stcsh@lo 'cat {}; true' ; echo $?; echo '### --cat and --fifo exit value in bash' diff --git a/testsuite/wanted-results/parallel-local157 b/testsuite/wanted-results/parallel-local157 index e69de29b..2a02d41c 100644 --- a/testsuite/wanted-results/parallel-local157 +++ b/testsuite/wanted-results/parallel-local157 @@ -0,0 +1 @@ +TEST diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 96874fb6..87afc501 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -43,25 +43,25 @@ echo '### bug #42041: Implement $PARALLEL_JOBSLOT' 2 echo '### bug #42363: --pipepart and --fifo/--cat does not work' ### bug #42363: --pipepart and --fifo/--cat does not work - seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe s:/tmp/.........pip:/tmp/XXXX: ; rm /tmp/bug42363 - 14 14 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 11 11 33 /tmp/XXXX - 9 9 28 /tmp/XXXX -14 14 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX -11 11 33 /tmp/XXXX - 9 9 28 /tmp/XXXX + seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:/tmp/fifo\d+:/tmp/fifo0000:'; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:/tmp/cat\d+:/tmp/cat0000:'; rm /tmp/bug42363 + 14 14 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 11 11 33 /tmp/fifo0000 + 9 9 28 /tmp/fifo0000 +14 14 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 +11 11 33 /tmp/cat0000 + 9 9 28 /tmp/cat0000 echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile' ### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile parallel --pipepart -a /etc/passwd -L 1 should not be run @@ -353,5 +353,5 @@ Send Receive Exitval Send Receive Exitval echo '### --tmux test - check termination' ### --tmux test - check termination - perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' | stdout parallel --tmux echo {} :::: - ::: a b | perl -pe 's:tmp.par.*tms:tmp/parXXXXX.tms:; s/\d+/0/g' + perl -e 'map {printf "$_%o%c\n",$_,$_}1..255' | stdout parallel --tmux echo {} :::: - ::: a b | perl -pe 's:/tmp/par......tms:/tmp/parXXXXX.tms:;' See output with: tmux -S /tmp/parXXXXX.tms attach diff --git a/testsuite/wanted-results/parallel-local7 b/testsuite/wanted-results/parallel-local7 index 632a21f1..3369845e 100644 --- a/testsuite/wanted-results/parallel-local7 +++ b/testsuite/wanted-results/parallel-local7 @@ -82,11 +82,14 @@ See output with: tmux1.8 -S /tmp/parXXXXX.tms attach seq 151 200 | TMUX=tmux1.8 par_tmux See output with: tmux1.8 -S /tmp/parXXXXX.tms attach 0 - seq 201 233 | TMUX=tmux1.8 par_tmux + seq 201 232 | TMUX=tmux1.8 par_tmux See output with: tmux1.8 -S /tmp/parXXXXX.tms attach 0 echo '### tmux1.8 fails' ### tmux1.8 fails + echo 233 | TMUX=tmux1.8 par_tmux +See output with: tmux1.8 -S /tmp/parXXXXX.tms attach +0 echo 234 | TMUX=tmux1.8 par_tmux See output with: tmux1.8 -S /tmp/parXXXXX.tms attach command too long diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 8058e9bd..18619688 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -406,7 +406,7 @@ B D parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}' -See output with: tmux -S /tmp/parXXXXX.tms attach +See output with: tmux -S /tmp/parXXXXX.tmux attach tmux -S /tmp/parXXXXX.tms attach no sessions parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3