diff --git a/doc/release_new_version b/doc/release_new_version index cb016b0a..3875d02c 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -215,10 +215,19 @@ GNU Parallel 20150422 ('Germanwings') has been released. It is available for dow Haiku of the month: - <<>> + SSH set up? + Instant cluster needed now? + Use GNU Parallel. + -- Ole Tange New in this release: +* Security fix. An attacker on the local system could make you overwrite one of your own files with a single byte. The problem requires: + + - you using --compress or --tmux + - the attacker must figure out the randomly chosen file name + - the attacker to create a symlink within a time window of 15 ms + * GNU Parallel now has a DOI: https://dx.doi.org/10.5281/zenodo.16303 * GNU Parallel was cited in: Scaling Machine Learning for Target Prediction in Drug Discovery using Apache Spark https://cris.cumulus.vub.ac.be/portal/files/5147244/spark.pdf diff --git a/src/parallel b/src/parallel index e19b2890..8b740a89 100755 --- a/src/parallel +++ b/src/parallel @@ -5579,23 +5579,22 @@ sub slot { # cat followed by tail (possibly with rm as soon at the file is opened) # If $writerpid dead: finish after this round use Fcntl; - $|=1; - my ($cmd, $writerpid, $read_file, $unlink_file) = @ARGV; - while(defined $unlink_file and not -e $unlink_file) { - # Writer has not opened file, so we cannot open it - $sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep); - usleep($sleep); - } - + my ($comfile, $cmd, $writerpid, $read_file, $unlink_file) = @ARGV; if($read_file) { open(IN,"<",$read_file) || die("cattail: Cannot open $read_file"); } else { *IN = *STDIN; } + while(! -s $comfile) { + # Writer has not opened the buffer file, so we cannot remove it yet + $sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep); + usleep($sleep); + } # The writer and we have both opened the file, so it is safe to unlink it unlink $unlink_file; + unlink $comfile; my $first_round = 1; my $flags; @@ -5630,8 +5629,8 @@ sub slot { } # TODO This could probably be done more efficiently using select(2) # Nothing read: Wait longer before next read - # Up to 30 milliseconds - $sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep); + # Up to 100 milliseconds + $sleep = ($sleep < 100) ? ($sleep * 1.001 + 0.01) : ($sleep); usleep($sleep); } } @@ -5741,27 +5740,29 @@ sub grouped { } } -sub empty_input_detector { - # If no input: exec true - # If some input: Pass input as input to pipe - # This avoids starting the $read command if there is no input. +sub empty_input_wrapper { + # If no input: exit(0) + # If some input: Pass input as input to command on STDIN + # This avoids starting the command if there is no input. + # Input: + # $command = command to pipe data to # Returns: - # $cmd = script to prepend to '| ($real command)' - - # The $tmpfile might exist if run on a remote system - we accept that risk - my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".chr"); - # Unlink to avoid leaving files if --dry-run or --sshlogin - unlink $tmpfile; - $Global::unlink{$tmpfile} = 1; - my $cmd = - # Exit value: - # empty input = true - # some input = exit val from command - # sh -c needed as csh cannot hide stderr - qq{ sh -c 'dd bs=1 count=1 of=$tmpfile 2>/dev/null'; }. - qq{ test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true; }. - qq{ (cat $tmpfile; rm $tmpfile; cat - ) }; - return $cmd; + # $wrapped_command = the wrapped command + my $command = shift; + my $script = '$c="'.::perl_quote_scalar($command).'";'. + ::spacefree(0,q{ + if(sysread(STDIN, $buf, 1)) { + open($fh, "|-", $c) || die; + syswrite($fh, $buf); + while($read = sysread(STDIN, $buf, 32768)) { + syswrite($fh, $buf); + } + close $fh; + exit ($?&127 ? 128+($?&127) : 1+$?>>8) + } + }); + ::debug("run",'Empty wrap: perl -e '.::shell_quote_scalar($script)."\n"); + return 'perl -e '.::shell_quote_scalar($script); } sub filter_through_compress { @@ -5772,15 +5773,18 @@ sub filter_through_compress { my $cattail = cattail(); for my $fdno (1,2) { - # The tmpfile is used to tell the reader that the writer has started, - # so unlink it to start with. - unlink $self->fh($fdno,'name'); - my $wpid = open(my $fdw,"|-", "(".empty_input_detector(). - "| ($opt::compress_program)) >>". + # Make a communication file. + my ($fh, $comfile) = ::tmpfile(SUFFIX => ".pac"); + close $fh; + # Compressor: (echo > $comfile; compress pipe) > output + # When the echo is written to $comfile, it is known that output file is opened, + # thus output file can then be removed by the decompressor. + my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".empty_input_wrapper($opt::compress_program).") >". $self->fh($fdno,'name')) || die $?; $self->set_fh($fdno,'w',$fdw); $self->set_fh($fdno,'wpid',$wpid); - my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, + # Decompressor: open output; -s $comfile > 0: rm $comfile output; decompress output > stdout + my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, $comfile, $opt::decompress_program, $wpid, $self->fh($fdno,'name'),$self->fh($fdno,'unlink')) || die $?; $self->set_fh($fdno,'r',$fdr); @@ -6233,8 +6237,8 @@ sub wrapped { # }' 0 0 0 11 | $command = (shift @Global::cat_partials). " | ($command)"; } elsif($opt::pipe) { - # Prepend EOF-detector to avoid starting $command if EOF. - $command = empty_input_detector(). "| ($command);"; + # Wrap with EOF-detector to avoid starting $command if EOF. + $command = empty_input_wrapper($command); } if($opt::tmux) { # Wrap command with 'tmux' diff --git a/src/parallel.pod b/src/parallel.pod index 5507eda1..5bd54174 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -765,6 +765,8 @@ B may be run on either B or B, B may be run on either B or B, but B will only be run on B. +See also: B<--sshlogin>. + =item B<-I> I @@ -1629,8 +1631,12 @@ seconds. =item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> +=item B<-S> I<@hostgroup> + =item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> +=item B<--sshlogin> I<@hostgroup> + Distribute jobs to remote computers. The jobs will be run on a list of remote computers. @@ -1638,8 +1644,8 @@ If I is given, the I will be added to that hostgroup. Multiple hostgroups are separated by '+'. The I will always be added to a hostgroup named the same as I. -If only the I is given, only the sshlogins in those -hostgroups will be used. +If only the I<@hostgroup> is given, only the sshlogins in that +hostgroup will be used. Multiple I<@hostgroup> can be given. GNU B will determine the number of CPU cores on the remote computers and run the number of jobs as specified by B<-j>. If the diff --git a/testsuite/Start.sh b/testsuite/Start.sh index 15f2ef70..2b249f2c 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -28,9 +28,9 @@ run_test() { # Check if it was cleaned up find /tmp -maxdepth 1 | - perl -ne '/\.(tmb|chr|tms|par)$/ and ++$a and print "TMP NOT CLEAN. FOUND: $_".`touch '$script'`;' + perl -ne '/\.(tmx|pac|arg|all|log|swp|loa|ssh|df|pip|tmb|chr|tms|par)$/ and ++$a and print "TMP NOT CLEAN. FOUND: $_".`touch '$script'`;' # May be owned by other users - sudo rm -f /tmp/*.{tmb,chr,tms,par} + sudo rm -f /tmp/*.{tmx,pac,arg,all,log,swp,loa,ssh,df,pip,tmb,chr,tms,par} } export -f run_test diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index a001c4dd..1aade44e 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -266,7 +266,6 @@ echo 'bug #44250: pxz complains File format not recognized but decompresses anyw bug #44250: pxz complains File format not recognized but decompresses anyway # The first line dumps core if run from make file. Why?! stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file -Segmentation fault (core dumped) parallel: Error: pxz failed. stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file ls: cannot access /OK-if-missing-file: No such file or directory diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index e231f867..8058e9bd 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -931,7 +931,7 @@ This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. parallel --version -GNU parallel 20150415 +GNU parallel 20150416 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange and Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later @@ -943,7 +943,7 @@ Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication please cite as described in 'parallel --bibtex'. parallel --minversion 20130722 && echo Your version is at least 20130722. -20150415 +20150416 Your version is at least 20130722. parallel --bibtex Academic tradition requires you to cite works you base your article on.