mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
parallel: Use a perl script wrapper to avoid security issue with race condition:
Attacker symlinks a file that will be created later.
This commit is contained in:
Ole Tange
parent
1d1d35cfdd
commit
b9be3f78ba
|
|
@ -215,10 +215,19 @@ GNU Parallel 20150422 ('Germanwings') has been released. It is available for dow
|
|||
|
||||
Haiku of the month:
|
||||
|
||||
<<>>
|
||||
SSH set up?
|
||||
Instant cluster needed now?
|
||||
Use GNU Parallel.
|
||||
-- Ole Tange
|
||||
|
||||
New in this release:
|
||||
|
||||
* Security fix. An attacker on the local system could make you overwrite one of your own files with a single byte. The problem requires:
|
||||
|
||||
- you using --compress or --tmux
|
||||
- the attacker must figure out the randomly chosen file name
|
||||
- the attacker to create a symlink within a time window of 15 ms
|
||||
|
||||
* GNU Parallel now has a DOI: https://dx.doi.org/10.5281/zenodo.16303
|
||||
|
||||
* GNU Parallel was cited in: Scaling Machine Learning for Target Prediction in Drug Discovery using Apache Spark https://cris.cumulus.vub.ac.be/portal/files/5147244/spark.pdf
|
||||
|
|
|
|||
80
src/parallel
80
src/parallel
|
|
@ -5579,23 +5579,22 @@ sub slot {
|
|||
# cat followed by tail (possibly with rm as soon at the file is opened)
|
||||
# If $writerpid dead: finish after this round
|
||||
use Fcntl;
|
||||
|
||||
$|=1;
|
||||
|
||||
my ($cmd, $writerpid, $read_file, $unlink_file) = @ARGV;
|
||||
while(defined $unlink_file and not -e $unlink_file) {
|
||||
# Writer has not opened file, so we cannot open it
|
||||
$sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep);
|
||||
usleep($sleep);
|
||||
}
|
||||
|
||||
my ($comfile, $cmd, $writerpid, $read_file, $unlink_file) = @ARGV;
|
||||
if($read_file) {
|
||||
open(IN,"<",$read_file) || die("cattail: Cannot open $read_file");
|
||||
} else {
|
||||
*IN = *STDIN;
|
||||
}
|
||||
while(! -s $comfile) {
|
||||
# Writer has not opened the buffer file, so we cannot remove it yet
|
||||
$sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep);
|
||||
usleep($sleep);
|
||||
}
|
||||
# The writer and we have both opened the file, so it is safe to unlink it
|
||||
unlink $unlink_file;
|
||||
unlink $comfile;
|
||||
|
||||
my $first_round = 1;
|
||||
my $flags;
|
||||
|
|
@ -5630,8 +5629,8 @@ sub slot {
|
|||
}
|
||||
# TODO This could probably be done more efficiently using select(2)
|
||||
# Nothing read: Wait longer before next read
|
||||
# Up to 30 milliseconds
|
||||
$sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep);
|
||||
# Up to 100 milliseconds
|
||||
$sleep = ($sleep < 100) ? ($sleep * 1.001 + 0.01) : ($sleep);
|
||||
usleep($sleep);
|
||||
}
|
||||
}
|
||||
|
|
@ -5741,27 +5740,29 @@ sub grouped {
|
|||
}
|
||||
}
|
||||
|
||||
sub empty_input_detector {
|
||||
# If no input: exec true
|
||||
# If some input: Pass input as input to pipe
|
||||
# This avoids starting the $read command if there is no input.
|
||||
sub empty_input_wrapper {
|
||||
# If no input: exit(0)
|
||||
# If some input: Pass input as input to command on STDIN
|
||||
# This avoids starting the command if there is no input.
|
||||
# Input:
|
||||
# $command = command to pipe data to
|
||||
# Returns:
|
||||
# $cmd = script to prepend to '| ($real command)'
|
||||
|
||||
# The $tmpfile might exist if run on a remote system - we accept that risk
|
||||
my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".chr");
|
||||
# Unlink to avoid leaving files if --dry-run or --sshlogin
|
||||
unlink $tmpfile;
|
||||
$Global::unlink{$tmpfile} = 1;
|
||||
my $cmd =
|
||||
# Exit value:
|
||||
# empty input = true
|
||||
# some input = exit val from command
|
||||
# sh -c needed as csh cannot hide stderr
|
||||
qq{ sh -c 'dd bs=1 count=1 of=$tmpfile 2>/dev/null'; }.
|
||||
qq{ test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true; }.
|
||||
qq{ (cat $tmpfile; rm $tmpfile; cat - ) };
|
||||
return $cmd;
|
||||
# $wrapped_command = the wrapped command
|
||||
my $command = shift;
|
||||
my $script = '$c="'.::perl_quote_scalar($command).'";'.
|
||||
::spacefree(0,q{
|
||||
if(sysread(STDIN, $buf, 1)) {
|
||||
open($fh, "|-", $c) || die;
|
||||
syswrite($fh, $buf);
|
||||
while($read = sysread(STDIN, $buf, 32768)) {
|
||||
syswrite($fh, $buf);
|
||||
}
|
||||
close $fh;
|
||||
exit ($?&127 ? 128+($?&127) : 1+$?>>8)
|
||||
}
|
||||
});
|
||||
::debug("run",'Empty wrap: perl -e '.::shell_quote_scalar($script)."\n");
|
||||
return 'perl -e '.::shell_quote_scalar($script);
|
||||
}
|
||||
|
||||
sub filter_through_compress {
|
||||
|
|
@ -5772,15 +5773,18 @@ sub filter_through_compress {
|
|||
my $cattail = cattail();
|
||||
|
||||
for my $fdno (1,2) {
|
||||
# The tmpfile is used to tell the reader that the writer has started,
|
||||
# so unlink it to start with.
|
||||
unlink $self->fh($fdno,'name');
|
||||
my $wpid = open(my $fdw,"|-", "(".empty_input_detector().
|
||||
"| ($opt::compress_program)) >>".
|
||||
# Make a communication file.
|
||||
my ($fh, $comfile) = ::tmpfile(SUFFIX => ".pac");
|
||||
close $fh;
|
||||
# Compressor: (echo > $comfile; compress pipe) > output
|
||||
# When the echo is written to $comfile, it is known that output file is opened,
|
||||
# thus output file can then be removed by the decompressor.
|
||||
my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".empty_input_wrapper($opt::compress_program).") >".
|
||||
$self->fh($fdno,'name')) || die $?;
|
||||
$self->set_fh($fdno,'w',$fdw);
|
||||
$self->set_fh($fdno,'wpid',$wpid);
|
||||
my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail,
|
||||
# Decompressor: open output; -s $comfile > 0: rm $comfile output; decompress output > stdout
|
||||
my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, $comfile,
|
||||
$opt::decompress_program, $wpid,
|
||||
$self->fh($fdno,'name'),$self->fh($fdno,'unlink')) || die $?;
|
||||
$self->set_fh($fdno,'r',$fdr);
|
||||
|
|
@ -6233,8 +6237,8 @@ sub wrapped {
|
|||
# }' 0 0 0 11 |
|
||||
$command = (shift @Global::cat_partials). " | ($command)";
|
||||
} elsif($opt::pipe) {
|
||||
# Prepend EOF-detector to avoid starting $command if EOF.
|
||||
$command = empty_input_detector(). "| ($command);";
|
||||
# Wrap with EOF-detector to avoid starting $command if EOF.
|
||||
$command = empty_input_wrapper($command);
|
||||
}
|
||||
if($opt::tmux) {
|
||||
# Wrap command with 'tmux'
|
||||
|
|
|
|||
|
|
@ -765,6 +765,8 @@ B<my_grp1_arg> may be run on either B<myserver1> or B<myserver2>,
|
|||
B<third_arg> may be run on either B<myserver1> or B<myserver3>,
|
||||
but B<arg_for_grp2> will only be run on B<myserver2>.
|
||||
|
||||
See also: B<--sshlogin>.
|
||||
|
||||
|
||||
=item B<-I> I<replace-str>
|
||||
|
||||
|
|
@ -1629,8 +1631,12 @@ seconds.
|
|||
|
||||
=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]>
|
||||
|
||||
=item B<-S> I<@hostgroup>
|
||||
|
||||
=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]>
|
||||
|
||||
=item B<--sshlogin> I<@hostgroup>
|
||||
|
||||
Distribute jobs to remote computers. The jobs will be run on a list of
|
||||
remote computers.
|
||||
|
||||
|
|
@ -1638,8 +1644,8 @@ If I<hostgroups> is given, the I<sshlogin> will be added to that
|
|||
hostgroup. Multiple hostgroups are separated by '+'. The I<sshlogin>
|
||||
will always be added to a hostgroup named the same as I<sshlogin>.
|
||||
|
||||
If only the I<hostgroups> is given, only the sshlogins in those
|
||||
hostgroups will be used.
|
||||
If only the I<@hostgroup> is given, only the sshlogins in that
|
||||
hostgroup will be used. Multiple I<@hostgroup> can be given.
|
||||
|
||||
GNU B<parallel> will determine the number of CPU cores on the remote
|
||||
computers and run the number of jobs as specified by B<-j>. If the
|
||||
|
|
|
|||
|
|
@ -28,9 +28,9 @@ run_test() {
|
|||
|
||||
# Check if it was cleaned up
|
||||
find /tmp -maxdepth 1 |
|
||||
perl -ne '/\.(tmb|chr|tms|par)$/ and ++$a and print "TMP NOT CLEAN. FOUND: $_".`touch '$script'`;'
|
||||
perl -ne '/\.(tmx|pac|arg|all|log|swp|loa|ssh|df|pip|tmb|chr|tms|par)$/ and ++$a and print "TMP NOT CLEAN. FOUND: $_".`touch '$script'`;'
|
||||
# May be owned by other users
|
||||
sudo rm -f /tmp/*.{tmb,chr,tms,par}
|
||||
sudo rm -f /tmp/*.{tmx,pac,arg,all,log,swp,loa,ssh,df,pip,tmb,chr,tms,par}
|
||||
}
|
||||
export -f run_test
|
||||
|
||||
|
|
|
|||
|
|
@ -266,7 +266,6 @@ echo 'bug #44250: pxz complains File format not recognized but decompresses anyw
|
|||
bug #44250: pxz complains File format not recognized but decompresses anyway
|
||||
# The first line dumps core if run from make file. Why?!
|
||||
stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
|
||||
Segmentation fault (core dumped)
|
||||
parallel: Error: pxz failed.
|
||||
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file
|
||||
ls: cannot access /OK-if-missing-file: No such file or directory
|
||||
|
|
|
|||
|
|
@ -931,7 +931,7 @@ This helps funding further development; and it won't cost you a cent.
|
|||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
parallel --version
|
||||
GNU parallel 20150415
|
||||
GNU parallel 20150416
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange
|
||||
and Free Software Foundation, Inc.
|
||||
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
||||
|
|
@ -943,7 +943,7 @@ Web site: http://www.gnu.org/software/parallel
|
|||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite as described in 'parallel --bibtex'.
|
||||
parallel --minversion 20130722 && echo Your version is at least 20130722.
|
||||
20150415
|
||||
20150416
|
||||
Your version is at least 20130722.
|
||||
parallel --bibtex
|
||||
Academic tradition requires you to cite works you base your article on.
|
||||
|
|
|
|||
Loading…
Reference in a new issue