Fixed bug #42089: --results with arg > 256 chars.

This commit is contained in:
Ole Tange 2014-04-19 10:11:32 +02:00
parent 34e131b894
commit cf358529b9
6 changed files with 72 additions and 8 deletions

View file

@ -220,18 +220,26 @@ GNU Parallel 20140422 ('') has been released. It is available for download at: h
New in this release: New in this release:
* GNU Parallel was cited in: Proactive System for Digital Forensic Investigation http://dspace.library.uvic.ca:8080/bitstream/handle/1828/5237/Alharbi_Soltan_PhD_2014.pdf
* GNU Parallel was cited in: Beyond MAP estimation with the track-oriented multiple hypothesis tracker http://ieeexplore.ieee.org/xpl/abstractReferences.jsp?tp=&arnumber=6766651&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D6766651 * GNU Parallel was cited in: Beyond MAP estimation with the track-oriented multiple hypothesis tracker http://ieeexplore.ieee.org/xpl/abstractReferences.jsp?tp=&arnumber=6766651&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D6766651
* GNU Parallel was cited in: Prokka: rapid prokaryotic genome annotation http://bioinformatics.oxfordjournals.org/content/early/2014/03/18/bioinformatics.btu153.short * GNU Parallel was cited in: Prokka: rapid prokaryotic genome annotation http://bioinformatics.oxfordjournals.org/content/early/2014/03/18/bioinformatics.btu153.short
* GNU Parallel was used (unfortunately with improper citation) in: Perspectives in magnetic resonance: NMR in the post-FFT era http://www.sciencedirect.com/science/article/pii/S1090780713003054 * GNU Parallel was used (unfortunately with improper citation) in: Perspectives in magnetic resonance: NMR in the post-FFT era http://www.sciencedirect.com/science/article/pii/S1090780713003054
* GNU Parallel is used in https://github.com/cc2qe/speedseq
* Batch XML validation at the command line. http://horothesia.blogspot.dk/2014/04/batch-xml-validation-at-command-line.html
* Org-mode with Parallel Babel http://draketo.de/english/emacs/parallel-babel#sec-2 * Org-mode with Parallel Babel http://draketo.de/english/emacs/parallel-babel#sec-2
* Speeding Up Grep Log Queries with GNU Parallel http://www.cybersecurity.io/speeding-grep-queries-gnu-parallel/ * Speeding Up Grep Log Queries with GNU Parallel http://www.cybersecurity.io/speeding-grep-queries-gnu-parallel/
* How to run tbss_2_reg in parallel http://tadpolebrainimaging.blogspot.dk/2014/03/how-to-run-tbss2reg-in-parallel.html * How to run tbss_2_reg in parallel http://tadpolebrainimaging.blogspot.dk/2014/03/how-to-run-tbss2reg-in-parallel.html
* GNU parallel example: blastn https://asciinema.org/a/8775
* Iterative DNS Brute Forcing http://www.room362.com/blog/2014/02/19/iterative-dns-brute-forcing/ * Iterative DNS Brute Forcing http://www.room362.com/blog/2014/02/19/iterative-dns-brute-forcing/
* Ejecutando comandos en paralelo http://jesusmercado.com/guias/ejecutando-comandos-en-paralelo/ * Ejecutando comandos en paralelo http://jesusmercado.com/guias/ejecutando-comandos-en-paralelo/

View file

@ -4309,7 +4309,17 @@ sub openoutputfiles {
my $args_as_dirname = $self->{'commandline'}->args_as_dirname(); my $args_as_dirname = $self->{'commandline'}->args_as_dirname();
# prefix/name1/val1/name2/val2/ # prefix/name1/val1/name2/val2/
my $dir = $opt::results."/".$args_as_dirname; my $dir = $opt::results."/".$args_as_dirname;
if(eval{ File::Path::mkpath($dir); }) {
# OK
} else {
# mkpath failed: Argument probably too long.
# Set $Global::max_file_length
max_file_name_length($opt::results);
$args_as_dirname = $self->{'commandline'}->args_as_dirname();
# prefix/name1/val1/name2/val2/
$dir = $opt::results."/".$args_as_dirname;
File::Path::mkpath($dir); File::Path::mkpath($dir);
}
# prefix/name1/val1/name2/val2/stdout # prefix/name1/val1/name2/val2/stdout
$outname = "$dir/stdout"; $outname = "$dir/stdout";
if(not open($outfhw, "+>", $outname)) { if(not open($outfhw, "+>", $outname)) {
@ -4399,6 +4409,36 @@ sub openoutputfiles {
} }
} }
sub max_file_name_length {
# Figure out the max length of a subdir and the max total length
# Ext4 = 255,130816
my $testdir = shift;
my $upper = 8_000_000;
my $len = 8;
my $dir="x"x$len;
do {
rmdir($testdir."/".$dir);
$len *= 16;
$dir="x"x$len;
} while (mkdir $testdir."/".$dir);
# Then search for the actual max length between $len/16 and $len
my $min = $len/16;
my $max = $len;
while($max-$min > 5) {
my $test = int(($min+$max)/2);
$dir="x"x$test;
if(mkdir $testdir."/".$dir) {
rmdir($testdir."/".$dir);
$min = $test;
} else {
$max = $test;
}
}
$Global::max_file_length = $min;
return $min;
}
sub set_fh { sub set_fh {
# Set file handle # Set file handle
my ($self, $fd_no, $key, $fh) = @_; my ($self, $fd_no, $key, $fh) = @_;
@ -5625,6 +5665,7 @@ sub args_as_dirname {
# Returns: # Returns:
# all unmodified arguments joined with '/' (similar to {}) # all unmodified arguments joined with '/' (similar to {})
# \t \0 \\ and / are quoted # \t \0 \\ and / are quoted
# If $Global::max_file_length: Keep labels < $Global::max_file_length
my $self = shift; my $self = shift;
my @res = (); my @res = ();
@ -5637,7 +5678,11 @@ sub args_as_dirname {
$s =~ s/\\/\\\\/g; $s =~ s/\\/\\\\/g;
$s =~ s/\t/\\t/g; $s =~ s/\t/\\t/g;
$s =~ s/\0/\\0/g; $s =~ s/\0/\\0/g;
$s =~ s:/:\\_:g; $s; } $s =~ s:/:\\_:g;
if($Global::max_file_length) {
$s = substr($s,0,$Global::max_file_length);
}
$s; }
$rec_ref->[$n-1]->orig()); $rec_ref->[$n-1]->orig());
} }
} }

Binary file not shown.

View file

@ -1122,12 +1122,12 @@ E.g:
will generate the files: will generate the files:
foo/a/I/b/III/stderr foo/a/I/b/III/stderr
foo/a/I/b/IIII/stderr
foo/a/II/b/III/stderr
foo/a/II/b/IIII/stderr
foo/a/I/b/III/stdout foo/a/I/b/III/stdout
foo/a/I/b/IIII/stderr
foo/a/I/b/IIII/stdout foo/a/I/b/IIII/stdout
foo/a/II/b/III/stderr
foo/a/II/b/III/stdout foo/a/II/b/III/stdout
foo/a/II/b/IIII/stderr
foo/a/II/b/IIII/stdout foo/a/II/b/IIII/stdout
and and
@ -1137,12 +1137,12 @@ and
will generate the files: will generate the files:
foo/1/I/2/III/stderr foo/1/I/2/III/stderr
foo/1/I/2/IIII/stderr
foo/1/II/2/III/stderr
foo/1/II/2/IIII/stderr
foo/1/I/2/III/stdout foo/1/I/2/III/stdout
foo/1/I/2/IIII/stderr
foo/1/I/2/IIII/stdout foo/1/I/2/IIII/stdout
foo/1/II/2/III/stderr
foo/1/II/2/III/stdout foo/1/II/2/III/stdout
foo/1/II/2/IIII/stderr
foo/1/II/2/IIII/stdout foo/1/II/2/IIII/stdout
See also B<--files>, B<--header>, B<--joblog>. See also B<--files>, B<--header>, B<--joblog>.

View file

@ -4,6 +4,13 @@
# Each should be taking 3-10s and be possible to run in parallel # Each should be taking 3-10s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)'
parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456;
ls parallel_test_dir/1/
rm -rf parallel_test_dir
echo '**'
echo '### Test --halt-on-error 0'; echo '### Test --halt-on-error 0';
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0;
echo $?; echo $?;

View file

@ -1,3 +1,7 @@
### bug #42089: --results with arg > 256 chars (should be 1 char shorter)
1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456
123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
**
### Test --halt-on-error 0 ### Test --halt-on-error 0
1 1
2 2