parallel: --results now with header \t value filenames.

This commit is contained in:
Ole Tange 2012-10-01 00:04:52 +02:00
parent fc1c13dca2
commit c8051ced91
5 changed files with 183 additions and 16 deletions

View file

@ -181,6 +181,9 @@ New in this release:
* GNU Parallel is accepted for Ubuntu Quantal. * GNU Parallel is accepted for Ubuntu Quantal.
http://packages.ubuntu.com/quantal/parallel http://packages.ubuntu.com/quantal/parallel
* GNU Parallel was requested for OpenSUSE (go vote!)
https://features.opensuse.org/314461
* Blog post in Japanese by Siguniang. * Blog post in Japanese by Siguniang.
https://siguniang.wordpress.com/2012/09/09/notes-on-gnu-parallel-the-command-line-power-tool/ https://siguniang.wordpress.com/2012/09/09/notes-on-gnu-parallel-the-command-line-power-tool/
(Siguniang mentions the logo is called 'The Cafe Wall Illusion') (Siguniang mentions the logo is called 'The Cafe Wall Illusion')

View file

@ -84,9 +84,16 @@ if($::opt_header and not $::opt_pipe) {
for my $s (split /$delimiter/o, $line) { for my $s (split /$delimiter/o, $line) {
::debug("Colname: '$s'"); ::debug("Colname: '$s'");
$command =~ s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; $command =~ s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
$Global::input_source_header{$id} = $s;
$id++; $id++;
} }
} }
} else {
my $id = 1;
for my $fh (@fhlist) {
$Global::input_source_header{$id} = $id;
$id++;
}
} }
# Parallel check for all hosts are up # Parallel check for all hosts are up
@ -465,7 +472,7 @@ sub options_hash {
"X" => \$::opt_X, "X" => \$::opt_X,
"v" => \@::opt_v, "v" => \@::opt_v,
"joblog=s" => \$::opt_joblog, "joblog=s" => \$::opt_joblog,
"results=s" => \$::opt_results, "results|result|res=s" => \$::opt_results,
"resume" => \$::opt_resume, "resume" => \$::opt_resume,
"silent" => \$::opt_silent, "silent" => \$::opt_silent,
#"silent-error|silenterror" => \$::opt_silent_error, #"silent-error|silenterror" => \$::opt_silent_error,
@ -597,7 +604,7 @@ sub get_options_from_array {
sub parse_options { sub parse_options {
# Returns: N/A # Returns: N/A
# Defaults: # Defaults:
$Global::version = 20120928; $Global::version = 20120930;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -1908,7 +1915,7 @@ sub die_bug {
sub version { sub version {
# Returns: N/A # Returns: N/A
if($::opt_tollef and not $::opt_gnu) { if($::opt_tollef and not $::opt_gnu) {
print "WARNING: YOU ARE USING --tollef. USE --gnu FOR GNU PARALLEL\n\n"; print "WARNING: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.\n";
} }
print join("\n", print join("\n",
"GNU $Global::progname $Global::version", "GNU $Global::progname $Global::version",
@ -1927,7 +1934,7 @@ sub version {
sub bibtex { sub bibtex {
# Returns: N/A # Returns: N/A
if($::opt_tollef and not $::opt_gnu) { if($::opt_tollef and not $::opt_gnu) {
print "WARNING: YOU ARE USING --tollef. USE --gnu FOR GNU PARALLEL\n\n"; print "WARNING: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.\n";
} }
print join("\n", print join("\n",
"\@article{Tange2011a,", "\@article{Tange2011a,",
@ -3204,10 +3211,16 @@ sub openresultsfile {
my $args_as_filename = $self->{'commandline'}->args_as_filename(); my $args_as_filename = $self->{'commandline'}->args_as_filename();
my ($outfh,$errfh,$name); my ($outfh,$errfh,$name);
$name = $::opt_results."stdout_".$args_as_filename; $name = $::opt_results."stdout_".$args_as_filename;
open($outfh,"+>",$name) or ::error("Cannot write to `$name'.\n"); if(not open($outfh,"+>",$name)) {
::error("Cannot write to `$name'.\n");
::wait_and_exit(255);
}
$self->set_stdoutfilename($name); $self->set_stdoutfilename($name);
$name = $::opt_results."stderr_".$args_as_filename; $name = $::opt_results."stderr_".$args_as_filename;
open($errfh,"+>",$name) or ::error("Cannot write to `$name'.\n"); if(not open($errfh,"+>",$name)) {
::error("Cannot write to `$name'.\n");
::wait_and_exit(255);
}
open OUT, '>&', $outfh or ::die_bug("Can't redirect STDOUT: $!"); open OUT, '>&', $outfh or ::die_bug("Can't redirect STDOUT: $!");
open ERR, '>&', $errfh or ::die_bug("Can't dup STDOUT: $!"); open ERR, '>&', $errfh or ::die_bug("Can't dup STDOUT: $!");
$self->set_stdout($outfh); $self->set_stdout($outfh);
@ -4207,12 +4220,20 @@ sub args_as_filename {
# all unmodified arguments joined with '\t' (similar to {}) # all unmodified arguments joined with '\t' (similar to {})
# \t \0 \\ and / are quoted # \t \0 \\ and / are quoted
my $self = shift; my $self = shift;
return (join "\t", my @res = ();
for my $rec_ref (@{$self->{'arg_list'}}) {
for my $n (0 .. $#$rec_ref) {
CORE::push(@res,
$Global::input_source_header{$n+1},
map { s/\\/\\\\/g; map { s/\\/\\\\/g;
s/\t/\\\t/g; s/\t/\\\t/g;
s/\0/\\0/g; s/\0/\\0/g;
s:/:\\_:g; $_ } s:/:\\_:g; $_ }
map { $_->orig() } map { @$_ } @{$self->{'arg_list'}}); $rec_ref->[$n]->orig());
}
}
return join "\t", @res;
} }
sub len { sub len {
@ -4876,6 +4897,10 @@ sub new {
my $fhs = shift; my $fhs = shift;
for my $fh (@$fhs) { for my $fh (@$fhs) {
if(-t $fh) { if(-t $fh) {
if($::opt_tollef and not $::opt_gnu) {
::warning("YOU ARE USING --tollef. "
. "IF THINGS ARE ACTING WEIRD USE --gnu.");
}
::warning("Input is read from the terminal. ". ::warning("Input is read from the terminal. ".
"Only experts do this on purpose. ". "Only experts do this on purpose. ".
"Press CTRL-D to exit.\n"); "Press CTRL-D to exit.\n");

View file

@ -972,6 +972,57 @@ it to the command.
Only used with B<--pipe>. Only used with B<--pipe>.
=item B<--results> I<prefix> (alpha testing)
=item B<--res> I<prefix> (alpha testing)
Results in files named by tab separated arguments. Save the output
into files. The file names will be prefixed with I<prefix> which can
contain a path with a prefix string. The file with output from stdout
(standard output) will prefixed with 'I<prefix>stdout_'. The file
with output from stderr (standard error) will prefixed with
'I<prefix>stderr_'.
The postfix is the header of the input source (if using B<--header :>)
or the number of the input source followed by the value of the input
source. This is repeated for every input source and is separated by
TAB (\t).
E.g:
parallel --header : --results foo/bar echo {a} {b} ::: a I II ::: b III IIII
will generate:
foo/barstderr_a I b III
foo/barstderr_a I b IIII
foo/barstderr_a II b III
foo/barstderr_a II b IIII
foo/barstdout_a I b III
foo/barstdout_a I b IIII
foo/barstdout_a II b III
foo/barstdout_a II b IIII
and
parallel --results foo/bar echo {1} {2} ::: 1 2 ::: 3 4
will generate:
foo/barstderr_1 I 2 III
foo/barstderr_1 I 2 IIII
foo/barstderr_1 II 2 III
foo/barstderr_1 II 2 IIII
foo/barstdout_1 I 2 III
foo/barstdout_1 I 2 IIII
foo/barstdout_1 II 2 III
foo/barstdout_1 II 2 IIII
where all spaces are TABs (\t);.
See also B<--files>, B<--header>, B<--joblog>.
=item B<--resume> =item B<--resume>
Resumes from the last unfinished job. By reading B<--joblog> GNU Resumes from the last unfinished job. By reading B<--joblog> GNU

View file

@ -47,4 +47,29 @@ echo "bug #34958: --pipe with record size measured in lines";
echo "bug #34958: --pipe with record size measured in lines"; echo "bug #34958: --pipe with record size measured in lines";
seq 10 | parallel --pipe -l 4 cat\;echo FOO seq 10 | parallel --pipe -l 4 cat\;echo FOO
echo "### Test --results";
mkdir -p /tmp/parallel_results_test;
parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII;
ls /tmp/parallel_results_test/testA*; rm /tmp/parallel_results_test/testA*
echo "### Test --res";
mkdir -p /tmp/parallel_results_test;
parallel -k --results /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII;
ls /tmp/parallel_results_test/testD*; rm /tmp/parallel_results_test/testD*
echo "### Test --result";
mkdir -p /tmp/parallel_results_test;
parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII;
ls /tmp/parallel_results_test/testE*; rm /tmp/parallel_results_test/testE*
echo "### Test --results --header :";
mkdir -p /tmp/parallel_results_test;
parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII
ls /tmp/parallel_results_test/testB*; rm /tmp/parallel_results_test/testB*
echo "### Test --results --header :";
mkdir -p /tmp/parallel_results_test;
parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: a I II ::: b III IIII
ls /tmp/parallel_results_test/testC*; rm /tmp/parallel_results_test/testC*
EOF EOF

View file

@ -77,10 +77,10 @@ a8 b1 2 3 4 5 6 7 8
a9 b1 2 3 4 5 6 7 8 9 a9 b1 2 3 4 5 6 7 8 9
a10 b1 2 3 4 5 6 7 8 9 10 a10 b1 2 3 4 5 6 7 8 9 10
### Test max line length -m -I ### Test max line length -m -I
31d9274be5fdc2de59487cb05ba57776 - 623249e8b509099d71745dbddc3fe5ce -
Chars per line (697800/6): 116300 Chars per line (697800/6): 116300
### Test max line length -X -I ### Test max line length -X -I
22074f9acada52462defb18ba912d744 - ae59907d4a33798608843da7ffcb29ed -
Chars per line (817788/7): 116826 Chars per line (817788/7): 116826
### bug #36659: --sshlogin strips leading slash from ssh command ### bug #36659: --sshlogin strips leading slash from ssh command
OK OK
@ -102,7 +102,6 @@ FOO
9 9
10 10
FOO FOO
FOO
bug #34958: --pipe with record size measured in lines bug #34958: --pipe with record size measured in lines
1 1
2 2
@ -117,4 +116,68 @@ FOO
9 9
10 10
FOO FOO
FOO ### Test --results
I III
I IIII
II III
II IIII
/tmp/parallel_results_test/testAstderr_1 I 2 III
/tmp/parallel_results_test/testAstderr_1 I 2 IIII
/tmp/parallel_results_test/testAstderr_1 II 2 III
/tmp/parallel_results_test/testAstderr_1 II 2 IIII
/tmp/parallel_results_test/testAstdout_1 I 2 III
/tmp/parallel_results_test/testAstdout_1 I 2 IIII
/tmp/parallel_results_test/testAstdout_1 II 2 III
/tmp/parallel_results_test/testAstdout_1 II 2 IIII
### Test --res
I III
I IIII
II III
II IIII
/tmp/parallel_results_test/testDstderr_1 I 2 III
/tmp/parallel_results_test/testDstderr_1 I 2 IIII
/tmp/parallel_results_test/testDstderr_1 II 2 III
/tmp/parallel_results_test/testDstderr_1 II 2 IIII
/tmp/parallel_results_test/testDstdout_1 I 2 III
/tmp/parallel_results_test/testDstdout_1 I 2 IIII
/tmp/parallel_results_test/testDstdout_1 II 2 III
/tmp/parallel_results_test/testDstdout_1 II 2 IIII
### Test --result
I III
I IIII
II III
II IIII
/tmp/parallel_results_test/testEstderr_1 I 2 III
/tmp/parallel_results_test/testEstderr_1 I 2 IIII
/tmp/parallel_results_test/testEstderr_1 II 2 III
/tmp/parallel_results_test/testEstderr_1 II 2 IIII
/tmp/parallel_results_test/testEstdout_1 I 2 III
/tmp/parallel_results_test/testEstdout_1 I 2 IIII
/tmp/parallel_results_test/testEstdout_1 II 2 III
/tmp/parallel_results_test/testEstdout_1 II 2 IIII
### Test --results --header :
I III
I IIII
II III
II IIII
/tmp/parallel_results_test/testBstderr_a I b III
/tmp/parallel_results_test/testBstderr_a I b IIII
/tmp/parallel_results_test/testBstderr_a II b III
/tmp/parallel_results_test/testBstderr_a II b IIII
/tmp/parallel_results_test/testBstdout_a I b III
/tmp/parallel_results_test/testBstdout_a I b IIII
/tmp/parallel_results_test/testBstdout_a II b III
/tmp/parallel_results_test/testBstdout_a II b IIII
### Test --results --header :
I III
I IIII
II III
II IIII
/tmp/parallel_results_test/testCstderr_a I b III
/tmp/parallel_results_test/testCstderr_a I b IIII
/tmp/parallel_results_test/testCstderr_a II b III
/tmp/parallel_results_test/testCstderr_a II b IIII
/tmp/parallel_results_test/testCstdout_a I b III
/tmp/parallel_results_test/testCstdout_a I b IIII
/tmp/parallel_results_test/testCstdout_a II b III
/tmp/parallel_results_test/testCstdout_a II b IIII