tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Better reading of columns fixed a few bugs. Passes unittest.

Browse source
This commit is contained in:
Ole Tange 2010-07-31 23:19:48 +02:00
parent 0d1711afbf
commit ab4133d44d
5 changed files with 168 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
doc/FUTURE_IDEAS
View file

@ -1,38 +1,12 @@
Unittest: parallel --trim fj ::: echo
Unittest: ending in space continuted on next line. Both needs quoting only once.
(echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo
example with colsep example with colsep
--colsep should default to remove whitespace before and after Unittest: --colsep + multiple -a
--donttrim --keepwhitespace
Unittest: eof string on ::: Unittest: eof string on :::
Unittest: quoting efter colsplit Unittest: quoting efter colsplit
echo '>/dev/null' | parallel echo
echo 'a%c%b' | parallel --colsep % echo {1} {3} {2}
(echo 'a%c%b'; echo a%c%b%d) | parallel --colsep % echo {1} {3} {2} {4}
(echo a%c%b; echo d%f%e) | parallel --colsep % echo {1} {3} {2}
parallel --colsep % echo {1} {3} {2} ::: a%c%b d%f%e
parallel --colsep % echo {1} {3} {2} ::: a%c%b
parallel --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d
(echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4}
parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4}
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"'
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"'
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"'
parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d'
Better screenshot on http://freshmeat.net/projects/parallel Better screenshot on http://freshmeat.net/projects/parallel
Better examples. Better examples.
Size: 640x480 Size: 640x480

147
src/parallel
View file

@ -2110,7 +2110,7 @@ sub read_args_from_command_line {
my $arg = shift @ARGV; my $arg = shift @ARGV;
if($arg eq $Global::arg_sep) { if($arg eq $Global::arg_sep) {
$Global::input_is_filename = (@new_argv); $Global::input_is_filename = (@new_argv);
push @Global::unget_lines, @ARGV; unget_argv(@ARGV);
$Global::total_jobs += @ARGV; $Global::total_jobs += @ARGV;
@ARGV=(); @ARGV=();
last; last;
@ -2158,7 +2158,7 @@ sub argfiles_xapply_style {
for (my $fileno = 0; $fileno <= $#::opt_a; $fileno++) { for (my $fileno = 0; $fileno <= $#::opt_a; $fileno++) {
$in_fh = open_or_exit($::opt_a[$fileno]); $in_fh = open_or_exit($::opt_a[$fileno]);
for (my $lineno=0; for (my $lineno=0;
$content[$fileno][$lineno] = get_arg_from_fh($in_fh); $content[$fileno][$lineno] = get_arg($in_fh);
$lineno++) { $lineno++) {
$max_lineno = max($max_lineno,$lineno); $max_lineno = max($max_lineno,$lineno);
} }
@ -2305,7 +2305,7 @@ sub generate_command_line {
$job_line = $arg; $job_line = $arg;
} }
} }
debug("Return jobline(".length($job_line)."): !$job_line!\n"); debug("Return jobline(",length($job_line),"): !",$job_line,"!\n");
} }
return ($job_line,$quoted_args); return ($job_line,$quoted_args);
} }
@ -2407,9 +2407,9 @@ sub xargs_computations {
- $no_of_replace * length($Global::replacestring) - $no_of_replace * length($Global::replacestring)
- $no_of_no_ext * length($Global::replace_no_ext); - $no_of_no_ext * length($Global::replace_no_ext);
$spaces = 0; $spaces = 0;
debug("length_of_command_no_args $length_of_command_no_args\n"); debug("length_of_command_no_args ",$length_of_command_no_args,"\n");
debug("length_of_context $length_of_context\n"); debug("length_of_context ",$length_of_context,"\n");
debug("no_of_replace $no_of_replace no_of_no_ext $no_of_no_ext\n"); debug("no_of_replace ",$no_of_replace," no_of_no_ext ",$no_of_no_ext,"\n");
} else { } else {
# remove all {}s # remove all {}s
$c =~ s/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E//og; $c =~ s/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E//og;
@ -2509,7 +2509,7 @@ sub max_length_limited_by_opt_s {
# Returns: # Returns:
# min(opt_s, number of chars on the longest command line allowed) # min(opt_s, number of chars on the longest command line allowed)
if(is_acceptable_command_line_length($::opt_s)) { if(is_acceptable_command_line_length($::opt_s)) {
debug("-s is OK: $::opt_s\n"); debug("-s is OK: ",$::opt_s,"\n");
return $::opt_s; return $::opt_s;
} }
# -s is too long: Find the correct # -s is too long: Find the correct
@ -3480,105 +3480,119 @@ sub more_arguments {
return (@Global::unget_arg or @Global::unget_lines or not eof $fh); return (@Global::unget_arg or @Global::unget_lines or not eof $fh);
} }
sub get_arg_from_fh { sub get_line_from_fh {
# Returns: # Returns:
# next argument from file handle - quoted if needed # next line from file handle from file or stdin. Delimiter removed
# undef if end of file # undef if end of file
my $fh = shift; my $fh = shift;
my $arg; my $arg;
if(not $Private::unget{$fh}) { if(@Global::unget_argv) {
@{$Private::unget{$fh}} = (); # Ungotten args from command line exists
debug("get_line_from_fh ",$Global::unget_argv[0],"\n");
return shift @Global::unget_argv;
} }
my $unget_ref = $Private::unget{$fh}; if(not $Global::unget_line{$fh}) {
@{$Global::unget_line{$fh}} = ();
}
my $unget_ref = $Global::unget_line{$fh};
if(@$unget_ref) { if(@$unget_ref) {
# Ungotten arg exists # Ungotten arg exists
$arg = shift @$unget_ref; debug("get_line_from_fh ",$$unget_ref[0],"\n");
} else { return shift @$unget_ref;
if(not more_arguments($fh)) { }
if(eof($fh)) {
return undef; return undef;
} }
$arg = <$fh>; $arg = <$fh>;
# Remove delimiter # Remove delimiter
$arg =~ s:$/$::; $arg =~ s:$/$::;
}
if($Global::end_of_file_string and if($Global::end_of_file_string and
$arg eq $Global::end_of_file_string) { $arg eq $Global::end_of_file_string) {
# Ignore the rest of input file # Ignore the rest of input file
while (<$fh>) {} while (<$fh>) {}
@$unget_ref = ();
return undef; return undef;
} }
if($Global::ignore_empty) { if($Global::ignore_empty) {
if($arg =~ /^\s*$/) { if($arg =~ /^\s*$/) {
return get_arg_from_fh($fh); return get_line_from_fh($fh);
} }
} }
if($Global::max_lines and more_arguments()) { if($Global::max_lines and not eof($fh)) {
if($arg =~ /\s$/) { if($arg =~ /\s$/) {
# Trailing space => continued on next line # Trailing space => continued on next line
$arg .= get_arg_from_fh($fh); $arg .= get_line_from_fh($fh);
} }
} }
if(not @$unget_ref and $::opt_colsep) { debug("get_line_from_fh ",$arg,"\n");
# split this into columns return $arg;
if($Global::trim ne 'n') { }
push @$unget_ref, split /$::opt_colsep/o, $arg;
} else { sub unget_line_from_fh {
push @$unget_ref, trim(split /$::opt_colsep/o, $arg); # Returns: N/A
my $fh = shift;
if(not $Global::unget_line{$fh}) {
@{$Global::unget_line{$fh}} = ();
} }
my $unget_ref = $Global::unget_line{$fh};
push @$unget_ref, @_;
}
sub unget_argv {
# Returns: N/A
push @Global::unget_argv, @_;
}
sub get_column {
# Return:
# Column unquoted untrimmed
# undef if no more
my $fh = shift;
if(not $Global::unget_col{$fh}) {
@{$Global::unget_col{$fh}} = ();
}
my $unget_ref = $Global::unget_col{$fh};
if(@$unget_ref) {
# Ungotten col exists
return shift @$unget_ref;
}
my $line = get_line_from_fh($fh);
if(defined $line) {
push @$unget_ref, split /$::opt_colsep/o, $line;
$::opt_N = $#$unget_ref+1; $::opt_N = $#$unget_ref+1;
$Global::max_number_of_args = $::opt_N; $Global::max_number_of_args = $::opt_N;
debug("unget_ref: @$unget_ref\n"); debug("col_unget_ref: @$unget_ref\n");
$arg = shift @$unget_ref; return shift @$unget_ref;
} else {
return undef;
} }
if($Global::input_is_filename) { }
$arg = shell_quote($arg);
sub unget_column {
# Returns: N/A
my $fh = shift;
if(not $Global::unget_col{$fh}) {
@{$Global::unget_col{$fh}} = ();
} }
debug($arg); my $unget_ref = $Global::unget_col{$fh};
return $arg; push @$unget_ref, @_;
} }
sub get_arg { sub get_arg {
# Returns: # Returns:
# next argument from input quoted and trimmed as needed # next argument quoted and trimmed as needed
# (from $Global::argfile or $fh if given)
# undef if end of file # undef if end of file
my $arg; my $arg;
my $fh = shift || $Global::argfile;
if(@Global::unget_arg) { if(@Global::unget_arg) {
return shift @Global::unget_arg; return shift @Global::unget_arg;
} elsif(@Global::unget_lines) {
$arg = shift @Global::unget_lines;
if($Global::end_of_file_string and
$arg eq $Global::end_of_file_string) {
# Ignore the rest of input file
@Global::unget_lines = ();
return undef;
}
if($Global::ignore_empty) {
if($arg =~ /^\s*$/) {
return get_arg();
}
}
if($Global::max_lines and more_arguments()) {
if($arg =~ /\s$/) {
# Trailing space => continued on next line
$arg .= get_arg();
}
} }
if($::opt_colsep) { if($::opt_colsep) {
# split this into columns $arg = get_column($fh);
my @columns = split /$::opt_colsep/o, $arg;
$::opt_N = $#columns+1;
$Global::max_number_of_args = $::opt_N;
if($Global::trim ne 'n') {
@columns = trim(@columns);
}
if($Global::input_is_filename) {
unget_arg(shell_quote(@columns));
} else { } else {
unget_arg(@columns); $arg = get_line_from_fh($fh);
} }
return get_arg(); if(defined $arg) {
} else {
if($Global::trim ne 'n') { if($Global::trim ne 'n') {
$arg = trim($arg); $arg = trim($arg);
} }
@ -3586,12 +3600,6 @@ sub get_arg {
$arg = shell_quote($arg); $arg = shell_quote($arg);
} }
} }
} else {
$arg = get_arg_from_fh($Global::argfile);
if(defined $arg) {
$Global::total_jobs++;
}
}
return $arg; return $arg;
} }
@ -3976,6 +3984,7 @@ sub __DEBUGGING__ {}
sub debug { sub debug {
# Returns: N/A # Returns: N/A
$Global::debug or return; $Global::debug or return;
@_ = grep { defined $_ ? $_ : "" } @_;
if($Global::original_stdout) { if($Global::original_stdout) {
print $Global::original_stdout @_; print $Global::original_stdout @_;
} else { } else {

35
unittest/tests-to-run/test29.sh Normal file
View file

@ -0,0 +1,35 @@
#!/bin/bash
echo '### Test of quoting of > bug'
echo '>/dev/null' | parallel echo
echo '### Test of quoting of > bug if line continuation'
(echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo
echo '### Test of --trim illegal'
stdout parallel --trim fj ::: echo
echo '### Test of --colsep'
echo 'a%c%b' | parallel --colsep % echo {1} {3} {2}
(echo 'a%c%b'; echo a%c%b%d) | parallel -k --colsep % echo {1} {3} {2} {4}
(echo a%c%b; echo d%f%e) | parallel -k --colsep % echo {1} {3} {2}
parallel -k --colsep % echo {1} {3} {2} ::: a%c%b d%f%e
parallel -k --colsep % echo {1} {3} {2} ::: a%c%b
parallel -k --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d
echo '### Test of --colsep as regexp'
(echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4}
parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4}
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"'
echo '### Test of -C'
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"'
echo '### Test of --trim n'
(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"'
parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d'

12
unittest/wanted-results/test27
View file

@ -113,8 +113,8 @@ Command line too long (42 >= 26) at number 1: \ \ \ \ \ '
88888888 999999999 1 22 88888888 999999999 1 22
333 4444 55555 666666 7777777 333 4444 55555 666666 7777777
88888888 999999999 88888888 999999999
1 22 333 4444 55555\ 666666 1 22 333 4444 55555 666666
7777777 88888888\ 999999999\\\ 1\\\ 22 333 4444 55555 7777777 88888888 999999999 1 22 333 4444 55555
666666 7777777 88888888 666666 7777777 88888888
999999999 999999999
### -L3 -0 echo < ldata-0.xi ### -L3 -0 echo < ldata-0.xi
@ -452,8 +452,8 @@ from to x y
1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22
333 4444 55555 666666 7777777 333 4444 55555 666666 7777777
88888888 999999999 88888888 999999999
1 22 333 4444 55555\ 666666 1 22 333 4444 55555 666666
7777777 88888888\ 999999999\\\ 1\\\ 22 333 4444 55555 7777777 88888888 999999999 1 22 333 4444 55555
666666 7777777 88888888 666666 7777777 88888888
999999999 999999999
### -L2 -n2 echo < ldata.xi ### -L2 -n2 echo < ldata.xi
@ -972,8 +972,8 @@ FIRST with 'single quotes' as well. IS OK
88888888 88888888
999999999 999999999
1 22 333 4444 55555\ 666666 1 22 333 4444 55555 666666
7777777 88888888\ 999999999\\\ 1\\\ 22 7777777 88888888 999999999 1 22
333 4444 55555 333 4444 55555
666666 7777777 666666 7777777
88888888 88888888

34
unittest/wanted-results/test29 Normal file
View file

@ -0,0 +1,34 @@
### Test of quoting of > bug
>/dev/null
### Test of quoting of > bug if line continuation
> > >
### Test of --trim illegal
parallel: --trim must be one of: r l rl lr
### Test of --colsep
a b c
a b c {4}
a b c d
a b c
d e f
a b c
d e f
a b c
a b c {4}
a b c d
### Test of --colsep as regexp
a b c {4}
a b c d
a b c {4}
a b c d
a b c {4}
a b c d
a_b_c_{4}
a_b_c_d
### Test of -C
a_b_c_{4}
a_b_c_d
### Test of --trim n
a_b_\ c\ _{4}
a_\ b\ _c_d
a_b_c_{4}
a_b_c_d