diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 16613606..277f948c 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,38 +1,12 @@ -Unittest: parallel --trim fj ::: echo - -Unittest: ending in space continuted on next line. Both needs quoting only once. -(echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo example with colsep ---colsep should default to remove whitespace before and after ---donttrim --keepwhitespace +Unittest: --colsep + multiple -a + Unittest: eof string on ::: Unittest: quoting efter colsplit -echo '>/dev/null' | parallel echo - -echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} -(echo 'a%c%b'; echo a%c%b%d) | parallel --colsep % echo {1} {3} {2} {4} -(echo a%c%b; echo d%f%e) | parallel --colsep % echo {1} {3} {2} -parallel --colsep % echo {1} {3} {2} ::: a%c%b d%f%e - -parallel --colsep % echo {1} {3} {2} ::: a%c%b - -parallel --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d - -(echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4} - -parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d - -(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} - -(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' -(echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' -(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' -parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' - Better screenshot on http://freshmeat.net/projects/parallel Better examples. Size: 640x480 diff --git a/src/parallel b/src/parallel index b20486fd..9ab51a6e 100755 --- a/src/parallel +++ b/src/parallel @@ -2110,7 +2110,7 @@ sub read_args_from_command_line { my $arg = shift @ARGV; if($arg eq $Global::arg_sep) { $Global::input_is_filename = (@new_argv); - push @Global::unget_lines, @ARGV; + unget_argv(@ARGV); $Global::total_jobs += @ARGV; @ARGV=(); last; @@ -2158,7 +2158,7 @@ sub argfiles_xapply_style { for (my $fileno = 0; $fileno <= $#::opt_a; $fileno++) { $in_fh = open_or_exit($::opt_a[$fileno]); for (my $lineno=0; - $content[$fileno][$lineno] = get_arg_from_fh($in_fh); + $content[$fileno][$lineno] = get_arg($in_fh); $lineno++) { $max_lineno = max($max_lineno,$lineno); } @@ -2305,7 +2305,7 @@ sub generate_command_line { $job_line = $arg; } } - debug("Return jobline(".length($job_line)."): !$job_line!\n"); + debug("Return jobline(",length($job_line),"): !",$job_line,"!\n"); } return ($job_line,$quoted_args); } @@ -2407,9 +2407,9 @@ sub xargs_computations { - $no_of_replace * length($Global::replacestring) - $no_of_no_ext * length($Global::replace_no_ext); $spaces = 0; - debug("length_of_command_no_args $length_of_command_no_args\n"); - debug("length_of_context $length_of_context\n"); - debug("no_of_replace $no_of_replace no_of_no_ext $no_of_no_ext\n"); + debug("length_of_command_no_args ",$length_of_command_no_args,"\n"); + debug("length_of_context ",$length_of_context,"\n"); + debug("no_of_replace ",$no_of_replace," no_of_no_ext ",$no_of_no_ext,"\n"); } else { # remove all {}s $c =~ s/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E//og; @@ -2509,7 +2509,7 @@ sub max_length_limited_by_opt_s { # Returns: # min(opt_s, number of chars on the longest command line allowed) if(is_acceptable_command_line_length($::opt_s)) { - debug("-s is OK: $::opt_s\n"); + debug("-s is OK: ",$::opt_s,"\n"); return $::opt_s; } # -s is too long: Find the correct @@ -3480,116 +3480,124 @@ sub more_arguments { return (@Global::unget_arg or @Global::unget_lines or not eof $fh); } -sub get_arg_from_fh { +sub get_line_from_fh { # Returns: - # next argument from file handle - quoted if needed + # next line from file handle from file or stdin. Delimiter removed # undef if end of file my $fh = shift; my $arg; - if(not $Private::unget{$fh}) { - @{$Private::unget{$fh}} = (); + if(@Global::unget_argv) { + # Ungotten args from command line exists + debug("get_line_from_fh ",$Global::unget_argv[0],"\n"); + return shift @Global::unget_argv; } - my $unget_ref = $Private::unget{$fh}; + if(not $Global::unget_line{$fh}) { + @{$Global::unget_line{$fh}} = (); + } + my $unget_ref = $Global::unget_line{$fh}; if(@$unget_ref) { # Ungotten arg exists - $arg = shift @$unget_ref; - } else { - if(not more_arguments($fh)) { - return undef; - } - $arg = <$fh>; - # Remove delimiter - $arg =~ s:$/$::; + debug("get_line_from_fh ",$$unget_ref[0],"\n"); + return shift @$unget_ref; } + if(eof($fh)) { + return undef; + } + $arg = <$fh>; + # Remove delimiter + $arg =~ s:$/$::; if($Global::end_of_file_string and $arg eq $Global::end_of_file_string) { # Ignore the rest of input file while (<$fh>) {} - @$unget_ref = (); return undef; } if($Global::ignore_empty) { if($arg =~ /^\s*$/) { - return get_arg_from_fh($fh); + return get_line_from_fh($fh); } } - if($Global::max_lines and more_arguments()) { + if($Global::max_lines and not eof($fh)) { if($arg =~ /\s$/) { # Trailing space => continued on next line - $arg .= get_arg_from_fh($fh); + $arg .= get_line_from_fh($fh); } } - if(not @$unget_ref and $::opt_colsep) { - # split this into columns - if($Global::trim ne 'n') { - push @$unget_ref, split /$::opt_colsep/o, $arg; - } else { - push @$unget_ref, trim(split /$::opt_colsep/o, $arg); - } + debug("get_line_from_fh ",$arg,"\n"); + return $arg; +} + +sub unget_line_from_fh { + # Returns: N/A + my $fh = shift; + if(not $Global::unget_line{$fh}) { + @{$Global::unget_line{$fh}} = (); + } + my $unget_ref = $Global::unget_line{$fh}; + push @$unget_ref, @_; +} + +sub unget_argv { + # Returns: N/A + push @Global::unget_argv, @_; +} + +sub get_column { + # Return: + # Column unquoted untrimmed + # undef if no more + my $fh = shift; + if(not $Global::unget_col{$fh}) { + @{$Global::unget_col{$fh}} = (); + } + my $unget_ref = $Global::unget_col{$fh}; + if(@$unget_ref) { + # Ungotten col exists + return shift @$unget_ref; + } + my $line = get_line_from_fh($fh); + if(defined $line) { + push @$unget_ref, split /$::opt_colsep/o, $line; $::opt_N = $#$unget_ref+1; $Global::max_number_of_args = $::opt_N; - debug("unget_ref: @$unget_ref\n"); - $arg = shift @$unget_ref; + debug("col_unget_ref: @$unget_ref\n"); + return shift @$unget_ref; + } else { + return undef; } - if($Global::input_is_filename) { - $arg = shell_quote($arg); +} + +sub unget_column { + # Returns: N/A + my $fh = shift; + if(not $Global::unget_col{$fh}) { + @{$Global::unget_col{$fh}} = (); } - debug($arg); - return $arg; + my $unget_ref = $Global::unget_col{$fh}; + push @$unget_ref, @_; } sub get_arg { # Returns: - # next argument from input quoted and trimmed as needed + # next argument quoted and trimmed as needed + # (from $Global::argfile or $fh if given) # undef if end of file my $arg; + my $fh = shift || $Global::argfile; if(@Global::unget_arg) { return shift @Global::unget_arg; - } elsif(@Global::unget_lines) { - $arg = shift @Global::unget_lines; - if($Global::end_of_file_string and - $arg eq $Global::end_of_file_string) { - # Ignore the rest of input file - @Global::unget_lines = (); - return undef; - } - if($Global::ignore_empty) { - if($arg =~ /^\s*$/) { - return get_arg(); - } - } - if($Global::max_lines and more_arguments()) { - if($arg =~ /\s$/) { - # Trailing space => continued on next line - $arg .= get_arg(); - } - } - if($::opt_colsep) { - # split this into columns - my @columns = split /$::opt_colsep/o, $arg; - $::opt_N = $#columns+1; - $Global::max_number_of_args = $::opt_N; - if($Global::trim ne 'n') { - @columns = trim(@columns); - } - if($Global::input_is_filename) { - unget_arg(shell_quote(@columns)); - } else { - unget_arg(@columns); - } - return get_arg(); - } else { - if($Global::trim ne 'n') { - $arg = trim($arg); - } - if($Global::input_is_filename) { - $arg = shell_quote($arg); - } - } + } + if($::opt_colsep) { + $arg = get_column($fh); } else { - $arg = get_arg_from_fh($Global::argfile); - if(defined $arg) { - $Global::total_jobs++; + $arg = get_line_from_fh($fh); + } + if(defined $arg) { + if($Global::trim ne 'n') { + $arg = trim($arg); + } + if($Global::input_is_filename) { + $arg = shell_quote($arg); } } return $arg; @@ -3976,6 +3984,7 @@ sub __DEBUGGING__ {} sub debug { # Returns: N/A $Global::debug or return; + @_ = grep { defined $_ ? $_ : "" } @_; if($Global::original_stdout) { print $Global::original_stdout @_; } else { diff --git a/unittest/tests-to-run/test29.sh b/unittest/tests-to-run/test29.sh new file mode 100644 index 00000000..8c548861 --- /dev/null +++ b/unittest/tests-to-run/test29.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +echo '### Test of quoting of > bug' +echo '>/dev/null' | parallel echo + +echo '### Test of quoting of > bug if line continuation' +(echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo + +echo '### Test of --trim illegal' +stdout parallel --trim fj ::: echo + +echo '### Test of --colsep' +echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} +(echo 'a%c%b'; echo a%c%b%d) | parallel -k --colsep % echo {1} {3} {2} {4} +(echo a%c%b; echo d%f%e) | parallel -k --colsep % echo {1} {3} {2} +parallel -k --colsep % echo {1} {3} {2} ::: a%c%b d%f%e + +parallel -k --colsep % echo {1} {3} {2} ::: a%c%b + +parallel -k --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d + +echo '### Test of --colsep as regexp' +(echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4} + +parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d + + +(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} + +(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' +echo '### Test of -C' +(echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' +echo '### Test of --trim n' +(echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' +parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' diff --git a/unittest/wanted-results/test27 b/unittest/wanted-results/test27 index eba8b2b6..c8c3a383 100644 --- a/unittest/wanted-results/test27 +++ b/unittest/wanted-results/test27 @@ -113,8 +113,8 @@ Command line too long (42 >= 26) at number 1: \ \ \ \ \ ' 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 - 1 22 333 4444 55555\ 666666 -7777777 88888888\ 999999999\\\ 1\\\ 22 333 4444 55555 + 1 22 333 4444 55555 666666 +7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 ### -L3 -0 echo < ldata-0.xi @@ -452,8 +452,8 @@ from to x y 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 - 1 22 333 4444 55555\ 666666 -7777777 88888888\ 999999999\\\ 1\\\ 22 333 4444 55555 + 1 22 333 4444 55555 666666 +7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 ### -L2 -n2 echo < ldata.xi @@ -972,8 +972,8 @@ FIRST with 'single quotes' as well. IS OK 88888888 999999999 -1 22 333 4444 55555\ 666666 -7777777 88888888\ 999999999\\\ 1\\\ 22 +1 22 333 4444 55555 666666 +7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 diff --git a/unittest/wanted-results/test29 b/unittest/wanted-results/test29 new file mode 100644 index 00000000..051b350c --- /dev/null +++ b/unittest/wanted-results/test29 @@ -0,0 +1,34 @@ +### Test of quoting of > bug +>/dev/null +### Test of quoting of > bug if line continuation +> > > +### Test of --trim illegal +parallel: --trim must be one of: r l rl lr +### Test of --colsep +a b c +a b c {4} +a b c d +a b c +d e f +a b c +d e f +a b c +a b c {4} +a b c d +### Test of --colsep as regexp +a b c {4} +a b c d +a b c {4} +a b c d +a b c {4} +a b c d +a_b_c_{4} +a_b_c_d +### Test of -C +a_b_c_{4} +a_b_c_d +### Test of --trim n +a_b_\ c\ _{4} +a_\ b\ _c_d +a_b_c_{4} +a_b_c_d