parallel: Fixed bug #60908: --compress --result ' ' fails.

This commit is contained in:
Ole Tange 2021-07-11 20:12:52 +02:00
parent 71d008a5a4
commit a00733e2af
10 changed files with 174 additions and 85 deletions

View file

@ -252,7 +252,7 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20210722 ('Bill Cosby/Moïse/Derek Chauvin') released <<[stable]>> Subject: GNU Parallel 20210722 ('Branson+VSS Unity/Bill Cosby/Moïse/Derek Chauvin') released <<[stable]>>
GNU Parallel 20210722 ('') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4 GNU Parallel 20210722 ('') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4

View file

@ -2553,7 +2553,7 @@ sub open_joblog() {
# $opt::joblog # $opt::joblog
# $opt::results # $opt::results
# $Global::job_already_run # $Global::job_already_run
# %Global::fd # %Global::fh
my $append = 0; my $append = 0;
if(($opt::resume or $opt::resume_failed) if(($opt::resume or $opt::resume_failed)
and and
@ -2659,7 +2659,7 @@ sub open_joblog() {
} else { } else {
if($opt::joblog eq "-") { if($opt::joblog eq "-") {
# Use STDOUT as joblog # Use STDOUT as joblog
$Global::joblog = $Global::fd{1}; $Global::joblog = $Global::fh{1};
} elsif(not open($Global::joblog, ">", $opt::joblog)) { } elsif(not open($Global::joblog, ">", $opt::joblog)) {
# Overwrite the joblog # Overwrite the joblog
::error("Cannot write to --joblog $opt::joblog."); ::error("Cannot write to --joblog $opt::joblog.");
@ -2688,8 +2688,8 @@ sub open_json_csv() {
# by forcing all other output to /dev/null # by forcing all other output to /dev/null
open my $fd, ">", "/dev/null" or open my $fd, ">", "/dev/null" or
::die_bug("Can't >/dev/null in csv: $!"); ::die_bug("Can't >/dev/null in csv: $!");
$Global::fd{1} = $fd; $Global::fh{1} = $fd;
$Global::fd{2} = $fd; $Global::fh{2} = $fd;
} elsif($Global::csvsep) { } elsif($Global::csvsep) {
if(not open($Global::csv_fh,">",$opt::results)) { if(not open($Global::csv_fh,">",$opt::results)) {
::error("Cannot open results file `$opt::results': ". ::error("Cannot open results file `$opt::results': ".
@ -3168,7 +3168,7 @@ sub save_stdin_stdout_stderr() {
# Remember the original STDIN, STDOUT and STDERR # Remember the original STDIN, STDOUT and STDERR
# and file descriptors opened by the shell (e.g. 3>/tmp/foo) # and file descriptors opened by the shell (e.g. 3>/tmp/foo)
# Uses: # Uses:
# %Global::fd # %Global::fh
# $Global::original_stderr # $Global::original_stderr
# $Global::original_stdin # $Global::original_stdin
# Returns: N/A # Returns: N/A
@ -3183,7 +3183,7 @@ sub save_stdin_stdout_stderr() {
# 2-argument-open is used to be compatible with old perl 5.8.0 # 2-argument-open is used to be compatible with old perl 5.8.0
# bug #43570: Perl 5.8.0 creates 61 files # bug #43570: Perl 5.8.0 creates 61 files
if(open($fh,">&=$fdno")) { if(open($fh,">&=$fdno")) {
$Global::fd{$fdno}=$fh; $Global::fh{$fdno}=$fh;
} }
} }
open $Global::original_stderr, ">&", "STDERR" or open $Global::original_stderr, ">&", "STDERR" or
@ -3199,7 +3199,7 @@ sub enough_file_handles() {
# another job # another job
# Uses: # Uses:
# $opt::ungroup # $opt::ungroup
# %Global::fd # %Global::fh
# Returns: # Returns:
# 1 if ungrouped (thus not needing extra filehandles) # 1 if ungrouped (thus not needing extra filehandles)
# 0 if too few filehandles # 0 if too few filehandles
@ -3211,7 +3211,7 @@ sub enough_file_handles() {
# open3 uses 2 extra filehandles temporarily # open3 uses 2 extra filehandles temporarily
# We need a filehandle for each redirected file descriptor # We need a filehandle for each redirected file descriptor
# (normally just STDOUT and STDERR) # (normally just STDOUT and STDERR)
for my $i (1..(7+2+keys %Global::fd)) { for my $i (1..(7+2+keys %Global::fh)) {
$enough_filehandles &&= open($fh{$i}, "<", "/dev/null"); $enough_filehandles &&= open($fh{$i}, "<", "/dev/null");
} }
for (values %fh) { close $_; } for (values %fh) { close $_; }
@ -6202,14 +6202,14 @@ sub __DEBUGGING__() {}
sub debug(@) { sub debug(@) {
# Uses: # Uses:
# $Global::debug # $Global::debug
# %Global::fd # %Global::fh
# Returns: N/A # Returns: N/A
$Global::debug or return; $Global::debug or return;
@_ = grep { defined $_ ? $_ : "" } @_; @_ = grep { defined $_ ? $_ : "" } @_;
if($Global::debug eq "all" or $Global::debug eq $_[0]) { if($Global::debug eq "all" or $Global::debug eq $_[0]) {
if($Global::fd{2}) { if($Global::fh{2}) {
# Original stderr was saved # Original stderr was saved
my $stderr = $Global::fd{2}; my $stderr = $Global::fh{2};
print $stderr @_[1..$#_]; print $stderr @_[1..$#_];
} else { } else {
print STDERR @_[1..$#_]; print STDERR @_[1..$#_];
@ -8503,8 +8503,8 @@ sub openoutputfiles($) {
} }
} else { } else {
# --ungroup # --ungroup
open($outfhw,">&",$Global::fd{1}) || die; open($outfhw,">&",$Global::fh{1}) || die;
open($errfhw,">&",$Global::fd{2}) || die; open($errfhw,">&",$Global::fh{2}) || die;
# File name must be empty as it will otherwise be printed # File name must be empty as it will otherwise be printed
$outname = ""; $outname = "";
$errname = ""; $errname = "";
@ -8643,7 +8643,7 @@ sub filter_through_compress($) {
# thus output file can then be removed by the decompressor. # thus output file can then be removed by the decompressor.
my $wpid = open(my $fdw,"|-", "(echo > $comfile; ". my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".
empty_input_wrapper($opt::compress_program).") >". empty_input_wrapper($opt::compress_program).") >".
$self->fh($fdno,'name')) || die $?; ::Q($self->fh($fdno,'name'))) || die $?;
$self->set_fh($fdno,'w',$fdw); $self->set_fh($fdno,'w',$fdw);
$self->set_fh($fdno,'wpid',$wpid); $self->set_fh($fdno,'wpid',$wpid);
# Decompressor: open output; -s $comfile > 0: rm $comfile output; # Decompressor: open output; -s $comfile > 0: rm $comfile output;
@ -10296,10 +10296,10 @@ sub print($) {
} }
my $returnsize = $self->returnsize(); my $returnsize = $self->returnsize();
for my $fdno (sort { $a <=> $b } keys %Global::fd) { for my $fdno (sort { $a <=> $b } keys %Global::fh) {
# Sort by file descriptor numerically: 1,2,3,..,9,10,11 # Sort by file descriptor numerically: 1,2,3,..,9,10,11
$fdno == 0 and next; $fdno == 0 and next;
my $out_fd = $Global::fd{$fdno}; my $out_fh = $Global::fh{$fdno};
my $in_fh = $self->fh($fdno,"r"); my $in_fh = $self->fh($fdno,"r");
if(not $in_fh) { if(not $in_fh) {
if(not $Job::file_descriptor_warning_printed{$fdno}++) { if(not $Job::file_descriptor_warning_printed{$fdno}++) {
@ -10310,13 +10310,15 @@ sub print($) {
::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n"); ::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n");
if($opt::linebuffer) { if($opt::linebuffer) {
# Line buffered print out # Line buffered print out
$self->print_linebuffer($fdno,$in_fh,$out_fd); $self->print_linebuffer($fdno,$in_fh,$out_fh);
} elsif($opt::files) { } elsif($opt::files) {
$self->print_files($fdno,$in_fh,$out_fd); $self->print_files($fdno,$in_fh,$out_fh);
} elsif($opt::results) {
$self->print_results($fdno,$in_fh,$out_fh);
} else { } else {
$self->print_normal($fdno,$in_fh,$out_fd); $self->print_normal($fdno,$in_fh,$out_fh);
} }
flush $out_fd; flush $out_fh;
} }
::debug("print", "<<joboutput\n"); ::debug("print", "<<joboutput\n");
if(defined $self->{'exitstatus'} if(defined $self->{'exitstatus'}
@ -10512,7 +10514,7 @@ sub print_files($) {
# $opt::linebuffer = Print ASAP # $opt::linebuffer = Print ASAP
# Returns: N/A # Returns: N/A
my $self = shift; my $self = shift;
my ($fdno,$in_fh,$out_fd) = @_; my ($fdno,$in_fh,$out_fh) = @_;
# If the job is dead: close printing fh. Needed for --compress # If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w"); close $self->fh($fdno,"w");
@ -10534,7 +10536,7 @@ sub print_files($) {
::rm($self->fh($fdno,"unlink")); ::rm($self->fh($fdno,"unlink"));
} }
} elsif($fdno == 1 and $self->fh($fdno,"name")) { } elsif($fdno == 1 and $self->fh($fdno,"name")) {
print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; print $out_fh $self->tag(),$self->fh($fdno,"name"),"\n";
if($Global::membuffer) { if($Global::membuffer) {
push @{$self->{'output'}{$fdno}}, push @{$self->{'output'}{$fdno}},
$self->tag(), $self->fh($fdno,"name"); $self->tag(), $self->fh($fdno,"name");
@ -10547,7 +10549,7 @@ sub print_files($) {
sub print_linebuffer($) { sub print_linebuffer($) {
my $self = shift; my $self = shift;
my ($fdno,$in_fh,$out_fd) = @_; my ($fdno,$in_fh,$out_fh) = @_;
if(defined $self->{'exitstatus'}) { if(defined $self->{'exitstatus'}) {
# If the job is dead: close printing fh. Needed for --compress # If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w"); close $self->fh($fdno,"w");
@ -10566,7 +10568,7 @@ sub print_linebuffer($) {
if($opt::files or ($opt::results and not $Global::csvsep)) { if($opt::files or ($opt::results and not $Global::csvsep)) {
# Print filename # Print filename
if($fdno == 1 and not $self->fh($fdno,"printed")) { if($fdno == 1 and not $self->fh($fdno,"printed")) {
print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; print $out_fh $self->tag(),$self->fh($fdno,"name"),"\n";
if($Global::membuffer) { if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, $self->tag(), push(@{$self->{'output'}{$fdno}}, $self->tag(),
$self->fh($fdno,"name")); $self->fh($fdno,"name"));
@ -10607,7 +10609,7 @@ sub print_linebuffer($) {
} }
} }
# Print the partial line (halfline) and the last half # Print the partial line (halfline) and the last half
print $out_fd @$halfline_ref, substr($buf,0,$i); print $out_fh @$halfline_ref, substr($buf,0,$i);
# Buffer in memory for SQL and CSV-output # Buffer in memory for SQL and CSV-output
if($Global::membuffer) { if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, push(@{$self->{'output'}{$fdno}},
@ -10640,7 +10642,7 @@ sub print_linebuffer($) {
unshift @$halfline_ref, $self->tag(); unshift @$halfline_ref, $self->tag();
} }
# Print the partial line (halfline) # Print the partial line (halfline)
print $out_fd @{$self->{'halfline'}{$fdno}}; print $out_fh @{$self->{'halfline'}{$fdno}};
# Buffer in memory for SQL and CSV-output # Buffer in memory for SQL and CSV-output
if($Global::membuffer) { if($Global::membuffer) {
push(@{$self->{'output'}{$fdno}}, @$halfline_ref); push(@{$self->{'output'}{$fdno}}, @$halfline_ref);
@ -10667,7 +10669,7 @@ sub free_ressources() {
my $self = shift; my $self = shift;
if(not $opt::ungroup) { if(not $opt::ungroup) {
my $fh; my $fh;
for my $fdno (sort { $a <=> $b } keys %Global::fd) { for my $fdno (sort { $a <=> $b } keys %Global::fh) {
$fh = $self->fh($fdno,"w"); $fh = $self->fh($fdno,"w");
$fh and close $fh; $fh and close $fh;
$fh = $self->fh($fdno,"r"); $fh = $self->fh($fdno,"r");
@ -10772,6 +10774,61 @@ sub print_normal($) {
} }
} }
sub print_results($) {
my $self = shift;
my ($fdno,$in_fh,$out_fh) = @_;
my $buf;
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::compress_program." failed.");
$self->set_exitstatus(255);
}
if(not $self->virgin()) {
seek $in_fh, 0, 0;
# $in_fh is now ready for reading at position 0
my $outputlength = 0;
my @output;
if($Global::membuffer) {
# Read data into membuffer
if($opt::tag or $opt::tagstring) {
# Read line by line
local $/ = "\n";
my $tag = $self->tag();
while(<$in_fh>) {
$outputlength += length $_;
# Tag lines with \r, too
$_ =~ s/(?<=[\r])(?=.|$)/$tag/gs;
push @{$self->{'output'}{$fdno}}, $tag, $_;
}
} else {
# Most efficient way of copying data from $in_fh to $out_fh
while(sysread($in_fh,$buf,131072)) {
$outputlength += length $buf;
push @{$self->{'output'}{$fdno}}, $buf;
}
}
} else {
# Not membuffer: No need to read the file
if($opt::compress) {
$outputlength = -1;
} else {
# Determine $outputlength = file length
seek($in_fh, 0, 2) || ::die_bug("cannot seek result");
$outputlength = tell($in_fh);
}
}
if($fdno == 1) {
$self->add_returnsize($outputlength);
}
close $in_fh;
if($? and $opt::compress) {
::error($opt::decompress_program." failed.");
$self->set_exitstatus(255);
}
}
}
sub print_joblog($) { sub print_joblog($) {
my $self = shift; my $self = shift;
my $cmd; my $cmd;
@ -13747,7 +13804,7 @@ sub main() {
save_stdin_stdout_stderr(); save_stdin_stdout_stderr();
save_original_signal_handler(); save_original_signal_handler();
parse_options(); parse_options();
::debug("init", "Open file descriptors: ", join(" ",keys %Global::fd), "\n"); ::debug("init", "Open file descriptors: ", join(" ",keys %Global::fh), "\n");
my $number_of_args; my $number_of_args;
if($Global::max_number_of_args) { if($Global::max_number_of_args) {
$number_of_args = $Global::max_number_of_args; $number_of_args = $Global::max_number_of_args;

View file

@ -393,6 +393,7 @@ par_pipe_unneeded_procs() {
par_results_arg_256() { par_results_arg_256() {
echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)' echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)'
parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456; parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456;
cat parallel_test_dir/1/*/stdout
ls parallel_test_dir/1/ ls parallel_test_dir/1/
rm -rf parallel_test_dir rm -rf parallel_test_dir
} }
@ -822,6 +823,7 @@ par_results() {
echo "### --results test.csv" echo "### --results test.csv"
tmp=$(mktemp) tmp=$(mktemp)
parallel -k --results "$tmp"-dir echo ::: a b c parallel -k --results "$tmp"-dir echo ::: a b c
cat "$tmp"-dir/*/*/stdout
rm -r $tmp "$tmp"-dir rm -r $tmp "$tmp"-dir
} }

View file

@ -173,36 +173,6 @@ par_nice() {
parallel --retries 10 '! kill -TERM' ::: $pid 2>/dev/null parallel --retries 10 '! kill -TERM' ::: $pid 2>/dev/null
} }
par_test_diff_roundrobin_k() {
echo '### test there is difference on -k'
. $(which env_parallel.bash)
mytest() {
K=$1
doit() {
# Sleep random time ever 1k line
# to mix up which process gets the next block
perl -ne '$t++ % 1000 or select(undef, undef, undef, rand()/10);print' |
md5sum
}
export -f doit
seq 1000000 |
parallel --block 65K --pipe $K --roundrobin doit |
sort
}
export -f mytest
parset a,b,c mytest ::: -k -k ''
# a == b and a != c or error
if [ "$a" == "$b" ]; then
if [ "$a" != "$c" ]; then
echo OK
else
echo error a c
fi
else
echo error a b
fi
}
par_colsep() { par_colsep() {
echo '### Test of --colsep' echo '### Test of --colsep'
echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} echo 'a%c%b' | parallel --colsep % echo {1} {3} {2}

View file

@ -38,11 +38,24 @@ par_sqlandworker_uninstalled_dbd() {
} }
par_results_compress() { par_results_compress() {
tmp=$(mktemp) tmpdir=$(mktemp)
rm "$tmp" rm -r "$tmpdir"
parallel --results $tmp --compress echo ::: 1 | wc -l parallel --results $tmpdir --compress echo ::: 1
parallel --results $tmp echo ::: 1 | wc -l cat "$tmpdir"/*/*/stdout | pzstd -qdc
rm -r "$tmp"
rm -r "$tmpdir"
parallel --results $tmpdir echo ::: 1
cat "$tmpdir"/*/*/stdout
rm -r "$tmpdir"
parallel --results $tmpdir --compress echo ::: ' ' /
cat "$tmpdir"/*/*/stdout | pzstd -qdc
rm -r "$tmpdir"
parallel --results $tmpdir echo ::: ' ' /
cat "$tmpdir"/*/*/stdout
rm -r "$tmpdir"
} }
par_I_X_m() { par_I_X_m() {
@ -302,36 +315,42 @@ par_result() {
echo "### Test --results" echo "### Test --results"
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII
cat /tmp/parallel_results_test/testA/*/*/*/*/stdout | LC_ALL=C sort
ls /tmp/parallel_results_test/testA/*/*/*/*/* | LC_ALL=C sort ls /tmp/parallel_results_test/testA/*/*/*/*/* | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testA* rm -rf /tmp/parallel_results_test/testA*
echo "### Test --res" echo "### Test --res"
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
parallel -k --res /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII parallel -k --res /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII
cat /tmp/parallel_results_test/testD/*/*/*/*/stdout | LC_ALL=C sort
ls /tmp/parallel_results_test/testD/*/*/*/*/* | LC_ALL=C sort ls /tmp/parallel_results_test/testD/*/*/*/*/* | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testD* rm -rf /tmp/parallel_results_test/testD*
echo "### Test --result" echo "### Test --result"
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII
cat /tmp/parallel_results_test/testE/*/*/*/*/stdout | LC_ALL=C sort
ls /tmp/parallel_results_test/testE/*/*/*/*/* | LC_ALL=C sort ls /tmp/parallel_results_test/testE/*/*/*/*/* | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testE* rm -rf /tmp/parallel_results_test/testE*
echo "### Test --results --header :" echo "### Test --results --header :"
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII
cat /tmp/parallel_results_test/testB/*/*/*/*/stdout | LC_ALL=C sort
ls /tmp/parallel_results_test/testB/*/*/*/*/* | LC_ALL=C sort ls /tmp/parallel_results_test/testB/*/*/*/*/* | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testB* rm -rf /tmp/parallel_results_test/testB*
echo "### Test --results --header : named - a/b swapped" echo "### Test --results --header : named - a/b swapped"
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: b III IIII ::: a I II parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: b III IIII ::: a I II
cat /tmp/parallel_results_test/testC/*/*/*/*/stdout | LC_ALL=C sort
ls /tmp/parallel_results_test/testC/*/*/*/*/* | LC_ALL=C sort ls /tmp/parallel_results_test/testC/*/*/*/*/* | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testC* rm -rf /tmp/parallel_results_test/testC*
echo "### Test --results --header : piped" echo "### Test --results --header : piped"
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
(echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel --header : --result /tmp/parallel_results_test/testF true (echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel --header : --result /tmp/parallel_results_test/testF true
cat /tmp/parallel_results_test/testF/*/*/*/*/stdout | LC_ALL=C sort
find /tmp/parallel_results_test/testF/*/*/* | LC_ALL=C sort find /tmp/parallel_results_test/testF/*/*/* | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testF* rm -rf /tmp/parallel_results_test/testF*
@ -339,6 +358,7 @@ par_result() {
mkdir -p /tmp/parallel_results_test mkdir -p /tmp/parallel_results_test
(printf "Col1\t\n"; printf "v1\tv2\tv3\n"; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | (printf "Col1\t\n"; printf "v1\tv2\tv3\n"; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') |
parallel --header : --result /tmp/parallel_results_test/testG true parallel --header : --result /tmp/parallel_results_test/testG true
cat /tmp/parallel_results_test/testG/*/*/*/*/stdout | LC_ALL=C sort
find /tmp/parallel_results_test/testG/ | LC_ALL=C sort find /tmp/parallel_results_test/testG/ | LC_ALL=C sort
rm -rf /tmp/parallel_results_test/testG* rm -rf /tmp/parallel_results_test/testG*
} }
@ -346,21 +366,32 @@ par_result() {
par_result_replace() { par_result_replace() {
echo '### bug #49983: --results with {1}' echo '### bug #49983: --results with {1}'
parallel --results /tmp/par_{}_49983 -k echo ::: foo bar baz parallel --results /tmp/par_{}_49983 -k echo ::: foo bar baz
cat /tmp/par_*_49983
find /tmp/par_*_49983 | LC_ALL=C sort find /tmp/par_*_49983 | LC_ALL=C sort
rm -rf /tmp/par_*_49983 rm -rf /tmp/par_*_49983
parallel --results /tmp/par_{}_49983 -k echo ::: foo bar baz ::: A B C parallel --results /tmp/par_{}_49983 -k echo ::: foo bar baz ::: A B C
cat /tmp/par_*_49983
find /tmp/par_*_49983 | LC_ALL=C sort find /tmp/par_*_49983 | LC_ALL=C sort
rm -rf /tmp/par_*_49983 rm -rf /tmp/par_*_49983
parallel --results /tmp/par_{1}-{2}_49983 -k echo ::: foo bar baz ::: A B C parallel --results /tmp/par_{1}-{2}_49983 -k echo ::: foo bar baz ::: A B C
cat /tmp/par_*_49983
find /tmp/par_*_49983 | LC_ALL=C sort find /tmp/par_*_49983 | LC_ALL=C sort
rm -rf /tmp/par_*_49983 rm -rf /tmp/par_*_49983
parallel --results /tmp/par__49983 -k echo ::: foo bar baz ::: A B C parallel --results /tmp/par__49983 -k echo ::: foo bar baz ::: A B C
cat /tmp/par_*_49983/*/*/*/*/stdout
find /tmp/par_*_49983 | LC_ALL=C sort find /tmp/par_*_49983 | LC_ALL=C sort
rm -rf /tmp/par_*_49983 rm -rf /tmp/par_*_49983
parallel --results /tmp/par__49983 --header : -k echo ::: foo bar baz ::: A B C parallel --results /tmp/par__49983 --header : -k echo ::: foo bar baz ::: A B C
cat /tmp/par_*_49983/*/*/*/*/stdout
find /tmp/par_*_49983 | LC_ALL=C sort find /tmp/par_*_49983 | LC_ALL=C sort
rm -rf /tmp/par_*_49983 rm -rf /tmp/par_*_49983
parallel --results /tmp/par__49983-{}/ --header : -k echo ::: foo bar baz ::: A B C parallel --results /tmp/par__49983-{}/ --header : -k echo ::: foo bar baz ::: A B C
cat /tmp/par_*_49983*/stdout
find /tmp/par_*_49983-* | LC_ALL=C sort find /tmp/par_*_49983-* | LC_ALL=C sort
rm -rf /tmp/par_*_49983-* rm -rf /tmp/par_*_49983-*
} }

View file

@ -8,6 +8,36 @@
# Each should be taking 30-100s and be possible to run in parallel # Each should be taking 30-100s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_test_diff_roundrobin_k() {
echo '### test there is difference on -k'
. $(which env_parallel.bash)
mytest() {
K=$1
doit() {
# Sleep random time ever 1k line
# to mix up which process gets the next block
perl -ne '$t++ % 1000 or select(undef, undef, undef, rand()/10);print' |
md5sum
}
export -f doit
seq 1000000 |
parallel --block 65K --pipe $K --roundrobin doit |
sort
}
export -f mytest
parset a,b,c mytest ::: -k -k ''
# a == b and a != c or error
if [ "$a" == "$b" ]; then
if [ "$a" != "$c" ]; then
echo OK
else
echo error a c
fi
else
echo error a b
fi
}
par_load_from_PARALLEL() { par_load_from_PARALLEL() {
echo "### Test reading load from PARALLEL" echo "### Test reading load from PARALLEL"
export PARALLEL="--load 300%" export PARALLEL="--load 300%"

View file

@ -1077,8 +1077,6 @@ par_sockets_cores_threads ### Test --use-sockets-instead-of-threads
par_sockets_cores_threads threads done par_sockets_cores_threads threads done
par_sockets_cores_threads sockets done par_sockets_cores_threads sockets done
par_sockets_cores_threads Threads should complete first on machines with less than 8 sockets par_sockets_cores_threads Threads should complete first on machines with less than 8 sockets
par_test_diff_roundrobin_k ### test there is difference on -k
par_test_diff_roundrobin_k OK
par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this
par_tmp_full parallel: Error: Output is incomplete. par_tmp_full parallel: Error: Output is incomplete.
par_tmp_full parallel: Error: Cannot append to buffer file in /tmp/shm/parallel. par_tmp_full parallel: Error: Cannot append to buffer file in /tmp/shm/parallel.

View file

@ -635,8 +635,8 @@ par_result /tmp/parallel_results_test/testB/a/II/b/IIII/stderr
par_result /tmp/parallel_results_test/testB/a/II/b/IIII/stdout par_result /tmp/parallel_results_test/testB/a/II/b/IIII/stdout
par_result ### Test --results --header : named - a/b swapped par_result ### Test --results --header : named - a/b swapped
par_result I III par_result I III
par_result II III
par_result I IIII par_result I IIII
par_result II III
par_result II IIII par_result II IIII
par_result /tmp/parallel_results_test/testC/a/I/b/III/seq par_result /tmp/parallel_results_test/testC/a/I/b/III/seq
par_result /tmp/parallel_results_test/testC/a/I/b/III/stderr par_result /tmp/parallel_results_test/testC/a/I/b/III/stderr
@ -680,21 +680,21 @@ par_result /tmp/parallel_results_test/testG/Col1/v1/2/v2/3/v3/seq
par_result /tmp/parallel_results_test/testG/Col1/v1/2/v2/3/v3/stderr par_result /tmp/parallel_results_test/testG/Col1/v1/2/v2/3/v3/stderr
par_result /tmp/parallel_results_test/testG/Col1/v1/2/v2/3/v3/stdout par_result /tmp/parallel_results_test/testG/Col1/v1/2/v2/3/v3/stdout
par_result_replace ### bug #49983: --results with {1} par_result_replace ### bug #49983: --results with {1}
par_result_replace foo
par_result_replace bar par_result_replace bar
par_result_replace baz par_result_replace baz
par_result_replace foo
par_result_replace /tmp/par_bar_49983 par_result_replace /tmp/par_bar_49983
par_result_replace /tmp/par_baz_49983 par_result_replace /tmp/par_baz_49983
par_result_replace /tmp/par_foo_49983 par_result_replace /tmp/par_foo_49983
par_result_replace foo A
par_result_replace foo B
par_result_replace foo C
par_result_replace bar A par_result_replace bar A
par_result_replace bar B par_result_replace bar B
par_result_replace bar C par_result_replace bar C
par_result_replace baz A par_result_replace baz A
par_result_replace baz B par_result_replace baz B
par_result_replace baz C par_result_replace baz C
par_result_replace foo A
par_result_replace foo B
par_result_replace foo C
par_result_replace /tmp/par_bar A_49983 par_result_replace /tmp/par_bar A_49983
par_result_replace /tmp/par_bar B_49983 par_result_replace /tmp/par_bar B_49983
par_result_replace /tmp/par_bar C_49983 par_result_replace /tmp/par_bar C_49983
@ -704,15 +704,15 @@ par_result_replace /tmp/par_baz C_49983
par_result_replace /tmp/par_foo A_49983 par_result_replace /tmp/par_foo A_49983
par_result_replace /tmp/par_foo B_49983 par_result_replace /tmp/par_foo B_49983
par_result_replace /tmp/par_foo C_49983 par_result_replace /tmp/par_foo C_49983
par_result_replace foo A
par_result_replace foo B
par_result_replace foo C
par_result_replace bar A par_result_replace bar A
par_result_replace bar B par_result_replace bar B
par_result_replace bar C par_result_replace bar C
par_result_replace baz A par_result_replace baz A
par_result_replace baz B par_result_replace baz B
par_result_replace baz C par_result_replace baz C
par_result_replace foo A
par_result_replace foo B
par_result_replace foo C
par_result_replace /tmp/par_bar-A_49983 par_result_replace /tmp/par_bar-A_49983
par_result_replace /tmp/par_bar-B_49983 par_result_replace /tmp/par_bar-B_49983
par_result_replace /tmp/par_bar-C_49983 par_result_replace /tmp/par_bar-C_49983
@ -722,15 +722,15 @@ par_result_replace /tmp/par_baz-C_49983
par_result_replace /tmp/par_foo-A_49983 par_result_replace /tmp/par_foo-A_49983
par_result_replace /tmp/par_foo-B_49983 par_result_replace /tmp/par_foo-B_49983
par_result_replace /tmp/par_foo-C_49983 par_result_replace /tmp/par_foo-C_49983
par_result_replace foo A
par_result_replace foo B
par_result_replace foo C
par_result_replace bar A par_result_replace bar A
par_result_replace bar B par_result_replace bar B
par_result_replace bar C par_result_replace bar C
par_result_replace baz A par_result_replace baz A
par_result_replace baz B par_result_replace baz B
par_result_replace baz C par_result_replace baz C
par_result_replace foo A
par_result_replace foo B
par_result_replace foo C
par_result_replace /tmp/par__49983 par_result_replace /tmp/par__49983
par_result_replace /tmp/par__49983/1 par_result_replace /tmp/par__49983/1
par_result_replace /tmp/par__49983/1/bar par_result_replace /tmp/par__49983/1/bar
@ -776,8 +776,8 @@ par_result_replace /tmp/par__49983/1/foo/2/C/seq
par_result_replace /tmp/par__49983/1/foo/2/C/stderr par_result_replace /tmp/par__49983/1/foo/2/C/stderr
par_result_replace /tmp/par__49983/1/foo/2/C/stdout par_result_replace /tmp/par__49983/1/foo/2/C/stdout
par_result_replace bar B par_result_replace bar B
par_result_replace bar C
par_result_replace baz B par_result_replace baz B
par_result_replace bar C
par_result_replace baz C par_result_replace baz C
par_result_replace /tmp/par__49983 par_result_replace /tmp/par__49983
par_result_replace /tmp/par__49983/A par_result_replace /tmp/par__49983/A
@ -821,8 +821,12 @@ par_result_replace /tmp/par__49983-baz C
par_result_replace /tmp/par__49983-baz C/seq par_result_replace /tmp/par__49983-baz C/seq
par_result_replace /tmp/par__49983-baz C/stderr par_result_replace /tmp/par__49983-baz C/stderr
par_result_replace /tmp/par__49983-baz C/stdout par_result_replace /tmp/par__49983-baz C/stdout
par_results_compress 0
par_results_compress 1 par_results_compress 1
par_results_compress 1
par_results_compress
par_results_compress /
par_results_compress
par_results_compress /
par_seqreplace_long_line ### Test --seqreplace and line too long par_seqreplace_long_line ### Test --seqreplace and line too long
par_seqreplace_long_line 9 1 1 101 par_seqreplace_long_line 9 1 1 101
par_seqreplace_long_line 90 1 1 201 par_seqreplace_long_line 90 1 1 201

View file

@ -1718,3 +1718,5 @@ par_test_detected_shell test_known_shell_pipe static-sh Global::shell /usr/bin/s
par_test_detected_shell test_known_shell_pipe tcsh Global::shell /usr/bin/tcsh par_test_detected_shell test_known_shell_pipe tcsh Global::shell /usr/bin/tcsh
par_test_detected_shell test_known_shell_pipe yash Global::shell /usr/bin/yash par_test_detected_shell test_known_shell_pipe yash Global::shell /usr/bin/yash
par_test_detected_shell test_known_shell_pipe zsh Global::shell /usr/bin/zsh par_test_detected_shell test_known_shell_pipe zsh Global::shell /usr/bin/zsh
par_test_diff_roundrobin_k ### test there is difference on -k
par_test_diff_roundrobin_k OK

View file

@ -462,10 +462,6 @@ echo C
/usr/bin/bash: line 7: outdir/1/C/stderr: No such file or directory /usr/bin/bash: line 7: outdir/1/C/stderr: No such file or directory
/usr/bin/bash: line 8: outdir/1/C/stdout: No such file or directory /usr/bin/bash: line 8: outdir/1/C/stdout: No such file or directory
parallel --header : --results outdir echo ::: f1 A B ::: f2 C D parallel --header : --results outdir echo ::: f1 A B ::: f2 C D
A C
A D
B C
B D
outdir/f1/A/f2/C/seq outdir/f1/A/f2/C/seq
outdir/f1/A/f2/C/stderr outdir/f1/A/f2/C/stderr
outdir/f1/A/f2/C/stdout outdir/f1/A/f2/C/stdout
@ -979,7 +975,6 @@ Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,
parallel --results outdir --sqlandworker csv:///%2Ftmp/log2.csv \ parallel --results outdir --sqlandworker csv:///%2Ftmp/log2.csv \
seq ::: 10 ::: 12 13 14 seq ::: 10 ::: 12 13 14
cat /tmp/log2.csv cat /tmp/log2.csv
99
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr 1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr 2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
@ -1285,4 +1280,4 @@ mentioned in the release notes of next version of GNU Parallel.
echo A echo A
echo B echo B
echo C echo C
8 7