From bf24d5e4db97187be5363377eefd49ff47e78fd5 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 3 Jan 2018 09:03:16 +0100 Subject: [PATCH] Fixed bug #52769: --lb puts stdout on stderr sometimes. Fixed bug #52507: parset arr1 -v echo ::: fails. --dryrun should not append to joblog. Cleaned up print_tag(). Some testsuite reorg. --- src/parallel | 322 ++++++++--------- src/parallel.pod | 15 +- src/parallel_alternatives.pod | 2 +- src/parallel_book.pod | 110 +++++- testsuite/tests-to-run/parallel-local-0.3s.sh | 36 +- testsuite/tests-to-run/parallel-local-1s.sh | 81 ++--- testsuite/tests-to-run/parallel-local-30s.sh | 58 +-- testsuite/tests-to-run/parallel-local-3s.sh | 18 +- .../tests-to-run/parallel-local-race02.sh | 38 +- testsuite/tests-to-run/parallel-local-ssh5.sh | 70 ++-- testsuite/tests-to-run/parallel-local-ssh7.sh | 7 +- testsuite/tests-to-run/parallel-local-ssh8.sh | 4 +- testsuite/tests-to-run/parallel-local7.sh | 2 +- testsuite/tests-to-run/parallel-local8.sh | 57 ++- testsuite/tests-to-run/sql03.sh | 131 ++++--- testsuite/tests-to-run/test47.sh | 9 +- testsuite/wanted-results/parallel-freebsd | 16 +- testsuite/wanted-results/parallel-local-0.3s | 57 ++- testsuite/wanted-results/parallel-local-100s | 12 +- testsuite/wanted-results/parallel-local-10s | 56 +-- testsuite/wanted-results/parallel-local-1s | 342 ++++++++---------- testsuite/wanted-results/parallel-local-30s | 2 +- .../wanted-results/parallel-local-race02 | 6 +- testsuite/wanted-results/parallel-local-sql | 60 +-- testsuite/wanted-results/parallel-local-sql01 | 6 +- testsuite/wanted-results/parallel-local-ssh5 | 192 +++++----- testsuite/wanted-results/parallel-local-ssh7 | 22 +- testsuite/wanted-results/parallel-local-ssh8 | 1 - testsuite/wanted-results/parallel-local12 | 4 +- testsuite/wanted-results/parallel-local150 | 18 +- testsuite/wanted-results/parallel-local18 | 3 +- testsuite/wanted-results/parallel-local19 | 2 +- testsuite/wanted-results/parallel-local22 | 4 +- testsuite/wanted-results/parallel-local23 | 4 +- testsuite/wanted-results/parallel-local8 | 57 ++- testsuite/wanted-results/parallel-local9 | 4 +- testsuite/wanted-results/sql03 | 220 +++++------ 37 files changed, 1126 insertions(+), 922 deletions(-) diff --git a/src/parallel b/src/parallel index 04509c94..bcb297bc 100755 --- a/src/parallel +++ b/src/parallel @@ -1692,6 +1692,7 @@ sub open_joblog { ::wait_and_exit(255); } if(defined $opt::joblog and $opt::joblog =~ s/^\+//) { + # --joblog +filename = append to filename $append = 1; } if($opt::joblog @@ -1699,6 +1700,7 @@ sub open_joblog { ($opt::sqlmaster or not $opt::sqlworker)) { + # Do not log if --sqlworker if($opt::resume || $opt::resume_failed || $opt::retry_failed) { if(open(my $joblog_fh, "<", $opt::joblog)) { # Read the joblog @@ -1771,20 +1773,20 @@ sub open_joblog { close $joblog_fh; } } - if($append) { + if($opt::dryrun) { + # Do not write to joblog in a dry-run + if(not open($Global::joblog, ">", "/dev/null")) { + ::error("Cannot write to --joblog $opt::joblog."); + ::wait_and_exit(255); + } + } elsif($append) { # Append to joblog if(not open($Global::joblog, ">>", $opt::joblog)) { ::error("Cannot append to --joblog $opt::joblog."); ::wait_and_exit(255); } } else { - if($opt::dryrun) { - # Do not write to joblog in a dry-run - if(not open($Global::joblog, ">", "/dev/null")) { - ::error("Cannot write to --joblog $opt::joblog."); - ::wait_and_exit(255); - } - } elsif($opt::joblog eq "-") { + if($opt::joblog eq "-") { # Use STDOUT as joblog $Global::joblog = $Global::fd{1}; } elsif(not open($Global::joblog, ">", $opt::joblog)) { @@ -5496,7 +5498,7 @@ sub loadavg_too_high { 'linux' => $bsd2, 'minix' => "ps el|awk '{print \$1,\$11}'", 'mirbsd' => $bsd, - 'msys' => $sysv, + 'msys' => $cygwin, 'MSWin32' => $sysv, 'netbsd' => $bsd, 'nto' => $dummy, @@ -5574,7 +5576,8 @@ sub loadavg { # save it to a tmp file before moving it to the correct file ::debug("load", "Cmd: ", $cmd,"\n"); my $file = $self->{'loadavg_file'}; - my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa"); + # tmpfile on same filesystem as $file + my $tmpfile = $file.$$; ::qqx("($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile & )"); } return $self->{'loadavg'}; @@ -5723,7 +5726,7 @@ sub compute_number_of_processes { # The child takes one process slot # It will be killed later $SIG{'TERM'} = $Global::original_sig{'TERM'}; - if($^O eq "cygwin") { + if($^O eq "cygwin" or $^O eq "msys") { # The exec does not work on Cygwin sleep 10101010; } else { @@ -6836,7 +6839,7 @@ sub new { 'virgin' => 1, # Output used for SQL and CSV-output 'output' => { 1 => [], 2 => [] }, - 'halfline' => [], + 'halfline' => { 1 => [], 2 => [] }, }, ref($class) || $class; } @@ -7059,6 +7062,25 @@ sub openoutputfiles { } } +sub print_verbose_dryrun { + # If -v set: print command to stdout (possibly buffered) + # This must be done before starting the command + my $self = shift; + if($Global::verbose or $opt::dryrun) { + my $fh = $self->fh(1,"w"); + if($Global::verbose <= 1) { + print $fh $self->replaced(),"\n"; + } else { + # Verbose level > 1: Print the rsync and stuff + print $fh $self->wrapped(),"\n"; + } + } + if($opt::sqlworker) { + $Global::sql->update("SET Command = ? WHERE Seq = ".$self->seq(), + $self->replaced()); + } +} + sub add_rm { # Files to remove when job is done my $self = shift; @@ -8320,12 +8342,10 @@ sub start { $command = "true"; } $job->openoutputfiles(); + $job->print_verbose_dryrun(); # Call slot to store the slot value $job->slot(); my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w")); - if($opt::ungroup or $opt::sqlworker) { - print_dryrun_and_verbose($stdout_fh,$job,$command); - } if($opt::dryrun or $opt::sqlmaster) { $command = "true"; } $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; @@ -8341,6 +8361,7 @@ sub start { ::set_fh_non_blocking($stdin_fh); } $job->set_fh(0,"w",$stdin_fh); + if($opt::tee) { $job->set_virgin(0); } } elsif ($opt::tty and not $Global::tty_taken and -c "/dev/tty" and open(my $devtty_fh, "<", "/dev/tty")) { # Give /dev/tty to the command if no one else is using it @@ -8360,8 +8381,10 @@ sub start { }; $Global::tty_taken = $pid; close $devtty_fh; + $job->set_virgin(0); } else { $pid = open3_setpgrp(::gensym(),$stdout_fh,$stderr_fh,$command); + $job->set_virgin(0); } if($pid) { # A job was started @@ -8400,26 +8423,6 @@ sub interactive_start { } } -sub print_dryrun_and_verbose { - # For $opt::ungroup we print these ASAP - # For $opt::group they are part of print() - my $stdout_fh = shift; - my $job = shift; - my $command = shift; - if($opt::dryrun or $Global::verbose) { - if($Global::verbose <= 1) { - print $stdout_fh $job->replaced(),"\n"; - } else { - # Verbose level > 1: Print the rsync and stuff - print $stdout_fh $command,"\n"; - } - } - if($opt::sqlworker) { - $Global::sql->update("SET Command = ? WHERE Seq = ".$job->seq(), - $job->replaced()); - } -} - { my $tmuxsocket; @@ -8628,27 +8631,8 @@ sub print { # Check for disk full ::exit_if_disk_full(); - - if(($opt::dryrun or $Global::verbose) - and - not $self->{'verbose_printed'} - and - not $opt::sqlmaster - and - not $opt::sqlworker) { - $self->{'verbose_printed'}++; - if($Global::verbose <= 1) { - print STDOUT $self->replaced(),"\n"; - } else { - # Verbose level > 1: Print the rsync and stuff - print STDOUT $self->wrapped(),"\n"; - } - # If STDOUT and STDERR are merged, - # we want the command to be printed first - # so flush to avoid STDOUT being buffered - flush STDOUT; - } } + my $returnsize = $self->returnsize(); for my $fdno (sort { $a <=> $b } keys %Global::fd) { # Sort by file descriptor numerically: 1,2,3,..,9,10,11 @@ -8816,7 +8800,7 @@ sub print_files { if($opt::pipe and $self->virgin()) { # Nothing was printed to this job: - # cleanup unused tmp files if --files was set + # cleanup unused tmp files because --files was set for my $fdno (1,2) { ::rm($self->fh($fdno,"name")); ::rm($self->fh($fdno,"unlink")); @@ -8824,7 +8808,8 @@ sub print_files { } elsif($fdno == 1 and $self->fh($fdno,"name")) { print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; if($Global::membuffer) { - push @{$self->{'output'}{$fdno}}, $self->tag(), $self->fh($fdno,"name"); + push @{$self->{'output'}{$fdno}}, + $self->tag(), $self->fh($fdno,"name"); } $self->add_returnsize(-s $self->fh($fdno,"name")); # Mark as printed - do not print again @@ -8835,7 +8820,6 @@ sub print_files { sub print_linebuffer { my $self = shift; my ($fdno,$in_fh,$out_fd) = @_; - if(defined $self->{'exitstatus'}) { # If the job is dead: close printing fh. Needed for --compress close $self->fh($fdno,"w"); @@ -8850,111 +8834,92 @@ sub print_linebuffer { } } } - if($opt::files or ($opt::results and not $Global::csv)) { - if($fdno == 1 and not $self->fh($fdno,"printed")) { - print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; - if($Global::membuffer) { - push @{$self->{'output'}{$fdno}}, $self->tag(), $self->fh($fdno,"name"); - } - $self->set_fh($fdno,"printed",1); - } - # No need for reading $in_fh, as it it from "cat >/dev/null" - } else { - my $outputlength = 0; - my $halfline_ref = $self->{'halfline'}; - my ($buf,$i,$rv); - while($rv = sysread($in_fh, $buf, 131072)) { - $outputlength += $rv; - $i = (rindex($buf,"\n")+1) || (rindex($buf,"\r")+1); - if($i) { - # One or more complete lines were found - if($opt::tag or defined $opt::tagstring) { - # Replace ^ with $tag within the full line - my $tag = $self->tag(); - substr($buf,0,$i-1) =~ s/(?<=[\n\r])/$tag/gm; - $i = (rindex($buf,"\n")+1) || (rindex($buf,"\r")+1); - unshift @$halfline_ref, $tag; - } - # Print the partial line (halfline) and the last half - print $out_fd @$halfline_ref, substr($buf,0,$i); - # Buffer in memory for SQL and CSV-output - if($Global::membuffer) { - push(@{$self->{'output'}{$fdno}}, - @$halfline_ref, substr($buf,0,$i)); - } - # Remove the printed part by keeping the unprinted part - @$halfline_ref = (substr($buf,$i)); - } else { - push @$halfline_ref, $buf; - } - } - $self->add_returnsize($outputlength); - } - if(defined $self->{'exitstatus'}) { + if(not $self->virgin()) { if($opt::files or ($opt::results and not $Global::csv)) { - $self->add_returnsize(-s $self->fh($fdno,"name")); - } else { - # If the job is dead: print the remaining partial line - # read remaining - my $halfline_ref = $self->{'halfline'}; - if(grep /./, @$halfline_ref) { - $self->add_returnsize(length join("",@$halfline_ref)); - if($opt::tag or defined $opt::tagstring) { - # Prepend $tag the the remaining half line - unshift @$halfline_ref, $self->tag(); - } - # Print the partial line (halfline) - print $out_fd @{$self->{'halfline'}}; - # Buffer in memory for SQL and CSV-output + # Print filename + if($fdno == 1 and not $self->fh($fdno,"printed")) { + print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; if($Global::membuffer) { - push(@{$self->{'output'}{$fdno}}, @$halfline_ref); + push(@{$self->{'output'}{$fdno}}, $self->tag(), + $self->fh($fdno,"name")); } - @$halfline_ref = (); + $self->set_fh($fdno,"printed",1); } - } - if($self->fh($fdno,"rpid") and CORE::kill 0, $self->fh($fdno,"rpid")) { - # decompress still running + # No need for reading $in_fh, as it is from "cat >/dev/null" } else { - # decompress done: close fh - close $in_fh; - if($? and $opt::compress) { - ::error($opt::decompress_program." failed."); - $self->set_exitstatus(255); + # Read halflines and print full lines + my $outputlength = 0; + my $halfline_ref = $self->{'halfline'}{$fdno}; + my ($buf,$i,$rv); + while($rv = sysread($in_fh, $buf, 131072)) { + $outputlength += $rv; + # Treat both \n and \r as line end + $i = (rindex($buf,"\n")+1) || (rindex($buf,"\r")+1); + if($i) { + # One or more complete lines were found + if($opt::tag or defined $opt::tagstring) { + # Replace ^ with $tag within the full line + my $tag = $self->tag(); + substr($buf,0,$i-1) =~ s/(?<=[\n\r])/$tag/gm; + # The length changed, so find the new ending pos + $i = (rindex($buf,"\n")+1) || (rindex($buf,"\r")+1); + unshift @$halfline_ref, $tag; + } + # Print the partial line (halfline) and the last half + print $out_fd @$halfline_ref, substr($buf,0,$i); + # Buffer in memory for SQL and CSV-output + if($Global::membuffer) { + push(@{$self->{'output'}{$fdno}}, + @$halfline_ref, substr($buf,0,$i)); + } + # Remove the printed part by keeping the unprinted part + @$halfline_ref = (substr($buf,$i)); + } else { + # No newline, so append to the halfline + push @$halfline_ref, $buf; + } + } + $self->add_returnsize($outputlength); + } + if(defined $self->{'exitstatus'}) { + if($opt::files or ($opt::results and not $Global::csv)) { + $self->add_returnsize(-s $self->fh($fdno,"name")); + } else { + # If the job is dead: print the remaining partial line + # read remaining + my $halfline_ref = $self->{'halfline'}{$fdno}; + if(grep /./, @$halfline_ref) { + $self->add_returnsize(length join("",@$halfline_ref)); + if($opt::tag or defined $opt::tagstring) { + # Prepend $tag the the remaining half line + unshift @$halfline_ref, $self->tag(); + } + # Print the partial line (halfline) + print $out_fd @{$self->{'halfline'}{$fdno}}; + # Buffer in memory for SQL and CSV-output + if($Global::membuffer) { + push(@{$self->{'output'}{$fdno}}, @$halfline_ref); + } + @$halfline_ref = (); + } + } + if($self->fh($fdno,"rpid") and + CORE::kill 0, $self->fh($fdno,"rpid")) { + # decompress still running + } else { + # decompress done: close fh + close $in_fh; + if($? and $opt::compress) { + ::error($opt::decompress_program." failed."); + $self->set_exitstatus(255); + } } } } } sub print_tag { - my $self = shift; - my ($fdno,$in_fh,$out_fd) = @_; - my $buf; - local $/ = "\n"; - close $self->fh($fdno,"w"); - if($? and $opt::compress) { - ::error($opt::compress_program." failed."); - $self->set_exitstatus(255); - } - seek $in_fh, 0, 0; - # $in_fh is now ready for reading at position 0 - my $tag = $self->tag(); - my $outputlength = 0; - my @output; - while(<$in_fh>) { - print $out_fd $tag,$_; - $outputlength += length $_; - if($Global::membuffer) { - push @{$self->{'output'}{$fdno}}, $tag, $_; - } - } - if($fdno == 1) { - $self->add_returnsize($outputlength); - } - close $in_fh; - if($? and $opt::compress) { - ::error($opt::decompress_program." failed."); - $self->set_exitstatus(255); - } + return print_normal(@_); } sub print_normal { @@ -8966,24 +8931,40 @@ sub print_normal { ::error($opt::compress_program." failed."); $self->set_exitstatus(255); } - seek $in_fh, 0, 0; - # $in_fh is now ready for reading at position 0 - my $outputlength = 0; - my @output; - while(sysread($in_fh,$buf,131072)) { - print $out_fd $buf; - $outputlength += length $buf; - if($Global::membuffer) { - push @{$self->{'output'}{$fdno}}, $buf; + if(not $self->virgin()) { + seek $in_fh, 0, 0; + # $in_fh is now ready for reading at position 0 + my $outputlength = 0; + my @output; + + if($opt::tag or $opt::tagstring) { + # Read line by line + local $/ = "\n"; + my $tag = $self->tag(); + while(<$in_fh>) { + print $out_fd $tag,$_; + $outputlength += length $_; + if($Global::membuffer) { + push @{$self->{'output'}{$fdno}}, $tag, $_; + } + } + } else { + while(sysread($in_fh,$buf,131072)) { + print $out_fd $buf; + $outputlength += length $buf; + if($Global::membuffer) { + push @{$self->{'output'}{$fdno}}, $buf; + } + } + } + if($fdno == 1) { + $self->add_returnsize($outputlength); + } + close $in_fh; + if($? and $opt::compress) { + ::error($opt::decompress_program." failed."); + $self->set_exitstatus(255); } - } - if($fdno == 1) { - $self->add_returnsize($outputlength); - } - close $in_fh; - if($? and $opt::compress) { - ::error($opt::decompress_program." failed."); - $self->set_exitstatus(255); } } @@ -9024,7 +9005,8 @@ sub tag { sub hostgroups { my $self = shift; if(not defined $self->{'hostgroups'}) { - $self->{'hostgroups'} = $self->{'commandline'}->{'arg_list'}[0][0]->{'hostgroups'}; + $self->{'hostgroups'} = + $self->{'commandline'}->{'arg_list'}[0][0]->{'hostgroups'}; } return @{$self->{'hostgroups'}}; } diff --git a/src/parallel.pod b/src/parallel.pod index 49018d0b..b93215bb 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -93,8 +93,6 @@ B: Use B. B: Use B. -The command cannot contain the character \257 (macron: ¯). - =item B<{}> Input line. This replacement string will be replaced by a full line @@ -470,18 +468,21 @@ Implies B<--semaphore>. =item B<--citation> -Print the BibTeX entry for GNU B, silence citation -notice and exit. It will not run any commands. +Print the citation notice and BibTeX entry for GNU B, +silence citation notice for all future runs, and exit. It will not run +any commands. -If it is impossible for you to run B<--bibtex> you can use -B<--will-cite>. +If it is impossible for you to run B<--citation> you can instead use +B<--will-cite>, which will run commands, but which will only silence +the citation notice for this single run. If you use B<--will-cite> in scripts to be run by others you are making it harder for others to see the citation notice. The development of GNU B is indirectly financed through citations, so if your users do not know they should cite then you are making it harder to finance development. However, if you pay 10000 -EUR, you should feel free to use B<--will-cite> in scripts. +EUR, you have done your part to finance future development and should +feel free to use B<--will-cite> in scripts. =item B<--block> I diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 658ec48d..29e931e0 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -1334,7 +1334,7 @@ but not stderr. It buffers in RAM. {} does not work as replacement string. It does not support running functions. B does not support composed commands if run with B<--lines>, -and fails on B. +and fails on B. =head3 Examples diff --git a/src/parallel_book.pod b/src/parallel_book.pod index 3ea05858..66b0c661 100644 --- a/src/parallel_book.pod +++ b/src/parallel_book.pod @@ -2,6 +2,23 @@ =encoding utf8 +=head1 Why should you read this book? + +If you write shell scripts to do the same processing for different +input, then GNU B will make your life easier and make your +scripts run faster. + +The book is written so you get the juicy parts first: The goal is that +you read just enough to get you going. GNU B has an +overwhelming amount of special features to help in different +situations, and to avoid overloading you with information, the most +used features are presented first. + +All the examples are tested in Bash, and most will work in other +shells, too, but there are a few exceptions. So you are recommened to +use Bash while testing out the examples. + + =head1 Learn GNU Parallel in 5 minutes You just need to run commands in parallel. You do not care about fine @@ -9,11 +26,8 @@ tuning. To get going please run this to make some example files: - # If your system does not have 'seq', we will use 'jot' instead - if ! seq 1 2>/dev/null; then alias seq=jot; fi - - seq 5 | parallel 'seq {} > example.{}' - + # If your system does not have 'seq', replace 'seq' with 'jot' + seq 5 | parallel seq {} '>' example.{} =head2 Input sources @@ -26,7 +40,8 @@ This makes it easy to run the same program on some files: parallel wc ::: example.* -If you give multiple B<:::>s, GNU B will make all combinations: +If you give multiple B<:::>s, GNU B will generate all +combinations: parallel wc ::: -l -c ::: example.* @@ -53,10 +68,16 @@ anywhere by using the replacement string B<{}>: parallel echo counting {}';' wc -l {} ::: example.* When using multiple input sources you use the positional replacement -strings: +strings B<{1}> and B<{2}>: parallel echo count {1} in {2}';' wc {1} {2} ::: -l -c ::: example.* +You can check what will be run with B<--dry-run>: + + parallel --dry-run echo count {1} in {2}';' wc {1} {2} ::: -l -c ::: example.* + +This is a good idea to do for every command until you are comfortable +with GNU B. =head2 Controlling the output @@ -124,8 +145,9 @@ input or '-'), files and fifos and they can be mixed. Files are given after B<-a> or B<::::>. So these all do the same: parallel echo Dice1={1} Dice2={2} ::: 1 2 3 4 5 6 ::: 6 5 4 3 2 1 - parallel echo Dice1={1} Dice2={2} ::: <(seq 6) ::: <(seq 6 -1 1) + parallel echo Dice1={1} Dice2={2} :::: <(seq 6) :::: <(seq 6 -1 1) parallel echo Dice1={1} Dice2={2} :::: seq6 seq-6 + parallel echo Dice1={1} Dice2={2} :::: seq6 :::: seq-6 parallel -a seq6 -a seq-6 echo Dice1={1} Dice2={2} parallel -a seq6 echo Dice1={1} Dice2={2} :::: seq-6 parallel echo Dice1={1} Dice2={2} ::: 1 2 3 4 5 6 :::: seq-6 @@ -135,11 +157,25 @@ If stdin (standard input) is the only input source, you do not need the '-': cat seq6 | parallel echo Dice1={1} +=head3 Linking input sources + You can link multiple input sources with B<:::+> and B<::::+>: parallel echo {1}={2} ::: I II III IV V VI :::+ 1 2 3 4 5 6 parallel echo {1}={2} ::: I II III IV V VI ::::+ seq6 +The B<:::+> (and B<::::+>) will link each value to the corresponding +value in the previous input source, so value number 3 from the first +input source will be linked to value number 3 from the second input +source. + +You can combine B<:::+> and B<:::>, so you link 2 input sources, but +generate all combinations with other input sources: + + parallel echo Dice1={1}={2} Dice2={3}={4} ::: I II III IV V VI ::::+ seq6 \ + ::: VI V IV III II I ::::+ seq-6 + + =head2 Building the command line =head3 The command @@ -154,14 +190,66 @@ function is exported using B: export -f my_func parallel my_func ::: 1 2 3 -=head2 Copying environment +If the command is complex, it often improves readability to make it +into a function. -env_parallel =head3 The replacement strings -GNU B has some replacement strings to make it easier +GNU B has some replacement strings to make it easier to +refer to the input read from the input sources. +If the input is B then: + + {} = mydir/mysubdir/myfile.myext + {.} = mydir/mysubdir/myfile + {/} = myfile.myext + {//} = mydir/mysubdir + {/.} = myfile + {#} = the sequence number of the job + {%} = the job slot number + +When a job is started it gets sequence number that starts at 1 and +increases with 1 for each new job. The job also gets assigned a slot +number. This number is from 1 to the number of jobs running in +parallel. It is unique between the running jobs, but is re-used as +soon as a job finishes. + +=head4 The positional replacement strings + +The replacement strings have corresponding positional replacement +strings. If the value from the 3rd input source is +B: + + {3} = mydir/mysubdir/myfile.myext + {3.} = mydir/mysubdir/myfile + {3/} = myfile.myext + {3//} = mydir/mysubdir + {3/.} = myfile + +So the number of the input source is simply prepended inside the {}'s. + + +=head1 Replacement strings + +--plus replacement strings + +change the replacement string (-I --extensionreplace --basenamereplace --basenamereplace --dirnamereplace --basenameextensionreplace --seqreplace --slotreplace + +--header with named replacement string + +{= =} + +Dynamic replacement strings + +=head2 Defining replacement strings + + + + +=head2 Copying environment + +env_parallel =head2 Controlling the output diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 963195e5..f3820b8e 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -708,8 +708,8 @@ par_wd_3dot_local() { parallel --wd / 'pwd; echo $OLDPWD; echo' ::: OK parallel --wd /tmp 'pwd; echo $OLDPWD; echo' ::: OK parallel --wd ... 'pwd; echo $OLDPWD; echo' ::: OK | - perl -pe 's:/mnt/4tb::; s/'`hostname`'/hostname/g' | - perl -pe 's/\d+/0/g' + perl -pe 's:/mnt/4tb::; s:/home/tange:~:;' | + perl -pe 's/'`hostname`'/hostname/g; s/\d+/0/g' parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK } @@ -814,6 +814,38 @@ par_blocking_redir() { ) 2>&1 | sort } +par_pipepart_recend_recstart() { + echo 'bug #52343: --recend/--recstart does wrong thing with --pipepart' + tmp1=$(tempfile) + seq 10 > $tmp1 + parallel -k --pipepart -a $tmp1 --recend '\n' --recstart '6' --block 1 'echo a; cat' + parallel -k --pipe < $tmp1 --recend '\n' --recstart '6' --block 1 'echo a; cat' + rm $tmp1 2>/dev/null +} + +par_parset_v() { + echo 'bug #52507: parset arr1 -v echo ::: fails' + . `which env_parallel.bash` + parset arr1 -v seq ::: 1 2 3 + echo "${arr1[2]}" +} + +par_pipe_tag_v() { + echo 'pipe with --tag -v' + seq 3 | parallel -v --pipe --tagstring foo cat + # This should only give the filename + seq 3 | parallel -v --pipe --tagstring foo --files cat | + perl -pe 's:/tmp/par.*.par:/tmp/tmpfile.par:' +} + +par_dryrun_append_joblog() { + echo '--dry-run should not append to joblog' + rm -f /tmp/jl.$$ + parallel --jl /tmp/jl.$$ echo ::: 1 2 3 + parallel --dryrun --jl +/tmp/jl.$$ echo ::: 1 2 3 4 + # Job 4 should not show up: 3 lines + header = 4 + wc -l < /tmp/jl.$$ +} export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 3f938abb..829e53e2 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -3,55 +3,56 @@ # Simple jobs that never fails # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj+0 -k --joblog /tmp/jl-`basename $0` -L1 -echo "### BUG: The length for -X is not close to max (131072)"; - seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc - seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc - seq 1 60000 | parallel -X echo |head -n 1 |wc - seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc - seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc - seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc +par_max_length_len_128k() { + echo "### BUG: The length for -X is not close to max (131072)" -echo '### Test --fifo under csh' + seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc + seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc + seq 1 60000 | parallel -X echo |head -n 1 |wc + seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc + seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc + seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc +} - csh -c "seq 3000000 | parallel -k --pipe --fifo 'sleep .{#};cat {}|wc -c ; false; echo \$status; false'"; echo exit $? +par_fifo_under_csh() { + echo '### Test --fifo under csh' -echo '**' + csh -c "seq 3000000 | parallel -k --pipe --fifo 'sleep .{#};cat {}|wc -c ; false; echo \$status; false'" + echo exit $? +} -echo '### bug #44546: If --compress-program fails: fail' +par_compress_prg_fails() { + echo '### bug #44546: If --compress-program fails: fail' + doit() { + (parallel $* --compress-program false \ + echo \; sleep 1\; ls ::: /no-existing + echo $?) | tail -n1 + } + export -f doit + parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files +} - parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? - parallel --tag --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? - (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1 - parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $? - parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? - parallel --compress-program false echo \;ls ::: /no-existing; echo $? +par_pxz_complains() { + echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' -echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' + # The first line dumps core if run from make file. Why?! + stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file + stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file + stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output + stdout parallel --compress --compress-program pxz true ::: OK-if-no-output +} - # The first line dumps core if run from make file. Why?! - stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output - stdout parallel --compress --compress-program pxz true ::: OK-if-no-output +par_test_XI_mI() { + echo "### Test -I" + seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' -echo '**' + echo "### Test -X -I" + seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::' -echo "### Test -I"; - - seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' - -echo "### Test -X -I"; - - seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::' - -echo "### Test -m -I"; - - seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::' - - -EOF + echo "### Test -m -I" + seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::' +} par_linebuffer_files() { echo 'bug #48658: --linebuffer --files' @@ -97,7 +98,6 @@ par_no_newline_compress() { ::: tagstring '--tagstring {#}' -k \ ::: compress --compress -k \ ::: pipe pipe nopipe - } par_failing_compressor() { @@ -105,7 +105,6 @@ par_failing_compressor() { echo 'Test --tag/--line-buffer/--files in all combinations' echo 'Test working/failing compressor/decompressor in all combinations' echo '(-k is used as a dummy argument)' - stdout parallel -vk --header : --argsep ,,, \ parallel -k {tag} {lb} {files} --compress --compress-program {comp} --decompress-program {decomp} echo ::: C={comp},D={decomp} \ ,,, tag --tag -k \ diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 029a420b..e27f3c79 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -17,7 +17,8 @@ par_sigterm() { par_race_condition1() { echo '### Test race condition on 8 CPU (my laptop)' seq 1 5000000 > /tmp/parallel_race_cond - seq 1 10 | parallel -k "cat /tmp/parallel_race_cond | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" + seq 1 10 | + parallel -k "cat /tmp/parallel_race_cond | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" rm /tmp/parallel_race_cond } @@ -46,26 +47,40 @@ par_memory_leak() { if [ $small_max -lt $big ] ; then echo "Bad: Memleak likely." else - echo "Good: No memleak detected." + echo "Good: No memleak detected." fi } -par_linebuffer_matters_compress_tag() { - echo "### (--linebuffer) --compress --tag should give different output" +linebuffer_matters() { + echo "### (--linebuffer) --compress $TAG should give different output" nolbfile=$(mktemp) lbfile=$(mktemp) controlfile=$(mktemp) randomfile=$(mktemp) # Random data because it does not compress well # forcing the compress tool to spit out compressed blocks - head -c 10000000 /dev/urandom > $randomfile + perl -pe 'y/[A-Za-z]//cd; $t++ % 1000 or print "\n"' < /dev/urandom | + head -c 10000000 > $randomfile + export randomfile testfunc() { linebuffer="$1" - # Sleep 3 sec to give time to linebuffer-print the first part - parallel -j0 $linebuffer --compress --tag \ - "shuf $randomfile; sleep 3; shuf $randomfile; true" ::: {0..10} | - perl -ne '/^(\S+)\t/ and print "$1\n"' | uniq | sort + + incompressible_ascii() { + # generate some incompressible ascii + # with lines starting with the same string + id=$1 + shuf $randomfile | perl -pe 's/^/'$id' /' + # Sleep 3 sec to give time to linebuffer-print the first part + sleep 3 + shuf $randomfile | perl -pe 's/^/'$id' /' + echo + } + export -f incompressible_ascii + + parallel -j0 $linebuffer --compress $TAG \ + incompressible_ascii ::: {0..10} | + perl -ne '/^(\d+)\s/ and print "$1\n"' | uniq | sort } testfunc > $nolbfile & @@ -88,26 +103,15 @@ par_linebuffer_matters_compress_tag() { echo "BAD: control and nolb are not the same" fi } +export -f linebuffer_matters + +par_linebuffer_matters_compress_tag() { + export TAG=--tag + linebuffer_matters +} par_linebuffer_matters_compress() { - echo "### (--linebuffer) --compress should give different output" - random_data_with_id_prepended() { - perl -pe 's/^/'$1'/' /dev/urandom | - pv -qL 300000 | head -c 1000000 - } - export -f random_data_with_id_prepended - - nolb=$(seq 10 | - parallel -j0 --compress random_data_with_id_prepended {#} | - field 1 | uniq) - lb=$(seq 10 | - parallel -j0 --linebuffer --compress random_data_with_id_prepended {#} | - field 1 | uniq) - if [ "$lb" == "$nolb" ] ; then - echo "BAD: --linebuffer makes no difference" - else - echo "OK: --linebuffer makes a difference" - fi + linebuffer_matters } par_memfree() { diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index 337d37f2..bea28452 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -107,10 +107,12 @@ par_keeporder_roundrobin() { par_multiline_commands() { echo 'bug #50781: joblog format with multiline commands' - seq 1 3 | parallel --jl jl --timeout 2 'sleep {}; echo {}; + rm -f /tmp/jl.$$ + seq 1 3 | parallel --jl /tmp/jl.$$ --timeout 2 'sleep {}; echo {}; echo finish {}' - seq 1 3 | parallel --jl jl --timeout 4 --retry-failed 'sleep {}; echo {}; + seq 1 3 | parallel --jl /tmp/jl.$$ --timeout 4 --retry-failed 'sleep {}; echo {}; echo finish {}' + rm -f /tmp/jl.$$ } par_dryrun_timeout_ungroup() { @@ -120,10 +122,12 @@ par_dryrun_timeout_ungroup() { par_sqlworker_hostname() { echo 'bug #50901: --sqlworker should use hostname in the joblog instead of :' - parallel --sqlmaster :my/hostname echo ::: 1 2 3 - parallel -k --sqlworker :my/hostname + + MY=:mysqlunittest + parallel --sqlmaster $MY/hostname echo ::: 1 2 3 + parallel -k --sqlworker $MY/hostname hostname=`hostname` - sql :my 'select host from hostname;' | + sql $MY 'select host from hostname;' | perl -pe "s/$hostname//g" } @@ -163,13 +167,13 @@ par_parcat_mixing() { slow_output() { string=$1 perl -e 'print "'$string'"x9000,"start\n"' - sleep 1 + sleep 3 perl -e 'print "'$string'"x9000,"end\n"' } tmp1=$(mktmpfifo) tmp2=$(mktmpfifo) slow_output a > $tmp1 & - sleep 0.5 + sleep 2 slow_output b > $tmp2 & parcat $tmp1 $tmp2 | tr -s ab } diff --git a/testsuite/tests-to-run/parallel-local-race02.sh b/testsuite/tests-to-run/parallel-local-race02.sh index 4f42a5a9..c708c354 100644 --- a/testsuite/tests-to-run/parallel-local-race02.sh +++ b/testsuite/tests-to-run/parallel-local-race02.sh @@ -6,26 +6,26 @@ echo 'bug #46120: Suspend should suspend (at least local) children' echo 'it should burn 1.9 CPU seconds, but no more than that' echo 'The 5 second sleep will make it be killed by timeout when it fgs' - stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 & - sleep 1.9; - kill -TSTP -$!; - sleep 5; - fg; + stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | \grep -q CPUTIME=1 & + sleep 1.9; + kill -TSTP -$!; + sleep 5; + fg; echo Zero=OK $?' | grep -v '\[1\]' | grep -v 'SHA256' - - stdout bash -i -c 'echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | grep -q CPUTIME=1 & - sleep 1.9; - kill -TSTP -$!; - sleep 5; - fg; + + stdout bash -i -c 'echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | \grep -q CPUTIME=1 & + sleep 1.9; + kill -TSTP -$!; + sleep 5; + fg; echo Zero=OK $?' | grep -v '\[1\]' | grep -v 'SHA256' - + echo Control case: Burn for 2.9 seconds - stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 & - sleep 2.9; - kill -TSTP -$!; - sleep 5; - fg; + stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | \grep -q CPUTIME=1 & + sleep 2.9; + kill -TSTP -$!; + sleep 5; + fg; echo 1=OK $?' | grep -v '\[1\]' | grep -v 'SHA256' #} @@ -33,7 +33,7 @@ par_testhalt() { testhalt_false() { echo '### testhalt --halt '$1; (yes 0 | head -n 10; seq 10) | - stdout parallel -kj4 --delay 0.23 --halt $1 \ + stdout parallel -kj4 --delay 0.27 --halt $1 \ 'echo job {#}; sleep {= $_=0.3*($_+1+seq()) =}; exit {}'; echo $?; } testhalt_true() { @@ -57,7 +57,7 @@ par_hostgroup() { echo '### --hostgroup two group arg' parallel -k --sshdelay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.3{} ::: {1..8}@g1+g2 | sort - + echo '### --hostgroup one group arg' parallel --delay 0.2 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8}@g2 diff --git a/testsuite/tests-to-run/parallel-local-ssh5.sh b/testsuite/tests-to-run/parallel-local-ssh5.sh index ef7600c9..69fc821f 100644 --- a/testsuite/tests-to-run/parallel-local-ssh5.sh +++ b/testsuite/tests-to-run/parallel-local-ssh5.sh @@ -1,35 +1,53 @@ #!/bin/bash # SSH only allowed to localhost/lo -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj3 --delay 0.1 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -echo '### --ssh autossh - add commands that fail here' - export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0; - parallel -S lo echo ::: OK; - echo OK | parallel --pipe -S lo cat; - parallel -S lo false ::: a || echo OK should fail; - touch foo_autossh; stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh; rm foo_autossh*; -echo '### bug #46520: --basefile cleans up without --cleanup' - touch bug_46520; - parallel -S parallel@lo --bf bug_46520 ls ::: bug_46520; - ssh parallel@lo ls bug_46520; - parallel -S parallel@lo --cleanup --bf bug_46520 ls ::: bug_46520; - stdout ssh parallel@lo ls bug_46520 # should not exist +par_autossh() { + echo '### --ssh autossh - add commands that fail here' + export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0 + parallel -S lo echo ::: OK + echo OK | parallel --pipe -S lo cat + parallel -S lo false ::: a || echo OK should fail + touch foo_autossh + stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh + rm foo_autossh* +} -echo '### bug #36595: silent loss of input with --pipe and --sshlogin' - seq 10000 | xargs | parallel --pipe -S 8/localhost cat 2>/dev/null | wc +par_basefile_cleanup() { + echo '### bug #46520: --basefile cleans up without --cleanup' + touch bug_46520 + parallel -S parallel@lo --bf bug_46520 ls ::: bug_46520 + ssh parallel@lo ls bug_46520 + parallel -S parallel@lo --cleanup --bf bug_46520 ls ::: bug_46520 + stdout ssh parallel@lo ls bug_46520 # should not exist +} -echo 'bug #36707: --controlmaster eats jobs' - seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{} +par_input_loss_pipe() { + echo '### bug #36595: silent loss of input with --pipe and --sshlogin' + seq 10000 | xargs | parallel --pipe -S 8/localhost cat 2>/dev/null | wc +} -echo '### --ssh lsh' - parallel --ssh 'lsh -c aes256-ctr' -S lo echo ::: OK - echo OK | parallel --ssh 'lsh -c aes256-ctr' --pipe -S csh@lo cat - # Todo rsync/trc csh@lo - # Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler +par_controlmaster_eats() { + echo 'bug #36707: --controlmaster eats jobs' + seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{} +} -echo '### bug #45025: --pipe --retries does not reschedule on other host' - seq 1 300030| stdout parallel -k --retries 2 -S a.a,: --pipe 'wc;hostname' | perl -pe 's/'`hostname`'/localhost-:/' - stdout parallel --retries 2 --roundrobin echo ::: should fail +par_lsh() { + echo '### --ssh lsh' + parallel --ssh 'lsh -c aes256-ctr' -S lo echo ::: OK + echo OK | parallel --ssh 'lsh -c aes256-ctr' --pipe -S csh@lo cat + # Todo rsync/trc csh@lo + # Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler +} -EOF +par_pipe_retries() { + echo '### bug #45025: --pipe --retries does not reschedule on other host' + seq 1 300030 | + stdout parallel -k --retries 2 -S a.a,: --pipe 'wc;hostname' | + perl -pe 's/'`hostname`'/localhost-:/' + stdout parallel --retries 2 --roundrobin echo ::: should fail +} + +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | sort -r | + parallel --joblog /tmp/jl-`basename $0` -j3 --tag -k --delay 0.1 --retries 3 '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index c538f862..ad58a856 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -108,7 +108,8 @@ par_bash_man() { echo exit value $? should be 2 env_parallel --no-such-option >/dev/null - echo exit value $? should be 255 + # Sleep 1 to delay output to stderr to avoid race + echo exit value $? should be 255 `sleep 1` _EOF ) ssh bash@lo "$myscript" @@ -512,7 +513,9 @@ par_bash_underscore() { echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; _EOF ) - ssh bash@lo "$myscript" + stdout ssh bash@lo "$myscript" | + perl -pe 's/line ..:/line XX:/; + s@environment:@/bin/bash:@;' } par_csh_underscore() { diff --git a/testsuite/tests-to-run/parallel-local-ssh8.sh b/testsuite/tests-to-run/parallel-local-ssh8.sh index 6e991fcd..2810b34d 100644 --- a/testsuite/tests-to-run/parallel-local-ssh8.sh +++ b/testsuite/tests-to-run/parallel-local-ssh8.sh @@ -7,7 +7,7 @@ par_path_remote_bash() { rm -rf /tmp/parallel cp /usr/local/bin/parallel /tmp - cat <<'_EOS' | stdout ssh nopathbash@lo -T | grep -Ev 'updates are security updates|packages can be updated|System restart required|Welcome to|https://|Ubuntu|http://|^$' | uniq + cat <<'_EOS' | stdout ssh nopathbash@lo -T | grep -Ev 'updates are security updates|packages can be updated|System restart required|Welcome to|https://|Ubuntu|http://|from 13 to 17 years|mentor:|New release|do-release-upgrade|\s*^$' | uniq echo BASH Path before: $PATH with no parallel parallel echo ::: 1 # Race condition stderr/stdout @@ -28,7 +28,7 @@ par_path_remote_csh() { rm -rf /tmp/parallel cp /usr/local/bin/parallel /tmp - cat <<'_EOS' | stdout ssh nopathcsh@lo -T | grep -Ev 'updates are security updates|packages can be updated|System restart required|Welcome to|https://|Ubuntu|http://' | uniq + cat <<'_EOS' | stdout ssh nopathcsh@lo -T | grep -Ev 'updates are security updates|packages can be updated|System restart required|Welcome to|https://|Ubuntu|http://|from 13 to 17 years|mentor:|New release|do-release-upgrade|\s*^$' | uniq echo CSH Path before: $PATH with no parallel which parallel >& /dev/stdout echo '^^^^^^^^ Not found is OK' diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index c07d0981..dc622bf0 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -24,7 +24,7 @@ export -f par_tmux # echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground' # stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null | perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj3 --timeout 60 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 60 --retries 1 -k --joblog /tmp/jl-`basename $0` -L1 echo '### tmux-1.9' seq 000 100 | PARALLEL_TMUX=tmux-1.9 par_tmux diff --git a/testsuite/tests-to-run/parallel-local8.sh b/testsuite/tests-to-run/parallel-local8.sh index 5048f3fa..8f47b31d 100644 --- a/testsuite/tests-to-run/parallel-local8.sh +++ b/testsuite/tests-to-run/parallel-local8.sh @@ -1,21 +1,48 @@ #!/bin/bash -echo "### Test : as delimiter. This can be confusing for uptime ie. --load"; -parallel -k --load 300% -d : echo ::: a:b:c - export PARALLEL="--load 300%" -echo PARALLEL=$PARALLEL -for i in $(seq 2 10); do - i2=$[i*i] - seq $i2 | parallel -j0 --load 300% -kX echo {} | wc - seq 1 ${i2}0000 | nice parallel -kj20 --recend "\n" --spreadstdin gzip -1 | zcat | sort -n | md5sum -done +par_test_delimiter() { + echo "### Test : as delimiter. This can be confusing for uptime ie. --load"; + parallel -k --load 300% -d : echo ::: a:b:c +} -echo "### Test if --load blocks. Bug."; - seq 1 1000 | parallel -kj2 --load 300% --recend "\n" --spreadstdin gzip -1 | zcat | sort -n | md5sum - seq 1 1000 | parallel -kj240 --load 300% --recend "\n" --spreadstdin gzip -1 | zcat | sort -n | md5sum +par_squared() { + squared() { + i=$1 + i2=$[i*i] + seq $i2 | parallel -j0 --load 300% -kX echo {} | wc + seq 1 ${i2}0000 | + parallel -kj20 --recend "\n" --spreadstdin gzip -1 | + zcat | sort -n | md5sum + } + export -f squared -echo "### Test reading load from PARALLEL" - seq 1 1000000 | parallel -kj240 --recend "\n" --spreadstdin gzip -1 | zcat | sort -n | md5sum - seq 1 1000000 | parallel -kj20 --recend "\n" --spreadstdin gzip -1 | zcat | sort -n | md5sum + seq 10 -1 2 | stdout parallel -j5 -k squared | + grep -Ev 'processes took|Consider adjusting -j' +} + +par_load_blocks() { + echo "### Test if --load blocks. Bug."; + (seq 1 1000 | + parallel -kj2 --load 300% --recend "\n" --spreadstdin gzip -1 | + zcat | sort -n | md5sum + seq 1 1000 | + parallel -kj200 --load 300% --recend "\n" --spreadstdin gzip -1 | + zcat | sort -n | md5sum) 2>&1 | + grep -Ev 'processes took|Consider adjusting -j' +} + +par_load_from_PARALLEL() { + echo "### Test reading load from PARALLEL" + seq 1 1000000 | + parallel -kj200 --recend "\n" --spreadstdin gzip -1 | + zcat | sort -n | md5sum + seq 1 1000000 | + parallel -kj20 --recend "\n" --spreadstdin gzip -1 | + zcat | sort -n | md5sum +} + +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | sort | + parallel -j0 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/sql03.sh b/testsuite/tests-to-run/sql03.sh index 35d780ed..22681f37 100644 --- a/testsuite/tests-to-run/sql03.sh +++ b/testsuite/tests-to-run/sql03.sh @@ -3,8 +3,6 @@ mysqlrootpass=${mysqlrootpass:-M-b+Ydjq4ejT4E} MYSQL_ADMIN_DBURL=mysql://root:$mysqlrootpass@/mysql -exec 2>&1 - # Setup sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'" sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest; @@ -13,63 +11,102 @@ sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5 sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';" MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@/sqlunittest +export MYSQL_TEST_DBURL +export MYSQL_ADMIN_DBURL -echo '### Test reading sql from url command line' -echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it works' as 'Test reading SQL from command line';" +uniqify() { + file=$1 + perl -ne '$seen{$_}++ || print' "$file" > "$file".$$ + chmod 600 "$file".$$ + mv "$file".$$ "$file" +} +export -f uniqify -echo '### Test reading sql from url command line %-quoting' -echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it%20works' as 'Test%20%-quoting%20SQL from command line';" +par_sql_from_url() { + echo '### Test reading sql from url command line' + echo | + sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it works' as 'Test reading SQL from command line';" -echo "### Test .sql/aliases with url on commandline" -echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> ~/.sql/aliases -perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases -echo | sql ":sqlunittest?SELECT 'Yes it%20works' as 'Test if .sql/aliases with %-quoting works';" + echo '### Test reading sql from url command line %-quoting' + echo | + sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it%20works' as 'Test%20%-quoting%20SQL from command line';" -echo "### Test cyclic alias .sql/aliases" -echo :cyclic :cyclic2 >> ~/.sql/aliases -echo :cyclic2 :cyclic3 >> ~/.sql/aliases -echo :cyclic3 :cyclic >> ~/.sql/aliases -perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases -stdout sql ":cyclic3?SELECT 'NO IT DID NOT' as 'Test if :cyclic is found works';" + echo "### Test .sql/aliases with url on commandline" + echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> ~/.sql/aliases + uniqify ~/.sql/aliases + echo | + sql ":sqlunittest?SELECT 'Yes it%20works' as 'Test if .sql/aliases with %-quoting works';" +} -echo "### Test alias with statement .sql/aliases" -echo ":testselect sqlite:///%2Ftmp%2Ffile.sqlite?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases -perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases -echo | stdout sql :testselect -echo ":testselectmysql mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases -perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases -echo | stdout sql :testselect -echo | stdout sql :testselectmysql +par_test_cyclic() { + echo "### Test cyclic alias .sql/aliases" + echo :cyclic :cyclic2 >> ~/.sql/aliases + echo :cyclic2 :cyclic3 >> ~/.sql/aliases + echo :cyclic3 :cyclic >> ~/.sql/aliases + uniqify ~/.sql/aliases + stdout sql ":cyclic3?SELECT 'NO IT DID NOT' as 'Test if :cyclic is found works';" +} -echo "### Test alias followed by SQL as arg" -echo ignored | stdout sql :testselect "select 'Arg on cmdline';" +par_test_alias_with_statement() { + echo "### Test alias with statement .sql/aliases" + echo ":testselect sqlite:///%2Ftmp%2Ffile.sqlite?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases + uniqify ~/.sql/aliases + echo | stdout sql :testselect + echo ":testselectmysql mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases + uniqify ~/.sql/aliases + echo | stdout sql :testselect + echo | stdout sql :testselectmysql -echo "### Test alias with query followed by SQL as arg" -echo ignored | stdout sql :testselect" select 'Query added to alias';" "select 'Arg on cmdline';" + echo "### Test alias followed by SQL as arg" + echo ignored | stdout sql :testselect "select 'Arg on cmdline';" -echo "### Test alias with statement .sql/aliases" -echo "select 'Query from stdin';" | sql :testselect" select 'Query added to alias';" -echo "select 'Query from stdin';" | sql :testselectmysql" select 'Query added to alias';" + echo "### Test alias with query followed by SQL as arg" + echo ignored | stdout sql :testselect" select 'Query added to alias';" "select 'Arg on cmdline';" -echo "### Test empty dburl" -stdout sql '' + echo "### Test alias with statement .sql/aliases" + echo "select 'Query from stdin';" | sql :testselect" select 'Query added to alias';" + echo "select 'Query from stdin';" | sql :testselectmysql" select 'Query added to alias';" +} -echo "### Test dburl :" -stdout sql ':' +par_test_empty_dburl() { + echo "### Test empty dburl" + stdout sql '' +} -echo "### Test oracle with multiple arguments on the command line" -echo ":oraunittest oracle://hr:hr@oracle11.tange.dk/xe" >> ~/.sql/aliases -perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases -sql :oraunittest "WHENEVER SQLERROR EXIT FAILURE" "SELECT 'arg2' FROM DUAL;" "SELECT 'arg3' FROM DUAL;" +par_test_dburl_colon() { + echo "### Test dburl :" + stdout sql ':' +} -echo "### Test oracle with \n arguments on the command line" -sql :oraunittest 'select 1 from dual;\nselect 2 from dual;\x0aselect 3 from dual;' +par_multiarg_on_command_line() { + echo "### Test oracle with multiple arguments on the command line" + echo ":oraunittest oracle://hr:hr@oracle11.tange.dk/xe" >> ~/.sql/aliases + uniqify ~/.sql/aliases + sql :oraunittest "WHENEVER SQLERROR EXIT FAILURE" "SELECT 'arg2' FROM DUAL;" "SELECT 'arg3' FROM DUAL;" +} -echo "### Test --show-tables" -sql --show-tables :oraunittest | sort +par_newline_on_commandline() { + echo "### Test oracle with \n arguments on the command line" + sql :oraunittest 'select 1 from dual;\nselect 2 from dual;\x0aselect 3 from dual;' +} -echo "### Test --show-databases" -sql --show-databases :oraunittest +par_showtables() { + echo "### Test --show-tables" + sql --show-tables :oraunittest | sort +} -echo "### Test --listproc" -sql --listproc :oraunittest +par_showdatabases() { + echo "### Test --show-databases" + sql --show-databases :oraunittest +} + +par_listproc() { + echo "### Test --listproc" + sql --listproc :oraunittest + sql --listproc $MYSQL_TEST_DBURL | + perl -pe 's/^\d+/XXX/' +} + +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | sort | + parallel -j0 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/test47.sh b/testsuite/tests-to-run/test47.sh index fdfeb531..bf433448 100644 --- a/testsuite/tests-to-run/test47.sh +++ b/testsuite/tests-to-run/test47.sh @@ -1,10 +1,12 @@ #!/bin/bash +# must be other servers than localhost SERVER1=parallel-server1 SERVER2=parallel-server2 SSHLOGIN1=parallel@$SERVER1 SSHLOGIN2=parallel@$SERVER2 - +export SSHLOGIN1 +export SSHLOGIN2 # Minimal version of test17 export PARALLEL=-j8 @@ -26,7 +28,8 @@ cat /tmp/test17 | parallel -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel echo '### --transfer - abspath' stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*' -cat /tmp/test17abs | parallel -k --transfer --sshlogin $SSHLOGIN1,$SSHLOGIN2 cat {}";"rm {} +cat /tmp/test17abs | + parallel -k --transfer --sshlogin $SSHLOGIN1,$SSHLOGIN2 cat {}";"rm {} # One of these should give the empty dir /tmp/parallel.file echo good if no file stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' @@ -134,7 +137,6 @@ stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK - echo '### --transfer --return --cleanup - relpath' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' @@ -198,4 +200,3 @@ echo '### --transfer --cleanup - multiple argument files' parallel --xapply -kv --transferfile {1} --transferfile {2} --cleanup -S$SSHLOGIN2 cat {2} {1} :::: /tmp/test17rel <(sort -r /tmp/test17abs) # Should give: No such file or directory stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK - diff --git a/testsuite/wanted-results/parallel-freebsd b/testsuite/wanted-results/parallel-freebsd index 961d80fe..9b8f13f3 100644 --- a/testsuite/wanted-results/parallel-freebsd +++ b/testsuite/wanted-results/parallel-freebsd @@ -1,20 +1,20 @@ ### These tests requires VirtualBox running with the following images tange@freebsd7 -par_compress_pipe 2>&1 +par_compress_pipe par_compress_pipe 2>&1 par_compress_pipe Test --compress --pipe par_compress_pipe 1000 1000 3893 par_compress_pipe bug #41613: --compress --line-buffer no newline par_compress_pipe It worked -par_env_parallel 2>&1 +par_env_parallel par_env_parallel 2>&1 par_env_parallel ### env_parallel on Freebsd par_env_parallel myalias1 myvar_line 1 par_env_parallel myvar_line 2 par_env_parallel myalias2 foo -par_load 2>&1 +par_load par_load 2>&1 par_load ### Test --load (must give 1=true) par_load par_load 1 -par_no_more_procs 2>&1 +par_no_more_procs par_no_more_procs 2>&1 par_no_more_procs bug #40136: FreeBSD: No more processes par_no_more_procs First started par_no_more_procs Second started @@ -28,14 +28,14 @@ par_no_more_procs The third finished par_no_more_procs par_no_more_procs The fourth finished par_no_more_procs -par_round_robin 2>&1 +par_round_robin par_round_robin 2>&1 par_round_robin bug #40133: FreeBSD: --round-robin gives no output par_round_robin 1000000 1000000 6888896 par_round_robin 1000000 1000000 6888896 -par_sem_fg 2>&1 +par_sem_fg par_sem_fg 2>&1 par_sem_fg bug #40135: FreeBSD: sem --fg does not finish under /bin/sh par_sem_fg The job finished -par_shebang 2>&1 +par_shebang par_shebang 2>&1 par_shebang bug #40134: FreeBSD: --shebang not working par_shebang It par_shebang worked @@ -51,7 +51,7 @@ par_shebang /usr/bin/perl -w ./shebang-wrap-opt with par_shebang with par_shebang /usr/bin/perl -w ./shebang-wrap-opt options par_shebang options -par_shellshock_bug 2>&1 +par_shellshock_bug par_shellshock_bug 2>&1 par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env _ par_shellshock_bug Non-shellshock-hardened to non-shellshock-hardened par_shellshock_bug Function non-shellshock-hardened diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 60b772fc..c2e0a643 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -552,7 +552,7 @@ e echo '### test too long args' ### test too long args perl -e 'print "z"x1000000' | parallel echo 2>&1 -parallel: Error: Command line too long (1000005 >= 131057) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +parallel: Error: Command line too long (1000005 >= 131049) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... perl -e 'print "z"x1000000' | xargs echo 2>&1 xargs: argument line too long (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -km -s 10 echo @@ -819,7 +819,7 @@ echo far ### Test --show-limits (echo b; echo c; echo f) | parallel -k --show-limits echo {}ar Maximal size of command: 131049 -Maximal used size of command: 131057 +Maximal used size of command: 131049 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please @@ -1358,6 +1358,15 @@ par_blocking_redir stdout par_colsep_0 bug --colsep 0 par_colsep_0 OK par_colsep_0 OK +par_dryrun_append_joblog --dry-run should not append to joblog +par_dryrun_append_joblog 1 +par_dryrun_append_joblog 2 +par_dryrun_append_joblog 3 +par_dryrun_append_joblog echo 1 +par_dryrun_append_joblog echo 2 +par_dryrun_append_joblog echo 3 +par_dryrun_append_joblog echo 4 +par_dryrun_append_joblog 4 par_empty bug #: par_empty true par_empty_line ### Test bug: empty line for | sh with -k @@ -1492,6 +1501,11 @@ par_parset 2 par_parset 3 par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 bar +par_parset_v bug #52507: parset arr1 -v echo ::: fails +par_parset_v seq 3 +par_parset_v 1 +par_parset_v 2 +par_parset_v 3 par_pipe_compress_blocks ### bug #41482: --pipe --compress blocks at different -j/seq combinations par_pipe_compress_blocks 1 par_pipe_compress_blocks echo 1-4 + 1-4 @@ -1552,8 +1566,43 @@ par_pipe_record_size_in_lines bug 34958-2 par_pipe_record_size_in_lines 9 par_pipe_record_size_in_lines 10 par_pipe_record_size_in_lines bug 34958-2 +par_pipe_tag_v pipe with --tag -v +par_pipe_tag_v foo cat +par_pipe_tag_v foo 1 +par_pipe_tag_v foo 2 +par_pipe_tag_v foo 3 +par_pipe_tag_v foo /tmp/tmpfile.par par_pipepart_block_bigger_2G ### Test that --pipepart can have blocks > 2GB par_pipepart_block_bigger_2G 1 1 4 +par_pipepart_recend_recstart bug #52343: --recend/--recstart does wrong thing with --pipepart +par_pipepart_recend_recstart a +par_pipepart_recend_recstart 1 +par_pipepart_recend_recstart 2 +par_pipepart_recend_recstart 3 +par_pipepart_recend_recstart 4 +par_pipepart_recend_recstart 5 +par_pipepart_recend_recstart a +par_pipepart_recend_recstart 6 +par_pipepart_recend_recstart 7 +par_pipepart_recend_recstart 8 +par_pipepart_recend_recstart 9 +par_pipepart_recend_recstart 10 +par_pipepart_recend_recstart parallel: Warning: A record was longer than 1. Increasing to --blocksize 3. +par_pipepart_recend_recstart parallel: Warning: A record was longer than 3. Increasing to --blocksize 5. +par_pipepart_recend_recstart parallel: Warning: A record was longer than 5. Increasing to --blocksize 8. +par_pipepart_recend_recstart parallel: Warning: A record was longer than 8. Increasing to --blocksize 12. +par_pipepart_recend_recstart a +par_pipepart_recend_recstart 1 +par_pipepart_recend_recstart 2 +par_pipepart_recend_recstart 3 +par_pipepart_recend_recstart 4 +par_pipepart_recend_recstart 5 +par_pipepart_recend_recstart a +par_pipepart_recend_recstart 6 +par_pipepart_recend_recstart 7 +par_pipepart_recend_recstart 8 +par_pipepart_recend_recstart 9 +par_pipepart_recend_recstart 10 par_python_children ### bug #49970: Python child process dies if --env is used par_retries_replacement_string 11 par_retries_replacement_string 22 @@ -1670,8 +1719,8 @@ par_wd_3dot_local OK par_wd_3dot_local /tmp par_wd_3dot_local ~/privat/parallel/testsuite par_wd_3dot_local OK -par_wd_3dot_local /home/tange/.parallel/tmp/hostname-0-0 -par_wd_3dot_local /home/tange/privat/parallel/testsuite +par_wd_3dot_local ~/.parallel/tmp/hostname-0-0 +par_wd_3dot_local ~/privat/parallel/testsuite par_wd_3dot_local OK par_wd_3dot_local ~/privat/parallel/testsuite par_wd_3dot_local ~/privat/parallel/testsuite diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index cafb8945..3fc58a9d 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -1,4 +1,4 @@ -par_halt_on_error 2>&1 +par_halt_on_error par_halt_on_error 2>&1 par_halt_on_error -2 true true 0 par_halt_on_error -2 true true 0 par_halt_on_error -2 true true parallel: This job succeeded: @@ -155,10 +155,10 @@ par_halt_on_error 2 false false parallel: This job failed: par_halt_on_error 2 false false sleep 1;false par_halt_on_error 2 false false parallel: This job failed: par_halt_on_error 2 false false sleep 1;false -par_mem_leak 2>&1 +par_mem_leak par_mem_leak 2>&1 par_mem_leak ### test for mem leak par_mem_leak no mem leak detected -par_outside_file_handle_limit 2>&1 +par_outside_file_handle_limit par_outside_file_handle_limit 2>&1 par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 999 jobs in parallel. par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 999 --pipe parallel -j0' or @@ -166,17 +166,17 @@ par_outside_file_handle_limit parallel: Warning: raising 'ulimit -n' or 'nofile' par_outside_file_handle_limit parallel: Warning: or /proc/sys/fs/file-max may help. par_outside_file_handle_limit Start par_outside_file_handle_limit end -par_over_4GB 2>&1 +par_over_4GB par_over_4GB 2>&1 par_over_4GB ### Test if we can deal with output > 4 GB par_over_4GB 46a318993dfc8e2afd71ff2bc6f605f1 - -par_retries_unreachable 2>&1 +par_retries_unreachable par_retries_unreachable 2>&1 par_retries_unreachable ### Test of --retries on unreachable host par_retries_unreachable parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. par_retries_unreachable echo 1 par_retries_unreachable 1 par_retries_unreachable echo 2 par_retries_unreachable 2 -par_timeout 2>&1 +par_timeout par_timeout 2>&1 par_timeout ### test --timeout par_timeout OK par_timeout OK diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index c0cfa4d7..037f08cd 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -7,7 +7,7 @@ par__pipe_tee bug #45479: --pipe/--pipepart --tee par__pipe_tee --pipe --tee par__pipe_tee 3221225472 par__pipepart_spawn ### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922 -par__pipepart_spawn 1:local / 8 / 999 +par__pipepart_spawn 1:local / 2 / 999 par__pipepart_tee bug #45479: --pipe/--pipepart --tee par__pipepart_tee --pipepart --tee par__pipepart_tee 3221225472 @@ -258,7 +258,7 @@ par_linebuffer_tag_slow_output lo par_linebuffer_tag_slow_output lo par_linebuffer_tag_slow_output lo par_maxlinelen_X_I ### Test max line length -X -I -par_maxlinelen_X_I 201ecb8ca789cecf39ee914a6d75611b - +par_maxlinelen_X_I 3cfc69ee81b0fe7fdbe8eb059ad2da61 - par_maxlinelen_X_I Chars per line (817788/7): 116826 par_maxlinelen_m_I ### Test max line length -m -I par_maxlinelen_m_I c78bd0799bc23d8946732f8b3ae3c94e - @@ -331,10 +331,10 @@ par_results_compress 0 par_results_compress 1 par_results_csv bug #: --results csv par_results_csv --header : --tag --files --compress Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,H2,H1,Stdout,Stderr -par_results_csv --header : --tag --files --compress 1,:,999.999,999.999,0,15,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --header : --tag --files --compress 2,:,999.999,999.999,0,15,0,0,"echo 22 12",22,12,"22 12 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --header : --tag --files --compress 3,:,999.999,999.999,0,15,0,0,"echo 23 11",23,11,"23 11 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --header : --tag --files --compress 4,:,999.999,999.999,0,15,0,0,"echo 23 12",23,12,"23 12 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --header : --tag --files --compress 1,:,999.999,999.999,0,31,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --header : --tag --files --compress 2,:,999.999,999.999,0,31,0,0,"echo 22 12",22,12,"22 12 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --header : --tag --files --compress 3,:,999.999,999.999,0,31,0,0,"echo 23 11",23,11,"23 11 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --header : --tag --files --compress 4,:,999.999,999.999,0,31,0,0,"echo 23 12",23,12,"23 12 /tmp/parallel-local-10s-tmpdir/tmpfile", par_results_csv --header : --tag --files Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,H2,H1,Stdout,Stderr par_results_csv --header : --tag --files 1,:,999.999,999.999,0,6,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile", par_results_csv --header : --tag --files 2,:,999.999,999.999,0,6,0,0,"echo 22 12",22,12,"22 12 /tmp/parallel-local-10s-tmpdir/tmpfile", @@ -359,10 +359,10 @@ par_results_csv --header : --tag ", par_results_csv --header : --tag 4,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12 23 12 par_results_csv --header : --tag ", par_results_csv --header : --files --compress Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,H2,H1,Stdout,Stderr -par_results_csv --header : --files --compress 1,:,999.999,999.999,0,15,0,0,"echo 22 11",22,11,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --header : --files --compress 2,:,999.999,999.999,0,15,0,0,"echo 22 12",22,12,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --header : --files --compress 3,:,999.999,999.999,0,15,0,0,"echo 23 11",23,11,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --header : --files --compress 4,:,999.999,999.999,0,15,0,0,"echo 23 12",23,12,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --header : --files --compress 1,:,999.999,999.999,0,31,0,0,"echo 22 11",22,11,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --header : --files --compress 2,:,999.999,999.999,0,31,0,0,"echo 22 12",22,12,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --header : --files --compress 3,:,999.999,999.999,0,31,0,0,"echo 23 11",23,11,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --header : --files --compress 4,:,999.999,999.999,0,31,0,0,"echo 23 12",23,12,/tmp/parallel-local-10s-tmpdir/tmpfile, par_results_csv --header : --files Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,H2,H1,Stdout,Stderr par_results_csv --header : --files 1,:,999.999,999.999,0,6,0,0,"echo 22 11",22,11,/tmp/parallel-local-10s-tmpdir/tmpfile, par_results_csv --header : --files 2,:,999.999,999.999,0,6,0,0,"echo 22 12",22,12,/tmp/parallel-local-10s-tmpdir/tmpfile, @@ -387,15 +387,15 @@ par_results_csv --header : ", par_results_csv --header : 4,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12 par_results_csv --header : ", par_results_csv --tag --files --compress Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,V1,V2,Stdout,Stderr -par_results_csv --tag --files --compress 1,:,999.999,999.999,0,15,0,0,"echo H2 H1",H2,H1,"H2 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 2,:,999.999,999.999,0,15,0,0,"echo H2 11",H2,11,"H2 11 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 3,:,999.999,999.999,0,15,0,0,"echo H2 12",H2,12,"H2 12 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 4,:,999.999,999.999,0,15,0,0,"echo 22 H1",22,H1,"22 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 5,:,999.999,999.999,0,15,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 6,:,999.999,999.999,0,15,0,0,"echo 22 12",22,12,"22 12 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 7,:,999.999,999.999,0,15,0,0,"echo 23 H1",23,H1,"23 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 8,:,999.999,999.999,0,15,0,0,"echo 23 11",23,11,"23 11 /tmp/parallel-local-10s-tmpdir/tmpfile", -par_results_csv --tag --files --compress 9,:,999.999,999.999,0,15,0,0,"echo 23 12",23,12,"23 12 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 1,:,999.999,999.999,0,31,0,0,"echo H2 H1",H2,H1,"H2 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 2,:,999.999,999.999,0,31,0,0,"echo H2 11",H2,11,"H2 11 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 3,:,999.999,999.999,0,31,0,0,"echo H2 12",H2,12,"H2 12 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 4,:,999.999,999.999,0,31,0,0,"echo 22 H1",22,H1,"22 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 5,:,999.999,999.999,0,31,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 6,:,999.999,999.999,0,31,0,0,"echo 22 12",22,12,"22 12 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 7,:,999.999,999.999,0,31,0,0,"echo 23 H1",23,H1,"23 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 8,:,999.999,999.999,0,31,0,0,"echo 23 11",23,11,"23 11 /tmp/parallel-local-10s-tmpdir/tmpfile", +par_results_csv --tag --files --compress 9,:,999.999,999.999,0,31,0,0,"echo 23 12",23,12,"23 12 /tmp/parallel-local-10s-tmpdir/tmpfile", par_results_csv --tag --files Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,V1,V2,Stdout,Stderr par_results_csv --tag --files 1,:,999.999,999.999,0,6,0,0,"echo H2 H1",H2,H1,"H2 H1 /tmp/parallel-local-10s-tmpdir/tmpfile", par_results_csv --tag --files 2,:,999.999,999.999,0,6,0,0,"echo H2 11",H2,11,"H2 11 /tmp/parallel-local-10s-tmpdir/tmpfile", @@ -445,15 +445,15 @@ par_results_csv --tag ", par_results_csv --tag 9,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12 23 12 par_results_csv --tag ", par_results_csv --files --compress Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,V1,V2,Stdout,Stderr -par_results_csv --files --compress 1,:,999.999,999.999,0,15,0,0,"echo H2 H1",H2,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 2,:,999.999,999.999,0,15,0,0,"echo H2 11",H2,11,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 3,:,999.999,999.999,0,15,0,0,"echo H2 12",H2,12,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 4,:,999.999,999.999,0,15,0,0,"echo 22 H1",22,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 5,:,999.999,999.999,0,15,0,0,"echo 22 11",22,11,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 6,:,999.999,999.999,0,15,0,0,"echo 22 12",22,12,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 7,:,999.999,999.999,0,15,0,0,"echo 23 H1",23,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 8,:,999.999,999.999,0,15,0,0,"echo 23 11",23,11,/tmp/parallel-local-10s-tmpdir/tmpfile, -par_results_csv --files --compress 9,:,999.999,999.999,0,15,0,0,"echo 23 12",23,12,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 1,:,999.999,999.999,0,31,0,0,"echo H2 H1",H2,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 2,:,999.999,999.999,0,31,0,0,"echo H2 11",H2,11,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 3,:,999.999,999.999,0,31,0,0,"echo H2 12",H2,12,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 4,:,999.999,999.999,0,31,0,0,"echo 22 H1",22,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 5,:,999.999,999.999,0,31,0,0,"echo 22 11",22,11,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 6,:,999.999,999.999,0,31,0,0,"echo 22 12",22,12,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 7,:,999.999,999.999,0,31,0,0,"echo 23 H1",23,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 8,:,999.999,999.999,0,31,0,0,"echo 23 11",23,11,/tmp/parallel-local-10s-tmpdir/tmpfile, +par_results_csv --files --compress 9,:,999.999,999.999,0,31,0,0,"echo 23 12",23,12,/tmp/parallel-local-10s-tmpdir/tmpfile, par_results_csv --files Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,V1,V2,Stdout,Stderr par_results_csv --files 1,:,999.999,999.999,0,6,0,0,"echo H2 H1",H2,H1,/tmp/parallel-local-10s-tmpdir/tmpfile, par_results_csv --files 2,:,999.999,999.999,0,6,0,0,"echo H2 11",H2,11,/tmp/parallel-local-10s-tmpdir/tmpfile, diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 28e5c749..9b7f3a36 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -1,185 +1,28 @@ -echo "### BUG: The length for -X is not close to max (131072)"; seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc -### BUG: The length for -X is not close to max (131072) - 1 12817 131016 - seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc - 1 10947 131046 - seq 1 60000 | parallel -X echo |head -n 1 |wc - 1 23693 131052 - seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc - 1 15808 131047 - seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc - 1 11789 131045 - seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc - 1 25543 131043 -echo '### Test --fifo under csh' -### Test --fifo under csh - csh -c "seq 3000000 | parallel -k --pipe --fifo 'sleep .{#};cat {}|wc -c ; false; echo \$status; false'"; echo exit $? -1048571 -1 -1048572 -1 -1048572 -1 -1048572 -1 -1048572 -1 -1048572 -1 -1048569 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -1048576 -1 -868832 -1 -exit 22 -echo '**' -** -echo '### bug #44546: If --compress-program fails: fail' -### bug #44546: If --compress-program fails: fail - parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? -1 -parallel: Error: false failed. -parallel: Error: false failed. - parallel --tag --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? -1 -parallel: Error: false failed. -parallel: Error: false failed. - (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1 -1 -parallel: Error: false failed. -parallel: Error: false failed. - parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $? -1 -parallel: Error: false failed. -parallel: Error: false failed. - parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? -1 -parallel: Error: false failed. -parallel: Error: false failed. - parallel --compress-program false echo \;ls ::: /no-existing; echo $? -1 -parallel: Error: false failed. -parallel: Error: false failed. -echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' -bug #44250: pxz complains File format not recognized but decompresses anyway - # The first line dumps core if run from make file. Why?! - stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file -ls: cannot access '/OK-if-missing-file': No such file or directory - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file -can not seek in input: Illegal seek -ls: cannot access '/OK-if-missing-file': No such file or directory - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output - stdout parallel --compress --compress-program pxz true ::: OK-if-no-output -echo '**' -** -echo "### Test -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' -### Test -I -1 1 -2 1 -2 2 -3 1 -3 2 -3 3 -4 1 -4 2 -4 3 -4 4 -5 1 -5 2 -5 3 -5 4 -5 5 -6 1 -6 2 -6 3 -6 4 -6 5 -6 6 -7 1 -7 2 -7 3 -7 4 -7 5 -7 6 -7 7 -8 1 -8 2 -8 3 -8 4 -8 5 -8 6 -8 7 -8 8 -9 1 -9 2 -9 3 -9 4 -9 5 -9 6 -9 7 -9 8 -9 9 -10 1 -10 2 -10 3 -10 4 -10 5 -10 6 -10 7 -10 8 -10 9 -10 10 -echo "### Test -X -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::' -### Test -X -I -a1 b1 -a2 b1 b2 -a3 b1 b2 b3 -a4 b1 b2 b3 b4 -a5 b1 b2 b3 b4 b5 -a6 b1 b2 b3 b4 b5 b6 -a7 b1 b2 b3 b4 b5 b6 b7 -a8 b1 b2 b3 b4 b5 b6 b7 b8 -a9 b1 b2 b3 b4 b5 b6 b7 b8 b9 -a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 -echo "### Test -m -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::' -### Test -m -I -a1 b1 -a2 b1 2 -a3 b1 2 3 -a4 b1 2 3 4 -a5 b1 2 3 4 5 -a6 b1 2 3 4 5 6 -a7 b1 2 3 4 5 6 7 -a8 b1 2 3 4 5 6 7 8 -a9 b1 2 3 4 5 6 7 8 9 -a10 b1 2 3 4 5 6 7 8 9 10 +par_compress_prg_fails ### bug #44546: If --compress-program fails: fail +par_compress_prg_fails 1 +par_compress_prg_fails parallel: Error: false failed. +par_compress_prg_fails parallel: Error: false failed. +par_compress_prg_fails --files 1 +par_compress_prg_fails --files parallel: Error: false failed. +par_compress_prg_fails --files parallel: Error: false failed. +par_compress_prg_fails --tag 1 +par_compress_prg_fails --tag parallel: Error: false failed. +par_compress_prg_fails --tag parallel: Error: false failed. +par_compress_prg_fails --tag --files 1 +par_compress_prg_fails --tag --files parallel: Error: false failed. +par_compress_prg_fails --tag --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer 1 +par_compress_prg_fails --line-buffer parallel: Error: false failed. +par_compress_prg_fails --line-buffer parallel: Error: false failed. +par_compress_prg_fails --line-buffer --files 1 +par_compress_prg_fails --line-buffer --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag 1 +par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag --files 1 +par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. par_failing_compressor Compress with failing (de)compressor par_failing_compressor Test --tag/--line-buffer/--files in all combinations par_failing_compressor Test working/failing compressor/decompressor in all combinations @@ -264,6 +107,52 @@ par_failing_compressor parallel: Error: cat;false failed. par_failing_compressor parallel -k -k -k -k --compress --compress-program cat\;false --decompress-program cat\;false echo ::: C=cat\;false,D=cat\;false par_failing_compressor C=cat;false,D=cat;false par_failing_compressor parallel: Error: cat;false failed. +par_fifo_under_csh ### Test --fifo under csh +par_fifo_under_csh 1048571 +par_fifo_under_csh 1 +par_fifo_under_csh 1048572 +par_fifo_under_csh 1 +par_fifo_under_csh 1048572 +par_fifo_under_csh 1 +par_fifo_under_csh 1048572 +par_fifo_under_csh 1 +par_fifo_under_csh 1048572 +par_fifo_under_csh 1 +par_fifo_under_csh 1048572 +par_fifo_under_csh 1 +par_fifo_under_csh 1048569 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 1048576 +par_fifo_under_csh 1 +par_fifo_under_csh 868832 +par_fifo_under_csh 1 +par_fifo_under_csh exit 22 par_incomplete_linebuffer bug #51337: --lb does not kill jobs at sigpipe par_incomplete_linebuffer 1 par_incomplete_linebuffer 2 @@ -372,6 +261,13 @@ par_linebuffer_files lrz --files par_linebuffer_files lrz 1 par_linebuffer_files lrz --results par_linebuffer_files lrz 1 +par_max_length_len_128k ### BUG: The length for -X is not close to max (131072) +par_max_length_len_128k 1 12817 131016 +par_max_length_len_128k 1 10946 131032 +par_max_length_len_128k 1 23691 131040 +par_max_length_len_128k 1 15806 131030 +par_max_length_len_128k 1 11788 131032 +par_max_length_len_128k 1 25543 131043 par_no_newline_compress bug #41613: --compress --line-buffer - no newline par_no_newline_compress tagstring=--tagstring {#} compress=--compress par_no_newline_compress 1 OK @@ -426,6 +322,10 @@ par_parset 9 par_parset Commands with newline require -0 par_parset line1 par_parset line2 +par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway +par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory +par_pxz_complains can not seek in input: Illegal seek +par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory par_result ### Test --results par_result I III par_result I IIII @@ -682,3 +582,81 @@ par_result_replace /tmp/par__49983-baz C par_result_replace /tmp/par__49983-baz C/seq par_result_replace /tmp/par__49983-baz C/stderr par_result_replace /tmp/par__49983-baz C/stdout +par_test_XI_mI ### Test -I +par_test_XI_mI 1 1 +par_test_XI_mI 2 1 +par_test_XI_mI 2 2 +par_test_XI_mI 3 1 +par_test_XI_mI 3 2 +par_test_XI_mI 3 3 +par_test_XI_mI 4 1 +par_test_XI_mI 4 2 +par_test_XI_mI 4 3 +par_test_XI_mI 4 4 +par_test_XI_mI 5 1 +par_test_XI_mI 5 2 +par_test_XI_mI 5 3 +par_test_XI_mI 5 4 +par_test_XI_mI 5 5 +par_test_XI_mI 6 1 +par_test_XI_mI 6 2 +par_test_XI_mI 6 3 +par_test_XI_mI 6 4 +par_test_XI_mI 6 5 +par_test_XI_mI 6 6 +par_test_XI_mI 7 1 +par_test_XI_mI 7 2 +par_test_XI_mI 7 3 +par_test_XI_mI 7 4 +par_test_XI_mI 7 5 +par_test_XI_mI 7 6 +par_test_XI_mI 7 7 +par_test_XI_mI 8 1 +par_test_XI_mI 8 2 +par_test_XI_mI 8 3 +par_test_XI_mI 8 4 +par_test_XI_mI 8 5 +par_test_XI_mI 8 6 +par_test_XI_mI 8 7 +par_test_XI_mI 8 8 +par_test_XI_mI 9 1 +par_test_XI_mI 9 2 +par_test_XI_mI 9 3 +par_test_XI_mI 9 4 +par_test_XI_mI 9 5 +par_test_XI_mI 9 6 +par_test_XI_mI 9 7 +par_test_XI_mI 9 8 +par_test_XI_mI 9 9 +par_test_XI_mI 10 1 +par_test_XI_mI 10 2 +par_test_XI_mI 10 3 +par_test_XI_mI 10 4 +par_test_XI_mI 10 5 +par_test_XI_mI 10 6 +par_test_XI_mI 10 7 +par_test_XI_mI 10 8 +par_test_XI_mI 10 9 +par_test_XI_mI 10 10 +par_test_XI_mI ### Test -X -I +par_test_XI_mI a1 b1 +par_test_XI_mI a2 b1 b2 +par_test_XI_mI a3 b1 b2 b3 +par_test_XI_mI a4 b1 b2 b3 b4 +par_test_XI_mI a5 b1 b2 b3 b4 b5 +par_test_XI_mI a6 b1 b2 b3 b4 b5 b6 +par_test_XI_mI a7 b1 b2 b3 b4 b5 b6 b7 +par_test_XI_mI a8 b1 b2 b3 b4 b5 b6 b7 b8 +par_test_XI_mI a9 b1 b2 b3 b4 b5 b6 b7 b8 b9 +par_test_XI_mI a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 +par_test_XI_mI ### Test -m -I +par_test_XI_mI a1 b1 +par_test_XI_mI a2 b1 2 +par_test_XI_mI a3 b1 2 3 +par_test_XI_mI a4 b1 2 3 4 +par_test_XI_mI a5 b1 2 3 4 5 +par_test_XI_mI a6 b1 2 3 4 5 6 +par_test_XI_mI a7 b1 2 3 4 5 6 7 +par_test_XI_mI a8 b1 2 3 4 5 6 7 8 +par_test_XI_mI a9 b1 2 3 4 5 6 7 8 9 +par_test_XI_mI a10 b1 2 3 4 5 6 7 8 9 10 diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index 20db9332..8f018c31 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -1,4 +1,4 @@ -par_linebuffer_matters_compress ### (--linebuffer) --compress should give different output +par_linebuffer_matters_compress ### (--linebuffer) --compress should give different output par_linebuffer_matters_compress OK: --linebuffer makes a difference par_linebuffer_matters_compress_tag ### (--linebuffer) --compress --tag should give different output par_linebuffer_matters_compress_tag OK: --linebuffer makes a difference diff --git a/testsuite/wanted-results/parallel-local-race02 b/testsuite/wanted-results/parallel-local-race02 index 1b83a881..18df8787 100644 --- a/testsuite/wanted-results/parallel-local-race02 +++ b/testsuite/wanted-results/parallel-local-race02 @@ -2,14 +2,14 @@ bug #46120: Suspend should suspend (at least local) children it should burn 1.9 CPU seconds, but no more than that The 5 second sleep will make it be killed by timeout when it fgs -stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 +stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | \grep -q CPUTIME=1 Zero=OK 0 -echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | grep -q CPUTIME=1 +echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | \grep -q CPUTIME=1 Zero=OK 0 Control case: Burn for 2.9 seconds -stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 +stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | \grep -q CPUTIME=1 1=OK 1 par_hostgroup ### --hostgroup force ncpu par_hostgroup parallel diff --git a/testsuite/wanted-results/parallel-local-sql b/testsuite/wanted-results/parallel-local-sql index 4ded5f6a..4fb32b32 100644 --- a/testsuite/wanted-results/parallel-local-sql +++ b/testsuite/wanted-results/parallel-local-sql @@ -1,4 +1,4 @@ -p_wrapper par_append_different_cmd \$MYSQL +par_append_different_cmd $MYSQL p_wrapper par_append_different_cmd \$MYSQL par_append_different_cmd $MYSQL Exit=0 par_append_different_cmd $MYSQL Exit=0 par_append_different_cmd $MYSQL Command V1 V2 Stdout Stderr @@ -52,7 +52,7 @@ par_append_different_cmd $MYSQL sleep .3;echo B-15 15 B B-15\n par_append_different_cmd $MYSQL sleep .3;echo C-15 15 C C-15\n par_append_different_cmd $MYSQL sleep .3;echo D-15 15 D D-15\n par_append_different_cmd $MYSQL sleep .3;echo E-15 15 E E-15\n -p_wrapper par_append_different_cmd \$PG +par_append_different_cmd $PG p_wrapper par_append_different_cmd \$PG par_append_different_cmd $PG Exit=0 par_append_different_cmd $PG Exit=0 par_append_different_cmd $PG command | v1 | v2 | stdout | stderr @@ -159,7 +159,7 @@ par_append_different_cmd $PG sleep .3;echo E-15 | 15 | E | E-15 +| par_append_different_cmd $PG | | | | par_append_different_cmd $PG (50 rows) par_append_different_cmd $PG -p_wrapper par_append_different_cmd \$SQLITE +par_append_different_cmd $SQLITE p_wrapper par_append_different_cmd \$SQLITE par_append_different_cmd $SQLITE Exit=0 par_append_different_cmd $SQLITE Exit=0 par_append_different_cmd $SQLITE Command|V1|V2|Stdout|Stderr @@ -263,7 +263,7 @@ par_append_different_cmd $SQLITE sleep .3;echo D-15|15|D|D-15 par_append_different_cmd $SQLITE | par_append_different_cmd $SQLITE sleep .3;echo E-15|15|E|E-15 par_append_different_cmd $SQLITE | -p_wrapper par_no_table \$MYSQL +par_no_table $MYSQL p_wrapper par_no_table \$MYSQL par_no_table $MYSQL bug #50018: --dburl without table dies par_no_table $MYSQL 255 par_no_table $MYSQL 255 @@ -275,7 +275,7 @@ par_no_table $MYSQL true dummy dummy dummy dummy par_no_table $MYSQL parallel: Error: The DBURL (mysql://tange:tange@lo/tange) must contain a table. par_no_table $MYSQL parallel: Error: The DBURL (mysql://tange:tange@lo/tange) must contain a table. par_no_table $MYSQL parallel: Error: The DBURL (mysql://tange:tange@lo/tange) must contain a table. -p_wrapper par_no_table \$PG +par_no_table $PG p_wrapper par_no_table \$PG par_no_table $PG bug #50018: --dburl without table dies par_no_table $PG 255 par_no_table $PG 255 @@ -290,7 +290,7 @@ par_no_table $PG par_no_table $PG parallel: Error: The DBURL (pg://tange:tange@lo/tange) must contain a table. par_no_table $PG parallel: Error: The DBURL (pg://tange:tange@lo/tange) must contain a table. par_no_table $PG parallel: Error: The DBURL (pg://tange:tange@lo/tange) must contain a table. -p_wrapper par_no_table \$SQLITE +par_no_table $SQLITE p_wrapper par_no_table \$SQLITE par_no_table $SQLITE bug #50018: --dburl without table dies par_no_table $SQLITE 255 par_no_table $SQLITE 255 @@ -302,7 +302,7 @@ par_no_table $SQLITE true dummy dummy|dummy|dummy|| par_no_table $SQLITE parallel: Error: The DBURL (sqlite3:///%2Frun%2Fshm%2Fparallel.db) must contain a table. par_no_table $SQLITE parallel: Error: The DBURL (sqlite3:///%2Frun%2Fshm%2Fparallel.db) must contain a table. par_no_table $SQLITE parallel: Error: The DBURL (sqlite3:///%2Frun%2Fshm%2Fparallel.db) must contain a table. -p_wrapper par_shuf \$MYSQL +par_shuf $MYSQL p_wrapper par_shuf \$MYSQL par_shuf $MYSQL 1 a par_shuf $MYSQL 1 b par_shuf $MYSQL 1 c @@ -357,7 +357,7 @@ par_shuf $MYSQL sleep .3;echo 5 b 5 b /tmp/parallel-bug49791-c20/1/5/2/b/stdout par_shuf $MYSQL sleep .3;echo 5 c 5 c /tmp/parallel-bug49791-c20/1/5/2/c/stdout /tmp/parallel-bug49791-c20/1/5/2/c/stderr par_shuf $MYSQL sleep .3;echo 5 d 5 d /tmp/parallel-bug49791-c20/1/5/2/d/stdout /tmp/parallel-bug49791-c20/1/5/2/d/stderr par_shuf $MYSQL sleep .3;echo 5 e 5 e /tmp/parallel-bug49791-c20/1/5/2/e/stdout /tmp/parallel-bug49791-c20/1/5/2/e/stderr -p_wrapper par_shuf \$PG +par_shuf $PG p_wrapper par_shuf \$PG par_shuf $PG 1 a par_shuf $PG 1 b par_shuf $PG 1 c @@ -415,7 +415,7 @@ par_shuf $PG sleep .3;echo 5 d | 5 | d | /tmp/parallel-bug49791-792/1/5/2/d/s par_shuf $PG sleep .3;echo 5 e | 5 | e | /tmp/parallel-bug49791-792/1/5/2/e/stdout | /tmp/parallel-bug49791-792/1/5/2/e/stderr par_shuf $PG (25 rows) par_shuf $PG -p_wrapper par_shuf \$SQLITE +par_shuf $SQLITE p_wrapper par_shuf \$SQLITE par_shuf $SQLITE 1 a par_shuf $SQLITE 1 b par_shuf $SQLITE 1 c @@ -470,7 +470,7 @@ par_shuf $SQLITE sleep .3;echo 5 b|5|b|/tmp/parallel-bug49791-932/1/5/2/b/stdout par_shuf $SQLITE sleep .3;echo 5 c|5|c|/tmp/parallel-bug49791-932/1/5/2/c/stdout|/tmp/parallel-bug49791-932/1/5/2/c/stderr par_shuf $SQLITE sleep .3;echo 5 d|5|d|/tmp/parallel-bug49791-932/1/5/2/d/stdout|/tmp/parallel-bug49791-932/1/5/2/d/stderr par_shuf $SQLITE sleep .3;echo 5 e|5|e|/tmp/parallel-bug49791-932/1/5/2/e/stdout|/tmp/parallel-bug49791-932/1/5/2/e/stderr -p_wrapper par_sql_joblog \$MYSQL +par_sql_joblog $MYSQL p_wrapper par_sql_joblog \$MYSQL par_sql_joblog $MYSQL ### should only give a single --joblog heading par_sql_joblog $MYSQL ### --sqlmaster/--sqlworker par_sql_joblog $MYSQL Seq Host Starttime JobRuntime Send Receive Exitval Signal Command @@ -604,7 +604,7 @@ par_sql_joblog $MYSQL sleep .3;echo 5 b 5 b 5 b\n par_sql_joblog $MYSQL sleep .3;echo 5 c 5 c 5 c\n par_sql_joblog $MYSQL sleep .3;echo 5 d 5 d 5 d\n par_sql_joblog $MYSQL sleep .3;echo 5 e 5 e 5 e\n -p_wrapper par_sql_joblog \$PG +par_sql_joblog $PG p_wrapper par_sql_joblog \$PG par_sql_joblog $PG ### should only give a single --joblog heading par_sql_joblog $PG ### --sqlmaster/--sqlworker par_sql_joblog $PG Seq Host Starttime JobRuntime Send Receive Exitval Signal Command @@ -766,7 +766,7 @@ par_sql_joblog $PG sleep .3;echo 5 e | 5 | e | 5 e +| par_sql_joblog $PG | | | | par_sql_joblog $PG (25 rows) par_sql_joblog $PG -p_wrapper par_sql_joblog \$SQLITE +par_sql_joblog $SQLITE p_wrapper par_sql_joblog \$SQLITE par_sql_joblog $SQLITE ### should only give a single --joblog heading par_sql_joblog $SQLITE ### --sqlmaster/--sqlworker par_sql_joblog $SQLITE Seq Host Starttime JobRuntime Send Receive Exitval Signal Command @@ -925,7 +925,7 @@ par_sql_joblog $SQLITE sleep .3;echo 5 d|5|d|5 d par_sql_joblog $SQLITE | par_sql_joblog $SQLITE sleep .3;echo 5 e|5|e|5 e par_sql_joblog $SQLITE | -p_wrapper par_sqlandworker \$MYSQL +par_sqlandworker $MYSQL p_wrapper par_sqlandworker \$MYSQL par_sqlandworker $MYSQL Exit=0 par_sqlandworker $MYSQL Exit=0 par_sqlandworker $MYSQL Command V1 V2 Stdout Stderr @@ -954,7 +954,7 @@ par_sqlandworker $MYSQL sleep .3;echo 5 b 5 b 5 b\n par_sqlandworker $MYSQL sleep .3;echo 5 c 5 c 5 c\n par_sqlandworker $MYSQL sleep .3;echo 5 d 5 d 5 d\n par_sqlandworker $MYSQL sleep .3;echo 5 e 5 e 5 e\n -p_wrapper par_sqlandworker \$PG +par_sqlandworker $PG p_wrapper par_sqlandworker \$PG par_sqlandworker $PG Exit=0 par_sqlandworker $PG Exit=0 par_sqlandworker $PG command | v1 | v2 | stdout | stderr @@ -1011,7 +1011,7 @@ par_sqlandworker $PG sleep .3;echo 5 e | 5 | e | 5 e +| par_sqlandworker $PG | | | | par_sqlandworker $PG (25 rows) par_sqlandworker $PG -p_wrapper par_sqlandworker \$SQLITE +par_sqlandworker $SQLITE p_wrapper par_sqlandworker \$SQLITE par_sqlandworker $SQLITE Exit=0 par_sqlandworker $SQLITE Exit=0 par_sqlandworker $SQLITE Command|V1|V2|Stdout|Stderr @@ -1065,7 +1065,7 @@ par_sqlandworker $SQLITE sleep .3;echo 5 d|5|d|5 d par_sqlandworker $SQLITE | par_sqlandworker $SQLITE sleep .3;echo 5 e|5|e|5 e par_sqlandworker $SQLITE | -p_wrapper par_sqlandworker_lo \$MYSQL +par_sqlandworker_lo $MYSQL p_wrapper par_sqlandworker_lo \$MYSQL par_sqlandworker_lo $MYSQL Exit=0 par_sqlandworker_lo $MYSQL Exit=0 par_sqlandworker_lo $MYSQL Command V1 V2 Stdout Stderr @@ -1094,7 +1094,7 @@ par_sqlandworker_lo $MYSQL sleep .3;echo 5 b 5 b 5 b\n par_sqlandworker_lo $MYSQL sleep .3;echo 5 c 5 c 5 c\n par_sqlandworker_lo $MYSQL sleep .3;echo 5 d 5 d 5 d\n par_sqlandworker_lo $MYSQL sleep .3;echo 5 e 5 e 5 e\n -p_wrapper par_sqlandworker_lo \$PG +par_sqlandworker_lo $PG p_wrapper par_sqlandworker_lo \$PG par_sqlandworker_lo $PG Exit=0 par_sqlandworker_lo $PG Exit=0 par_sqlandworker_lo $PG command | v1 | v2 | stdout | stderr @@ -1151,7 +1151,7 @@ par_sqlandworker_lo $PG sleep .3;echo 5 e | 5 | e | 5 e +| par_sqlandworker_lo $PG | | | | par_sqlandworker_lo $PG (25 rows) par_sqlandworker_lo $PG -p_wrapper par_sqlandworker_lo \$SQLITE +par_sqlandworker_lo $SQLITE p_wrapper par_sqlandworker_lo \$SQLITE par_sqlandworker_lo $SQLITE Exit=0 par_sqlandworker_lo $SQLITE Exit=0 par_sqlandworker_lo $SQLITE Command|V1|V2|Stdout|Stderr @@ -1205,7 +1205,7 @@ par_sqlandworker_lo $SQLITE sleep .3;echo 5 d|5|d|5 d par_sqlandworker_lo $SQLITE | par_sqlandworker_lo $SQLITE sleep .3;echo 5 e|5|e|5 e par_sqlandworker_lo $SQLITE | -p_wrapper par_sqlandworker_results \$MYSQL +par_sqlandworker_results $MYSQL p_wrapper par_sqlandworker_results \$MYSQL par_sqlandworker_results $MYSQL Exit=0 par_sqlandworker_results $MYSQL Exit=0 par_sqlandworker_results $MYSQL Command V1 V2 Stdout Stderr @@ -1234,7 +1234,7 @@ par_sqlandworker_results $MYSQL sleep .3;echo 5 b 5 b /tmp/out--sql/1/5/2/b/stdo par_sqlandworker_results $MYSQL sleep .3;echo 5 c 5 c /tmp/out--sql/1/5/2/c/stdout /tmp/out--sql/1/5/2/c/stderr par_sqlandworker_results $MYSQL sleep .3;echo 5 d 5 d /tmp/out--sql/1/5/2/d/stdout /tmp/out--sql/1/5/2/d/stderr par_sqlandworker_results $MYSQL sleep .3;echo 5 e 5 e /tmp/out--sql/1/5/2/e/stdout /tmp/out--sql/1/5/2/e/stderr -p_wrapper par_sqlandworker_results \$PG +par_sqlandworker_results $PG p_wrapper par_sqlandworker_results \$PG par_sqlandworker_results $PG Exit=0 par_sqlandworker_results $PG Exit=0 par_sqlandworker_results $PG command | v1 | v2 | stdout | stderr @@ -1266,7 +1266,7 @@ par_sqlandworker_results $PG sleep .3;echo 5 d | 5 | d | /tmp/out--sql/1/5/2/ par_sqlandworker_results $PG sleep .3;echo 5 e | 5 | e | /tmp/out--sql/1/5/2/e/stdout | /tmp/out--sql/1/5/2/e/stderr par_sqlandworker_results $PG (25 rows) par_sqlandworker_results $PG -p_wrapper par_sqlandworker_results \$SQLITE +par_sqlandworker_results $SQLITE p_wrapper par_sqlandworker_results \$SQLITE par_sqlandworker_results $SQLITE Exit=0 par_sqlandworker_results $SQLITE Exit=0 par_sqlandworker_results $SQLITE Command|V1|V2|Stdout|Stderr @@ -1295,7 +1295,7 @@ par_sqlandworker_results $SQLITE sleep .3;echo 5 b|5|b|/tmp/out--sql/1/5/2/b/std par_sqlandworker_results $SQLITE sleep .3;echo 5 c|5|c|/tmp/out--sql/1/5/2/c/stdout|/tmp/out--sql/1/5/2/c/stderr par_sqlandworker_results $SQLITE sleep .3;echo 5 d|5|d|/tmp/out--sql/1/5/2/d/stdout|/tmp/out--sql/1/5/2/d/stderr par_sqlandworker_results $SQLITE sleep .3;echo 5 e|5|e|/tmp/out--sql/1/5/2/e/stdout|/tmp/out--sql/1/5/2/e/stderr -p_wrapper par_sqlandworker_tag \$MYSQL +par_sqlandworker_tag $MYSQL p_wrapper par_sqlandworker_tag \$MYSQL par_sqlandworker_tag $MYSQL Exit=0 par_sqlandworker_tag $MYSQL Exit=0 par_sqlandworker_tag $MYSQL Command V1 V2 Stdout Stderr @@ -1324,7 +1324,7 @@ par_sqlandworker_tag $MYSQL sleep .3;echo 5 b 5 b 5 b\t5 b\n par_sqlandworker_tag $MYSQL sleep .3;echo 5 c 5 c 5 c\t5 c\n par_sqlandworker_tag $MYSQL sleep .3;echo 5 d 5 d 5 d\t5 d\n par_sqlandworker_tag $MYSQL sleep .3;echo 5 e 5 e 5 e\t5 e\n -p_wrapper par_sqlandworker_tag \$PG +par_sqlandworker_tag $PG p_wrapper par_sqlandworker_tag \$PG par_sqlandworker_tag $PG Exit=0 par_sqlandworker_tag $PG Exit=0 par_sqlandworker_tag $PG command | v1 | v2 | stdout | stderr @@ -1381,7 +1381,7 @@ par_sqlandworker_tag $PG sleep .3;echo 5 e | 5 | e | 5 e 5 e+| par_sqlandworker_tag $PG | | | | par_sqlandworker_tag $PG (25 rows) par_sqlandworker_tag $PG -p_wrapper par_sqlandworker_tag \$SQLITE +par_sqlandworker_tag $SQLITE p_wrapper par_sqlandworker_tag \$SQLITE par_sqlandworker_tag $SQLITE Exit=0 par_sqlandworker_tag $SQLITE Exit=0 par_sqlandworker_tag $SQLITE Command|V1|V2|Stdout|Stderr @@ -1435,7 +1435,7 @@ par_sqlandworker_tag $SQLITE sleep .3;echo 5 d|5|d|5 d 5 d par_sqlandworker_tag $SQLITE | par_sqlandworker_tag $SQLITE sleep .3;echo 5 e|5|e|5 e 5 e par_sqlandworker_tag $SQLITE | -p_wrapper par_sqlandworker_total_jobs \$MYSQL +par_sqlandworker_total_jobs $MYSQL p_wrapper par_sqlandworker_total_jobs \$MYSQL par_sqlandworker_total_jobs $MYSQL Exit=0 par_sqlandworker_total_jobs $MYSQL Exit=0 par_sqlandworker_total_jobs $MYSQL Command V1 V2 Stdout Stderr @@ -1464,7 +1464,7 @@ par_sqlandworker_total_jobs $MYSQL echo 22 of 25; sleep .3;echo 5 b 22 of 25\n\n par_sqlandworker_total_jobs $MYSQL echo 23 of 25; sleep .3;echo 5 c 23 of 25\n\n par_sqlandworker_total_jobs $MYSQL echo 24 of 25; sleep .3;echo 5 d 24 of 25\n\n par_sqlandworker_total_jobs $MYSQL echo 25 of 25; sleep .3;echo 5 e 25 of 25\n\n -p_wrapper par_sqlandworker_total_jobs \$PG +par_sqlandworker_total_jobs $PG p_wrapper par_sqlandworker_total_jobs \$PG par_sqlandworker_total_jobs $PG Exit=0 par_sqlandworker_total_jobs $PG Exit=0 par_sqlandworker_total_jobs $PG command | v1 | v2 | stdout | stderr @@ -1546,7 +1546,7 @@ par_sqlandworker_total_jobs $PG | | | par_sqlandworker_total_jobs $PG | | | | par_sqlandworker_total_jobs $PG (25 rows) par_sqlandworker_total_jobs $PG -p_wrapper par_sqlandworker_total_jobs \$SQLITE +par_sqlandworker_total_jobs $SQLITE p_wrapper par_sqlandworker_total_jobs \$SQLITE par_sqlandworker_total_jobs $SQLITE Exit=0 par_sqlandworker_total_jobs $SQLITE Exit=0 par_sqlandworker_total_jobs $SQLITE Command|V1|V2|Stdout|Stderr @@ -1625,7 +1625,7 @@ par_sqlandworker_total_jobs $SQLITE | par_sqlandworker_total_jobs $SQLITE echo 25 of 25; sleep .3;echo|5|e|25 of 25 par_sqlandworker_total_jobs $SQLITE par_sqlandworker_total_jobs $SQLITE | -p_wrapper par_sqlandworker_unbuffer \$MYSQL +par_sqlandworker_unbuffer $MYSQL p_wrapper par_sqlandworker_unbuffer \$MYSQL par_sqlandworker_unbuffer $MYSQL Exit=0 par_sqlandworker_unbuffer $MYSQL Exit=0 par_sqlandworker_unbuffer $MYSQL Command V1 V2 Stdout Stderr @@ -1654,7 +1654,7 @@ par_sqlandworker_unbuffer $MYSQL sleep .3;echo 5 b 5 b par_sqlandworker_unbuffer $MYSQL sleep .3;echo 5 c 5 c par_sqlandworker_unbuffer $MYSQL sleep .3;echo 5 d 5 d par_sqlandworker_unbuffer $MYSQL sleep .3;echo 5 e 5 e -p_wrapper par_sqlandworker_unbuffer \$PG +par_sqlandworker_unbuffer $PG p_wrapper par_sqlandworker_unbuffer \$PG par_sqlandworker_unbuffer $PG Exit=0 par_sqlandworker_unbuffer $PG Exit=0 par_sqlandworker_unbuffer $PG command | v1 | v2 | stdout | stderr @@ -1686,7 +1686,7 @@ par_sqlandworker_unbuffer $PG sleep .3;echo 5 d | 5 | d | | par_sqlandworker_unbuffer $PG sleep .3;echo 5 e | 5 | e | | par_sqlandworker_unbuffer $PG (25 rows) par_sqlandworker_unbuffer $PG -p_wrapper par_sqlandworker_unbuffer \$SQLITE +par_sqlandworker_unbuffer $SQLITE p_wrapper par_sqlandworker_unbuffer \$SQLITE par_sqlandworker_unbuffer $SQLITE Exit=0 par_sqlandworker_unbuffer $SQLITE Exit=0 par_sqlandworker_unbuffer $SQLITE Command|V1|V2|Stdout|Stderr diff --git a/testsuite/wanted-results/parallel-local-sql01 b/testsuite/wanted-results/parallel-local-sql01 index a1f01e84..c1faa8eb 100644 --- a/testsuite/wanted-results/parallel-local-sql01 +++ b/testsuite/wanted-results/parallel-local-sql01 @@ -1,6 +1,6 @@ -p_wrapper par_empty \$MYSQL +par_empty $MYSQL p_wrapper par_empty \$MYSQL par_empty $MYSQL /bin/bash: p_wrapper: command not found -p_wrapper par_empty \$PG +par_empty $PG p_wrapper par_empty \$PG par_empty $PG /bin/bash: p_wrapper: command not found -p_wrapper par_empty \$SQLITE +par_empty $SQLITE p_wrapper par_empty \$SQLITE par_empty $SQLITE /bin/bash: p_wrapper: command not found diff --git a/testsuite/wanted-results/parallel-local-ssh5 b/testsuite/wanted-results/parallel-local-ssh5 index 3dd3e1f8..d03906c6 100644 --- a/testsuite/wanted-results/parallel-local-ssh5 +++ b/testsuite/wanted-results/parallel-local-ssh5 @@ -1,103 +1,89 @@ -echo '### --ssh autossh - add commands that fail here' -### --ssh autossh - add commands that fail here - export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0; parallel -S lo echo ::: OK; echo OK | parallel --pipe -S lo cat; parallel -S lo false ::: a || echo OK should fail; touch foo_autossh; stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh; rm foo_autossh*; -OK -OK -/usr/lib/autossh/autossh: invalid option -- '-' -usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS] - - -M specifies monitor port. Overrides the environment - variable AUTOSSH_PORT. 0 turns monitoring loop off. - Alternatively, a port for an echo service on the remote - machine may be specified. (Normally port 7.) - -f run in background (autossh handles this, and does not - pass it to ssh.) - -V print autossh version and exit. - -Environment variables are: - AUTOSSH_GATETIME - how long must an ssh session be established - before we decide it really was established - (in seconds). Default is 30 seconds; use of -f - flag sets this to 0. - AUTOSSH_LOGFILE - file to log to (default is to use the syslog - facility) - AUTOSSH_LOGLEVEL - level of log verbosity - AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds) - AUTOSSH_MAXSTART - max times to restart (default is no limit) - AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes) - AUTOSSH_PATH - path to ssh if not default - AUTOSSH_PIDFILE - write pid to this file - AUTOSSH_POLL - how often to check the connection (seconds) - AUTOSSH_FIRST_POLL - time before first connection check (seconds) - AUTOSSH_PORT - port to use for monitor connection - AUTOSSH_DEBUG - turn logging to maximum verbosity and log to - stderr - -rsync: connection unexpectedly closed (0 bytes received so far) [sender] -rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.1] -/usr/lib/autossh/autossh: invalid option -- '-' -usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS] - - -M specifies monitor port. Overrides the environment - variable AUTOSSH_PORT. 0 turns monitoring loop off. - Alternatively, a port for an echo service on the remote - machine may be specified. (Normally port 7.) - -f run in background (autossh handles this, and does not - pass it to ssh.) - -V print autossh version and exit. - -Environment variables are: - AUTOSSH_GATETIME - how long must an ssh session be established - before we decide it really was established - (in seconds). Default is 30 seconds; use of -f - flag sets this to 0. - AUTOSSH_LOGFILE - file to log to (default is to use the syslog - facility) - AUTOSSH_LOGLEVEL - level of log verbosity - AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds) - AUTOSSH_MAXSTART - max times to restart (default is no limit) - AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes) - AUTOSSH_PATH - path to ssh if not default - AUTOSSH_PIDFILE - write pid to this file - AUTOSSH_POLL - how often to check the connection (seconds) - AUTOSSH_FIRST_POLL - time before first connection check (seconds) - AUTOSSH_PORT - port to use for monitor connection - AUTOSSH_DEBUG - turn logging to maximum verbosity and log to - stderr - -rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] -rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.1] -echo '### bug #46520: --basefile cleans up without --cleanup' -### bug #46520: --basefile cleans up without --cleanup - touch bug_46520; parallel -S parallel@lo --bf bug_46520 ls ::: bug_46520; ssh parallel@lo ls bug_46520; parallel -S parallel@lo --cleanup --bf bug_46520 ls ::: bug_46520; stdout ssh parallel@lo ls bug_46520 # should not exist -bug_46520 -bug_46520 -bug_46520 -ls: cannot access 'bug_46520': No such file or directory -echo '### bug #36595: silent loss of input with --pipe and --sshlogin' -### bug #36595: silent loss of input with --pipe and --sshlogin - seq 10000 | xargs | parallel --pipe -S 8/localhost cat 2>/dev/null | wc - 1 10000 48894 -echo 'bug #36707: --controlmaster eats jobs' -bug #36707: --controlmaster eats jobs - seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{} -OK1 -OK2 -echo '### --ssh lsh' -### --ssh lsh - parallel --ssh 'lsh -c aes256-ctr' -S lo echo ::: OK -OK - echo OK | parallel --ssh 'lsh -c aes256-ctr' --pipe -S csh@lo cat -OK - # Todo rsync/trc csh@lo - # Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler -echo '### bug #45025: --pipe --retries does not reschedule on other host' -### bug #45025: --pipe --retries does not reschedule on other host - seq 1 300030| stdout parallel -k --retries 2 -S a.a,: --pipe 'wc;hostname' | perl -pe 's/'`hostname`'/localhost-:/' -parallel: Warning: Could not figure out number of cpus on a.a (). Using 1. - 165668 165668 1048571 -localhost-: - 134362 134362 940534 -localhost-: - stdout parallel --retries 2 --roundrobin echo ::: should fail -parallel: Error: --retries cannot be combined with --roundrobin. +par_pipe_retries ### bug #45025: --pipe --retries does not reschedule on other host +par_pipe_retries parallel: Warning: Could not figure out number of cpus on a.a (). Using 1. +par_pipe_retries 165668 165668 1048571 +par_pipe_retries localhost-: +par_pipe_retries 134362 134362 940534 +par_pipe_retries localhost-: +par_pipe_retries parallel: Error: --retries cannot be combined with --roundrobin. +par_lsh ### --ssh lsh +par_lsh parallel: Warning: Could not figure out number of cpus on lo (). Using 1. +par_lsh OK +par_lsh parallel: Warning: Could not figure out number of cpus on lo (). Using 1. +par_lsh OK +par_input_loss_pipe ### bug #36595: silent loss of input with --pipe and --sshlogin +par_input_loss_pipe 1 10000 48894 +par_controlmaster_eats bug #36707: --controlmaster eats jobs +par_controlmaster_eats OK1 +par_controlmaster_eats OK2 +par_basefile_cleanup ### bug #46520: --basefile cleans up without --cleanup +par_basefile_cleanup bug_46520 +par_basefile_cleanup bug_46520 +par_basefile_cleanup bug_46520 +par_basefile_cleanup ls: cannot access 'bug_46520': No such file or directory +par_autossh ### --ssh autossh - add commands that fail here +par_autossh OK +par_autossh OK +par_autossh /usr/lib/autossh/autossh: invalid option -- '-' +par_autossh usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS] +par_autossh +par_autossh -M specifies monitor port. Overrides the environment +par_autossh variable AUTOSSH_PORT. 0 turns monitoring loop off. +par_autossh Alternatively, a port for an echo service on the remote +par_autossh machine may be specified. (Normally port 7.) +par_autossh -f run in background (autossh handles this, and does not +par_autossh pass it to ssh.) +par_autossh -V print autossh version and exit. +par_autossh +par_autossh Environment variables are: +par_autossh AUTOSSH_GATETIME - how long must an ssh session be established +par_autossh before we decide it really was established +par_autossh (in seconds). Default is 30 seconds; use of -f +par_autossh flag sets this to 0. +par_autossh AUTOSSH_LOGFILE - file to log to (default is to use the syslog +par_autossh facility) +par_autossh AUTOSSH_LOGLEVEL - level of log verbosity +par_autossh AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds) +par_autossh AUTOSSH_MAXSTART - max times to restart (default is no limit) +par_autossh AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes) +par_autossh AUTOSSH_PATH - path to ssh if not default +par_autossh AUTOSSH_PIDFILE - write pid to this file +par_autossh AUTOSSH_POLL - how often to check the connection (seconds) +par_autossh AUTOSSH_FIRST_POLL - time before first connection check (seconds) +par_autossh AUTOSSH_PORT - port to use for monitor connection +par_autossh AUTOSSH_DEBUG - turn logging to maximum verbosity and log to +par_autossh stderr +par_autossh +par_autossh rsync: connection unexpectedly closed (0 bytes received so far) [sender] +par_autossh rsync error: error in rsync protocol data stream (code 12) at io.c(235) [sender=3.1.2] +par_autossh /usr/lib/autossh/autossh: invalid option -- '-' +par_autossh usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS] +par_autossh +par_autossh -M specifies monitor port. Overrides the environment +par_autossh variable AUTOSSH_PORT. 0 turns monitoring loop off. +par_autossh Alternatively, a port for an echo service on the remote +par_autossh machine may be specified. (Normally port 7.) +par_autossh -f run in background (autossh handles this, and does not +par_autossh pass it to ssh.) +par_autossh -V print autossh version and exit. +par_autossh +par_autossh Environment variables are: +par_autossh AUTOSSH_GATETIME - how long must an ssh session be established +par_autossh before we decide it really was established +par_autossh (in seconds). Default is 30 seconds; use of -f +par_autossh flag sets this to 0. +par_autossh AUTOSSH_LOGFILE - file to log to (default is to use the syslog +par_autossh facility) +par_autossh AUTOSSH_LOGLEVEL - level of log verbosity +par_autossh AUTOSSH_MAXLIFETIME - set the maximum time to live (seconds) +par_autossh AUTOSSH_MAXSTART - max times to restart (default is no limit) +par_autossh AUTOSSH_MESSAGE - message to append to echo string (max 64 bytes) +par_autossh AUTOSSH_PATH - path to ssh if not default +par_autossh AUTOSSH_PIDFILE - write pid to this file +par_autossh AUTOSSH_POLL - how often to check the connection (seconds) +par_autossh AUTOSSH_FIRST_POLL - time before first connection check (seconds) +par_autossh AUTOSSH_PORT - port to use for monitor connection +par_autossh AUTOSSH_DEBUG - turn logging to maximum verbosity and log to +par_autossh stderr +par_autossh +par_autossh rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] +par_autossh rsync error: error in rsync protocol data stream (code 12) at io.c(235) [Receiver=3.1.2] diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 23063f56..4dd45a54 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -278,7 +278,7 @@ par_tcsh_funky par_tcsh_funky 3 arg alias_works par_tcsh_funky 3 arg alias_works_over_ssh par_tcsh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works -par_tcsh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works_over_ssh +par_tcsh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky alias_var_works_over_ssh par_tcsh_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_tcsh_funky func_echo: Command not found. par_tcsh_funky func_echo: Command not found. @@ -966,21 +966,21 @@ par_bash_underscore variables in aliases in and arrays in functions work par_bash_underscore variables in aliases in and arrays in functions work par_bash_underscore variables in aliases in and arrays in functions work par_bash_underscore variables in aliases in and arrays in functions work -par_bash_underscore /bin/bash: line 56: not_copied_alias: command not found -par_bash_underscore /bin/bash: line 56: not_copied_func: command not found +par_bash_underscore /bin/bash: line XX: not_copied_alias: command not found +par_bash_underscore /bin/bash: line XX: not_copied_func: command not found par_bash_underscore error=OK par_bash_underscore error=OK par_bash_underscore aliases in and arrays in functions work par_bash_underscore aliases in and arrays in functions work par_bash_underscore aliases in functions work par_bash_underscore aliases in functions work -par_bash_underscore /bin/bash: line 49: myecho: command not found +par_bash_underscore /bin/bash: line XX: myecho: command not found par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 49: myecho: command not found +par_bash_underscore /bin/bash: line XX: myecho: command not found par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 49: myfunc: command not found +par_bash_underscore /bin/bash: line XX: myfunc: command not found par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 49: myfunc: command not found +par_bash_underscore /bin/bash: line XX: myfunc: command not found par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ par_bash_parset parset par_bash_parset ### parset into array @@ -1079,13 +1079,11 @@ par_bash_man exit value 2 should be 2 par_bash_man Unknown option: no-such-option par_bash_man exit value 255 should be 255 par_bash_funky -par_bash_funky -par_bash_funky par_bash_funky \\\\\\\\ par_bash_funky  par_bash_funky  -par_bash_funky -par_bash_funky +par_bash_funky -funkymultiline +par_bash_funky -funkymultiline par_bash_funky space 6 par_bash_funky space 6 par_bash_funky 3 arg alias_works @@ -1102,8 +1100,6 @@ par_bash_funky function_works par_bash_funky function_works_over_ssh par_bash_funky myvar works par_bash_funky myvar works -par_bash_funky -funkymultiline -par_bash_funky -funkymultiline par_bash_environment_too_big bug #50815: env_parallel should warn if the environment is too big par_bash_environment_too_big OK_bigvar par_bash_environment_too_big OK_bigvar_remote diff --git a/testsuite/wanted-results/parallel-local-ssh8 b/testsuite/wanted-results/parallel-local-ssh8 index f53a8d53..c29c8769 100644 --- a/testsuite/wanted-results/parallel-local-ssh8 +++ b/testsuite/wanted-results/parallel-local-ssh8 @@ -21,7 +21,6 @@ par_path_remote_bash ^^^^^^^^ Not found is OK par_path_remote_bash /bin:/usr/bin:/tmp OK par_path_remote_bash par_path_remote_csh bug #47695: How to set $PATH on remote? csh -par_path_remote_csh par_path_remote_csh Warning: no access to tty (Bad file descriptor). par_path_remote_csh Thus no job control in this shell. par_path_remote_csh CSH Path before: /bin:/usr/bin with no parallel diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 85475fc4..5a7463f7 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -14,10 +14,8 @@ To silence this citation notice: run 'parallel --citation'. echo a a -echo a cat via pseudotty -cat ### Test stdin as tty input for 'vi' NB: If this changes and the diff is printed to terminal, then the terminal settings may be fucked up. Use 'reset' to get back. @@ -35,7 +33,7 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice: run 'parallel --citation'. 10 files to edit -[?1049h[?1h=[?12;25h[?12l[?25h[?25l"file1" [New File]~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ [?12l[?25h[?25lE173: 9 more files to edit[?12l[?25h[?1l>[?1049lAcademic tradition requires you to cite works you base your article on. +[?1049h[?1h=[?12;25h[?12l[?25h[?25l"file1" [New File]~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ [?12l[?25h[?25lE173: 9 more files to edit[?12l[?25h[?1l>[?1049lAcademic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: diff --git a/testsuite/wanted-results/parallel-local150 b/testsuite/wanted-results/parallel-local150 index c04db525..9c73d198 100644 --- a/testsuite/wanted-results/parallel-local150 +++ b/testsuite/wanted-results/parallel-local150 @@ -11,22 +11,22 @@ echo '300 ms jobs:' echo '### Test --tagstring' ### Test --tagstring nice parallel -j1 -X -v --tagstring a{}b echo ::: 3 4 -echo 3 4 +a3b a4b echo 3 4 a3b a4b 3 4 nice parallel -j1 -k -v --tagstring a{}b echo ::: 3 4 -echo 3 +a3b echo 3 a3b 3 -echo 4 +a4b echo 4 a4b 4 nice parallel -j1 -k -v --tagstring a{}b echo job{#} ::: 3 4 -echo job1 +a3b echo job1 a3b job1 -echo job2 +a4b echo job2 a4b job2 nice parallel -j1 -k -v --tagstring ajob{#}b echo job{#} ::: 3 4 -echo job1 +ajob1b echo job1 ajob1b job1 -echo job2 +ajob2b echo job2 ajob2b job2 echo '### Bug in --load'; nice parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3 ### Bug in --load @@ -146,8 +146,8 @@ echo '### Test make .deb package'; cd ~/privat/parallel/packager/debian; std To install the GNU Parallel Debian package, run: echo '### Test of segfaulting issue' ### Test of segfaulting issue - echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; echo 'before adding wait() before exit'; seq 1 300 | stdout parallel ./trysegfault -This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 + echo 'This gave ~/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; echo 'before adding wait() before exit'; seq 1 300 | stdout parallel ./trysegfault +This gave ~/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 before adding wait() before exit echo '### Test basic --arg-sep' ### Test basic --arg-sep diff --git a/testsuite/wanted-results/parallel-local18 b/testsuite/wanted-results/parallel-local18 index e7dc95bf..d26bdeb1 100644 --- a/testsuite/wanted-results/parallel-local18 +++ b/testsuite/wanted-results/parallel-local18 @@ -7,6 +7,8 @@ par_env_newline_backslash_bash 8 par_env_newline_backslash_csh ### Test --env for \n and \\ - single and double (*csh only) - no output is good but csh fails par_env_newline_backslash_csh 2 2\ \92V2=\ \92 par_env_newline_backslash_csh 2 2\\ \92V2=\\ \92 +par_env_newline_backslash_csh 2 Unmatched ". +par_env_newline_backslash_csh 2 Unmatched '"'. par_env_newline_backslash_onall_bash ### Test --env for \n and \\ - single and double --onall (bash only) - no output is good par_env_newline_backslash_onall_bash 16 par_env_newline_backslash_onall_bash 16 @@ -15,7 +17,6 @@ par_env_newline_backslash_onall_bash 8 10 par_env_newline_backslash_onall_bash 8 10V2= par_env_newline_backslash_onall_bash 8 2\\ \92V2=\\ \92 par_env_newline_backslash_onall_csh ### Test --env for \n and \\ - single and double --onall (*csh only) - no output is good but csh fails -par_env_newline_backslash_onall_csh 8 Unmatched ". par_space ### Test --env - https://savannah.gnu.org/bugs/?37351 par_space a 2 spaces b 1 par_space a 2 spaces b 1 diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index 44b5d465..9afc2fdd 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -107,7 +107,7 @@ a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6 a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -km echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1 ### Test -m with 60000 args -d025d2d552a9ff809216d17d408de0b4 - +f5e1ea298b25c5516d63061df5c56f79 - 10 179980 1286692 echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1 ### Test -X with 60000 args diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 2a2ca2d9..052b4ffb 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -5,12 +5,12 @@ Chars per line: 116300 6 119994 697800 echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$ ### Test of xargs -X command lines > 130k -201ecb8ca789cecf39ee914a6d75611b - +3cfc69ee81b0fe7fdbe8eb059ad2da61 - Chars per line: 116826 7 60000 817788 echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum ### Test of xargs -m command lines > 130k -590091fd85dcb98f91c5e89cdddf21ef - +00b8aed2c1bc28368381273aa2ae104e - echo '### This causes problems if we kill child processes'; # 2>/dev/null to avoid parallel: Warning: Starting 45 processes took > 2 sec. ### This causes problems if we kill child processes seq 2 40 | parallel -j 0 seq 1 10 2>/dev/null | sort | md5sum diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index f3307ff7..2863ec0d 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -74,8 +74,8 @@ please cite as described in 'parallel --citation'. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' -131052 -37842 +131040 +37854 echo '### --delay should grow by 3 sec per arg' ### --delay should grow by 3 sec per arg stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' diff --git a/testsuite/wanted-results/parallel-local8 b/testsuite/wanted-results/parallel-local8 index d1028897..28c1f9c3 100644 --- a/testsuite/wanted-results/parallel-local8 +++ b/testsuite/wanted-results/parallel-local8 @@ -1,29 +1,28 @@ -### Test : as delimiter. This can be confusing for uptime ie. --load -a -b -c -PARALLEL=--load 300% - 4 4 8 -1c0f34fee7176dc367bead8f96cba6bc - - 9 9 18 -fa364205fcf6665c6f3e6cb868f65fd6 - - 16 16 39 -6f5db0373227d2281dc26b1bf63b4027 - - 25 25 66 -17e914b4a407dccd370c13173865deb1 - - 36 36 99 -5ee21398ecde0f3ea9b6093fbaf5a3c2 - - 49 49 138 -2af8be7306df18164a68e30e427217e0 - - 64 64 183 -f78c5b3d13146c60c9b586f51d05a4ae - - 81 81 234 -c88e1757ddc619efd9ee507a7702b53c - - 100 100 292 -8a7095c1c23bfadc311fe6b16d950582 - -### Test if --load blocks. Bug. -53d025127ae99ab79e8502aae2d9bea6 - -53d025127ae99ab79e8502aae2d9bea6 - -### Test reading load from PARALLEL -8a7095c1c23bfadc311fe6b16d950582 - -8a7095c1c23bfadc311fe6b16d950582 - +par_load_blocks ### Test if --load blocks. Bug. +par_load_blocks 53d025127ae99ab79e8502aae2d9bea6 - +par_load_blocks 53d025127ae99ab79e8502aae2d9bea6 - +par_load_from_PARALLEL ### Test reading load from PARALLEL +par_load_from_PARALLEL 8a7095c1c23bfadc311fe6b16d950582 - +par_load_from_PARALLEL 8a7095c1c23bfadc311fe6b16d950582 - +par_squared 100 100 292 +par_squared 8a7095c1c23bfadc311fe6b16d950582 - +par_squared 81 81 234 +par_squared c88e1757ddc619efd9ee507a7702b53c - +par_squared 64 64 183 +par_squared f78c5b3d13146c60c9b586f51d05a4ae - +par_squared 49 49 138 +par_squared 2af8be7306df18164a68e30e427217e0 - +par_squared 36 36 99 +par_squared 5ee21398ecde0f3ea9b6093fbaf5a3c2 - +par_squared 25 25 66 +par_squared 17e914b4a407dccd370c13173865deb1 - +par_squared 16 16 39 +par_squared 6f5db0373227d2281dc26b1bf63b4027 - +par_squared 9 9 18 +par_squared fa364205fcf6665c6f3e6cb868f65fd6 - +par_squared 4 4 8 +par_squared 1c0f34fee7176dc367bead8f96cba6bc - +par_test_delimiter ### Test : as delimiter. This can be confusing for uptime ie. --load +par_test_delimiter a +par_test_delimiter b +par_test_delimiter c diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 0ca69816..2eaa3c4a 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -63,7 +63,7 @@ echo '### Test --number-of-cpus'; stdout $NICEPAR --number-of-cpus 1 echo '### Test --number-of-cores'; stdout $NICEPAR --number-of-cores ### Test --number-of-cores -8 +2 echo '### Test --use-cpus-instead-of-cores'; (seq 1 8 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 8 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 8 physical CPUs'; wait ### Test --use-cpus-instead-of-cores Cores should complete first on machines with less than 8 physical CPUs @@ -400,7 +400,7 @@ echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit echo | $NICEPAR --show-limits Maximal size of command: 131049 -Maximal used size of command: 131057 +Maximal used size of command: 131049 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please diff --git a/testsuite/wanted-results/sql03 b/testsuite/wanted-results/sql03 index b7fdb23b..8850ae9e 100644 --- a/testsuite/wanted-results/sql03 +++ b/testsuite/wanted-results/sql03 @@ -1,109 +1,111 @@ -### Test reading sql from url command line -Test reading SQL from command line -Yes it works -### Test reading sql from url command line %-quoting -Test %-quoting SQL from command line -Yes it works -### Test .sql/aliases with url on commandline -Test if .sql/aliases with %-quoting works -Yes it works -### Test cyclic alias .sql/aliases -:cyclic3 is a cyclic alias -### Test alias with statement .sql/aliases -Test statement in alias -It works -Test statement in alias -It works -Test statement in alias -It works -### Test alias followed by SQL as arg -Test statement in alias -It works -'Arg on cmdline' -Arg on cmdline -### Test alias with query followed by SQL as arg -Test statement in alias -It works -Query added to alias -'Arg on cmdline' -Arg on cmdline -### Test alias with statement .sql/aliases -Test statement in alias -It works -Query added to alias -'Query from stdin' -Query from stdin -Test statement in alias -It works -Query added to alias -Query added to alias -Query from stdin -Query from stdin -### Test empty dburl -Error: - is not a valid DBURL - -sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command] -### Test dburl : -Error: -: is not defined in ~/.sql/aliases ~/.dburl.aliases /etc/sql/aliases /usr/local/bin/dburl.aliases /usr/local/bin/dburl.aliases.dist - -sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command] -### Test oracle with multiple arguments on the command line - -'ARG ----- -arg2 - - -'ARG ----- -arg3 - -### Test oracle with \n arguments on the command line - - 1 ----------- - 1 - - - 2 ----------- - 2 - - - 3 ----------- - 3 - -### Test --show-tables - - - --------------------------------------------------------------------------------------------------------------------------------- -7 rows selected. -COUNTRIES -DEPARTMENTS -EMPLOYEES -JOBS -JOB_HISTORY -LOCATIONS -OBJECT_NAME -REGIONS -### Test --show-databases - -TABLESPACE_NAME BLOCK_SIZE INITIAL_EXTENT NEXT_EXTENT MIN_EXTENTS MAX_EXTENTS MAX_SIZE PCT_INCREASE MIN_EXTLEN STATUS CONTENTS LOGGING FOR EXTENT_MAN ALLOCATIO SEGMEN DEF_TAB_ RETENTION BIG PREDICA ENC COMPRESS_FOR ------------------------------- ---------- -------------- ----------- ----------- ----------- ---------- ------------ ---------- --------- --------- --------- --- ---------- --------- ------ -------- ----------- --- ------- --- ------------ -SYSTEM 8192 65536 1 2147483645 2147483645 65536 ONLINE PERMANENT LOGGING NO LOCAL SYSTEM MANUAL DISABLED NOT APPLY NO HOST NO -SYSAUX 8192 65536 1 2147483645 2147483645 65536 ONLINE PERMANENT LOGGING NO LOCAL SYSTEM AUTO DISABLED NOT APPLY NO HOST NO -UNDOTBS1 8192 65536 1 2147483645 2147483645 65536 ONLINE UNDO LOGGING NO LOCAL SYSTEM MANUAL DISABLED NOGUARANTEE NO HOST NO -TEMP 8192 1048576 1048576 1 2147483645 0 1048576 ONLINE TEMPORARY NOLOGGING NO LOCAL UNIFORM MANUAL DISABLED NOT APPLY NO HOST NO -USERS 8192 65536 1 2147483645 2147483645 65536 ONLINE PERMANENT LOGGING NO LOCAL SYSTEM AUTO DISABLED NOT APPLY NO HOST NO - -### Test --listproc -SELECT CPU_TIME/100000, SYS.V_$SQL.SQL_TEXT, USERNAME FROM SYS.V_$SQL, SYS.V_$SESSION WHERE SYS.V_$SQL.SQL_ID = SYS.V_$SESSION.SQL_ID(+) AND username IS NOT NULL ORDER BY CPU_TIME DESC - * -ERROR at line 1: -ORA-00942: table or view does not exist - - +par_listproc ### Test --listproc +par_listproc SELECT CPU_TIME/100000, SYS.V_$SQL.SQL_TEXT, USERNAME FROM SYS.V_$SQL, SYS.V_$SESSION WHERE SYS.V_$SQL.SQL_ID = SYS.V_$SESSION.SQL_ID(+) AND username IS NOT NULL ORDER BY CPU_TIME DESC +par_listproc * +par_listproc ERROR at line 1: +par_listproc ORA-00942: table or view does not exist +par_listproc +par_listproc +par_listproc Id User Host db Command Time State Info +par_listproc XXX sqlunittest localhost sqlunittest Query 0 starting show processlist +par_multiarg_on_command_line ### Test oracle with multiple arguments on the command line +par_multiarg_on_command_line +par_multiarg_on_command_line 'ARG +par_multiarg_on_command_line ---- +par_multiarg_on_command_line arg2 +par_multiarg_on_command_line +par_multiarg_on_command_line +par_multiarg_on_command_line 'ARG +par_multiarg_on_command_line ---- +par_multiarg_on_command_line arg3 +par_multiarg_on_command_line +par_newline_on_commandline ### Test oracle with \n arguments on the command line +par_newline_on_commandline +par_newline_on_commandline 1 +par_newline_on_commandline ---------- +par_newline_on_commandline 1 +par_newline_on_commandline +par_newline_on_commandline +par_newline_on_commandline 2 +par_newline_on_commandline ---------- +par_newline_on_commandline 2 +par_newline_on_commandline +par_newline_on_commandline +par_newline_on_commandline 3 +par_newline_on_commandline ---------- +par_newline_on_commandline 3 +par_newline_on_commandline +par_showdatabases ### Test --show-databases +par_showdatabases +par_showdatabases TABLESPACE_NAME BLOCK_SIZE INITIAL_EXTENT NEXT_EXTENT MIN_EXTENTS MAX_EXTENTS MAX_SIZE PCT_INCREASE MIN_EXTLEN STATUS CONTENTS LOGGING FOR EXTENT_MAN ALLOCATIO SEGMEN DEF_TAB_ RETENTION BIG PREDICA ENC COMPRESS_FOR +par_showdatabases ------------------------------ ---------- -------------- ----------- ----------- ----------- ---------- ------------ ---------- --------- --------- --------- --- ---------- --------- ------ -------- ----------- --- ------- --- ------------ +par_showdatabases SYSTEM 8192 65536 1 2147483645 2147483645 65536 ONLINE PERMANENT LOGGING NO LOCAL SYSTEM MANUAL DISABLED NOT APPLY NO HOST NO +par_showdatabases SYSAUX 8192 65536 1 2147483645 2147483645 65536 ONLINE PERMANENT LOGGING NO LOCAL SYSTEM AUTO DISABLED NOT APPLY NO HOST NO +par_showdatabases UNDOTBS1 8192 65536 1 2147483645 2147483645 65536 ONLINE UNDO LOGGING NO LOCAL SYSTEM MANUAL DISABLED NOGUARANTEE NO HOST NO +par_showdatabases TEMP 8192 1048576 1048576 1 2147483645 0 1048576 ONLINE TEMPORARY NOLOGGING NO LOCAL UNIFORM MANUAL DISABLED NOT APPLY NO HOST NO +par_showdatabases USERS 8192 65536 1 2147483645 2147483645 65536 ONLINE PERMANENT LOGGING NO LOCAL SYSTEM AUTO DISABLED NOT APPLY NO HOST NO +par_showdatabases +par_showtables ### Test --show-tables +par_showtables +par_showtables +par_showtables +par_showtables -------------------------------------------------------------------------------------------------------------------------------- +par_showtables 7 rows selected. +par_showtables COUNTRIES +par_showtables DEPARTMENTS +par_showtables EMPLOYEES +par_showtables JOBS +par_showtables JOB_HISTORY +par_showtables LOCATIONS +par_showtables OBJECT_NAME +par_showtables REGIONS +par_sql_from_url ### Test reading sql from url command line +par_sql_from_url Test reading SQL from command line +par_sql_from_url Yes it works +par_sql_from_url ### Test reading sql from url command line %-quoting +par_sql_from_url Test %-quoting SQL from command line +par_sql_from_url Yes it works +par_sql_from_url ### Test .sql/aliases with url on commandline +par_sql_from_url Test if .sql/aliases with %-quoting works +par_sql_from_url Yes it works +par_test_alias_with_statement ### Test alias with statement .sql/aliases +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement ### Test alias followed by SQL as arg +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement 'Arg on cmdline' +par_test_alias_with_statement Arg on cmdline +par_test_alias_with_statement ### Test alias with query followed by SQL as arg +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement Query added to alias +par_test_alias_with_statement 'Arg on cmdline' +par_test_alias_with_statement Arg on cmdline +par_test_alias_with_statement ### Test alias with statement .sql/aliases +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement Query added to alias +par_test_alias_with_statement 'Query from stdin' +par_test_alias_with_statement Query from stdin +par_test_alias_with_statement Test statement in alias +par_test_alias_with_statement It works +par_test_alias_with_statement Query added to alias +par_test_alias_with_statement Query added to alias +par_test_alias_with_statement Query from stdin +par_test_alias_with_statement Query from stdin +par_test_cyclic ### Test cyclic alias .sql/aliases +par_test_cyclic :cyclic3 is a cyclic alias +par_test_dburl_colon ### Test dburl : +par_test_dburl_colon Error: +par_test_dburl_colon : is not defined in ~/.sql/aliases ~/.dburl.aliases /etc/sql/aliases /usr/local/bin/dburl.aliases /usr/local/bin/dburl.aliases.dist +par_test_dburl_colon +par_test_dburl_colon sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command] +par_test_empty_dburl ### Test empty dburl +par_test_empty_dburl Error: +par_test_empty_dburl is not a valid DBURL +par_test_empty_dburl +par_test_empty_dburl sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]