diff --git a/README b/README index 21ebe960..12f58a9d 100644 --- a/README +++ b/README @@ -123,7 +123,7 @@ When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, - DOI https://doi.org/10.5281/zenodo.11460 + DOI https://doi.org/10.5281/zenodo.1146014 = New versions = diff --git a/doc/release_new_version b/doc/release_new_version index 68cdb901..02127d29 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -209,9 +209,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20191222 ('Impeachment') released <<[stable]>> +Subject: GNU Parallel 20200122 ('Soleimani') released <<[stable]>> -GNU Parallel 20191222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20200122 ('Soleimani') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -221,6 +221,8 @@ See https://www.gnu.org/software/parallel/10-years-anniversary.html Quote of the month: + GNU parallel is straight up incredible. + -- Ben Johnson @biobenkj@twtter New in this release: @@ -228,7 +230,9 @@ New in this release: * GNU Parallel course in Århus https://www.prosa.dk/nc/arrangementer/arrangement/gnu-parallel-og-parallelisering-i-unix-shellen/ -* GNU Parallel is used in https://github.com/JeffersonLab/rfw_tsf_extractor + +https://net2.com/how-to-copy-a-file-to-multiple-directories-in-linux/ +https://dev.to/voyeg3r/runing-linux-commands-in-parallel-4ff8 * Bug fixes and man page updates. diff --git a/src/env_parallel b/src/env_parallel index 6524b3f2..75fe2e4d 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Copyright (C) 2016,2017,2018 -# Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 86a4fb8b..1a9e4ce6 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 6dc92b61..ed50e393 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.csh b/src/env_parallel.csh index 83289093..d4d5e6c1 100755 --- a/src/env_parallel.csh +++ b/src/env_parallel.csh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.dash b/src/env_parallel.dash index e9666991..a57684e8 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.fish b/src/env_parallel.fish index 1fe45e75..55ca64a4 100755 --- a/src/env_parallel.fish +++ b/src/env_parallel.fish @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 6a312047..fd8ce837 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 5fe13e62..bd555f46 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.pdksh b/src/env_parallel.pdksh index 55ea2586..f2343e68 100755 --- a/src/env_parallel.pdksh +++ b/src/env_parallel.pdksh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 76a3b05c..88654030 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -802,14 +802,12 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk -Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. =head1 LICENSE -Copyright (C) 2016,2017 Free Software Foundation, Inc. - This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or diff --git a/src/env_parallel.sh b/src/env_parallel.sh index ab4d6e4f..85ac4b21 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index 01c7fa41..a6be815c 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 3a2972d0..dbb8f3a8 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -7,7 +7,8 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/niceload b/src/niceload index 012e9f9b..17926838 100755 --- a/src/niceload +++ b/src/niceload @@ -2,7 +2,7 @@ # Copyright (C) 2004-2010 Ole Tange, http://ole.tange.dk # -# Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and +# Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and # Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify diff --git a/src/niceload.pod b/src/niceload.pod index 946be9b4..ab1b6562 100644 --- a/src/niceload.pod +++ b/src/niceload.pod @@ -305,7 +305,7 @@ Copyright (C) 2004-11-19 Ole Tange, http://ole.tange.dk Copyright (C) 2005-2010 Ole Tange, http://ole.tange.dk -Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. =head1 LICENSE diff --git a/src/parallel b/src/parallel index 48977448..28cbad5a 100755 --- a/src/parallel +++ b/src/parallel @@ -1,6 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2007-2019 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2007-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -504,7 +505,7 @@ sub pipe_shard_setup() { } else { $script = sharder_script(); } - + # cat foo | sharder sep col fifo1 fifo2 fifo3 ... fifoN if($shardbin =~ /^[a-z_][a-z_0-9]*(\s|$)/i) { @@ -589,8 +590,7 @@ sub find_header($$) { if($opt::header eq ":") { $opt::header = "(.*\n)"; } # Number = number of lines $opt::header =~ s/^(\d+)$/"(.*\n)"x$1/e; - while(read($fh,substr($$buf_ref,length $$buf_ref,0), - $Global::blocksize)) { + while(sysread($fh,$$buf_ref,$Global::blocksize,length $$buf_ref)) { if($$buf_ref =~ s/^($opt::header)//) { $header = $1; last; @@ -637,11 +637,11 @@ sub find_split_positions($$$) { push(@pos,$pos); } else { # Seek the the block start - if(not seek($fh, $pos, 0)) { + if(not sysseek($fh, $pos, 0)) { ::error("Cannot seek to $pos in $file"); edit(255); } - while(read($fh,substr($buf,length $buf,0),$dd_block_size)) { + while(sysread($fh,$buf,$dd_block_size,length $buf)) { if($opt::regexp) { # If match /$recend$recstart/ => Record position if($buf =~ m:^(.*$recend)$recstart:os) { @@ -667,7 +667,7 @@ sub find_split_positions($$$) { } } if($pos[$#pos] != $size) { - # Last splitpoint was not at end of the file: add it + # Last splitpoint was not at end of the file: add $size as the last push @pos, $size; } close $fh; @@ -948,131 +948,197 @@ sub spreadstdin() { my $two_gb = 2**31-1; my $blocksize = $Global::blocksize; my $in = *STDIN; + my $timeout = ::multiply_time_units($opt::blocktimeout); + my $header = find_header(\$buf,$in); + my $anything_written; + my $eof; + + sub read_block() { + # Read a --blocksize from STDIN + # possibly interrupted by --blocktimeout + # Add up to the next full block + my $readsize = $blocksize - (length $buf) % $blocksize; + my ($nread,$alarm); + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; # NB: \n required + # --blocktimeout (or 0 if not set) + alarm $timeout; + do { + $nread = sysread $in, $buf, $readsize, length $buf; + $readsize -= $nread; + } while($readsize and $nread); + alarm 0; + }; + if ($@) { + die unless $@ eq "alarm\n"; # propagate unexpected errors + $alarm = 1; + } else { + $alarm = 0; + } + $eof = not ($nread or $alarm); + } + + sub pass_n_line_records() { + # Pass records of N lines + my $n_lines = $buf =~ tr/\n/\n/; + my $last_newline_pos = rindex64(\$buf,"\n"); + # Go backwards until there are full n-line records + while($n_lines % $Global::max_lines) { + $n_lines--; + $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1); + } + # Chop at $last_newline_pos as that is where n-line record ends + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$last_newline_pos+1); + shorten(\$buf,$last_newline_pos+1); + } + + sub pass_n_regexps() { + # Pass records of N regexps + # -N => (start..*?end){n} + # -L -N => (start..*?end){n*l} + my $read_n_lines = -1+ + $Global::max_number_of_args * ($Global::max_lines || 1); + # (?!negative lookahead) is needed to avoid backtracking + # See: https://unix.stackexchange.com/questions/439356/ + while($buf =~ + /( + # Either recstart or at least one char from start + ^(?: $recstart | .) + # followed something + (?:(?!$recend$recstart).)*? + # and then recend + $recend + # Then n-1 times recstart.*recend + (?:$recstart(?:(?!$recend$recstart).)*?$recend){$read_n_lines} + ) + # Followed by recstart + (?=$recstart)/osx) { + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,length $1); + shorten(\$buf,length $1); + } + } + + sub pass_regexp() { + # Find the last recend-recstart in $buf + $eof and return; + if($buf =~ /^(.*$recend)$recstart.*?$/os) { + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,length $1); + shorten(\$buf,length $1); + } + } + + sub pass_csv_record() { + # Pass CVS record + # We define a CSV record as an even number of " + end of line + # This works if you use " as quoting character + my $last_newline_pos = length $buf; + # Go backwards from the last \n and search for a position + # where there is an even number of " + do { + # find last EOL + $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1); + # While uneven " + } while((substr($buf,0,$last_newline_pos) =~ y/"/"/)%2 + and $last_newline_pos >= 0); + # Chop at $last_newline_pos as that is where CSV record ends + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$last_newline_pos+1); + shorten(\$buf,$last_newline_pos+1); + } + + sub pass_n() { + # Pass n records of --recend/--recstart + # -N => (start..*?end){n} + my $i = 0; + my $read_n_lines = + $Global::max_number_of_args * ($Global::max_lines || 1); + while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1 + and + length $buf) { + $i += length $recend; # find the actual splitting location + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$i); + shorten(\$buf,$i); + } + } + + sub pass() { + # Pass records of --recend/--recstart + # Split record at fixed string + # Find the last recend+recstart in $buf + $eof and return; + my $i = rindex64(\$buf,$recendrecstart); + if($i != -1) { + $i += length $recend; # find the actual splitting location + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$i); + shorten(\$buf,$i); + } + } + + sub increase_blocksize_maybe() { + if(not $anything_written + and not $opt::blocktimeout + and not $Global::no_autoexpand_block) { + # Nothing was written - maybe the block size < record size? + # Increase blocksize exponentially up to 2GB-1 (2GB causes problems) + if($blocksize < $two_gb) { + my $old_blocksize = $blocksize; + $blocksize = ::min(ceil($blocksize * 1.3 + 1), $two_gb); + ::warning("A record was longer than $old_blocksize. " . + "Increasing to --blocksize $blocksize."); + } + } + } + while(1) { - my $anything_written = 0; - my $buflen = length $buf; - my $readsize = ($buflen < $blocksize) ? $blocksize-$buflen : $blocksize; - # If $buf < $blocksize, append so it is $blocksize long after reading. - # Otherwise append a full $blocksize - if(not read($in,substr($buf,$buflen,0),$readsize)) { - # End-of-file - $chunk_number != 1 and last; - # Force the while-loop once if everything was read by header reading - $one_time_through++ and last; - } - if($opt::r) { - # Remove empty lines - $buf =~ s/^\s*\n//gm; - if(length $buf == 0) { - next; - } - } - if($Global::max_lines and not $Global::max_number_of_args) { - # Read n-line records - my $n_lines = $buf =~ tr/\n/\n/; - my $last_newline_pos = rindex64(\$buf,"\n"); - # Go backwards until there are full n-line records - while($n_lines % $Global::max_lines) { - $n_lines--; - $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1); - } - # Chop at $last_newline_pos as that is where n-line record ends - $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$buf, - $recstart,$recend,$last_newline_pos+1); - shorten(\$buf,$last_newline_pos+1); - } elsif($opt::regexp) { - if($Global::max_number_of_args) { - # -N => (start..*?end){n} - # -L -N => (start..*?end){n*l} - my $read_n_lines = -1+ - $Global::max_number_of_args * ($Global::max_lines || 1); - # (?!negative lookahead) is needed to avoid backtracking - # See: https://unix.stackexchange.com/questions/439356/ - while($buf =~ - /( - # Either recstart or at least one char from start - ^(?: $recstart | .) - # followed something - (?:(?!$recend$recstart).)*? - # and then recend - $recend - # Then n-1 times recstart.*recend - (?:$recstart(?:(?!$recend$recstart).)*?$recend){$read_n_lines} - ) - # Followed by recstart - (?=$recstart)/osx) { - $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$buf, - $recstart,$recend,length $1); - shorten(\$buf,length $1); - } - } else { - eof($in) and last; - # Find the last recend-recstart in $buf - if($buf =~ /^(.*$recend)$recstart.*?$/os) { - $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$buf, - $recstart,$recend,length $1); - shorten(\$buf,length $1); - } - } - } elsif($opt::csv) { - # Read a full CSV record - # even number of " + end of line - my $last_newline_pos = length $buf; - do { - # find last EOL - $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1); - # While uneven " - } while((substr($buf,0,$last_newline_pos) =~ y/"/"/)%2 - and $last_newline_pos >= 0); - # Chop at $last_newline_pos as that is where CSV record ends - $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$buf, - $recstart,$recend,$last_newline_pos+1); - shorten(\$buf,$last_newline_pos+1); - } else { - if($Global::max_number_of_args) { - # -N => (start..*?end){n} - my $i = 0; - my $read_n_lines = - $Global::max_number_of_args * ($Global::max_lines || 1); - while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1 - and - length $buf) { - $i += length $recend; # find the actual splitting location - $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$buf, - $recstart,$recend,$i); - shorten(\$buf,$i); - } - } else { - eof($in) and last; - # Find the last recend+recstart in $buf - my $i = rindex64(\$buf,$recendrecstart); - if($i != -1) { - $i += length $recend; # find the actual splitting location - $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$buf, - $recstart,$recend,$i); - shorten(\$buf,$i); - } - } - } - if(not $anything_written - and not eof($in) - and not $Global::no_autoexpand_block) { - # Nothing was written - maybe the block size < record size? - # Increase blocksize exponentially up to 2GB-1 (2GB causes problems) - if($blocksize < $two_gb) { - my $old_blocksize = $blocksize; - $blocksize = ::min(ceil($blocksize * 1.3 + 1), $two_gb); - ::warning("A record was longer than $old_blocksize. " . - "Increasing to --blocksize $blocksize."); - } - } + $anything_written = 0; + read_block(); + if($opt::r) { + # Remove empty lines + $buf =~ s/^\s*\n//gm; + if(length $buf == 0) { + if($eof) { + last; + } else { + next; + } + } + } + if($Global::max_lines and not $Global::max_number_of_args) { + # Pass n-line records + pass_n_line_records(); + } elsif($opt::csv) { + # Pass a full CSV record + pass_csv_record(); + } elsif($opt::regexp) { + # Split record at regexp + if($Global::max_number_of_args) { + pass_n_regexps(); + } else { + pass_regexp(); + } + } else { + # Pass normal --recend/--recstart record + if($Global::max_number_of_args) { + pass_n(); + } else { + pass(); + } + } + $eof and last; + increase_blocksize_maybe(); + ::debug("init", "Round\n"); } ::debug("init", "Done reading input\n"); @@ -1092,7 +1158,7 @@ sub spreadstdin() { } } # Wait for running jobs to be done - my $sleep =1; + my $sleep = 1; while($Global::total_running > 0) { $sleep = ::reap_usleep($sleep); start_more_jobs(); @@ -1530,6 +1596,7 @@ sub options_hash() { "remove-rec-sep|removerecsep|rrs" => \$opt::remove_rec_sep, "files|output-as-files|outputasfiles" => \$opt::files, "block|block-size|blocksize=s" => \$opt::blocksize, + "blocktimeout|block-timeout|bt=s" => \$opt::blocktimeout, "tollef" => \$opt::tollef, "gnu" => \$opt::gnu, "link|xapply" => \$opt::link, @@ -2071,7 +2138,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20191222; + $Global::version = 20191223; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -2280,7 +2347,7 @@ sub parse_replacement_string_options() { for(@opt::rpl) { # Create $Global::rpl entries for --rpl options # E.g: "{..} s:\.[^.]+$:;s:\.[^.]+$:;" - my ($shorthand,$long) = split/ /,$_,2; + my ($shorthand,$long) = split/\s/,$_,2; $Global::rpl{$shorthand} = $long; } } @@ -4920,7 +4987,8 @@ sub version() { print join ("\n", "GNU $Global::progname $Global::version", - "Copyright (C) 2007-2019 Ole Tange and Free Software Foundation, Inc.", + "Copyright (C) 2007-2020 Ole Tange, http://ole.tange.dk and Free Software", + "Foundation, Inc.", "License GPLv3+: GNU GPL version 3 or later ", "This is free software: you are free to change and redistribute it.", "GNU $Global::progname comes with no warranty.", @@ -5050,8 +5118,8 @@ sub embed() { } print "#!$Global::shell -# Copyright (C) 2007-2019 $user, Ole Tange and Free Software -# Foundation, Inc. +# Copyright (C) 2007-2020 $user, Ole Tange, http://ole.tange.dk +# and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -5263,7 +5331,7 @@ sub max(@) { return $max; } -sub sum() { +sub sum(@) { # Returns: # Sum of values of array my @args = @_; @@ -10397,7 +10465,7 @@ sub push($) { push @{$self->{'arg_list'}}, $record; # Make @arg available for {= =} *Arg::arg = $self->{'arg_list_flat_orig'}; - + my $quote_arg = ($Global::quote_replace and not $Global::quoting); for my $perlexpr (keys %{$self->{'replacecount'}}) { if($perlexpr =~ /^(\d+) /) { diff --git a/src/parallel.pod b/src/parallel.pod index 5effa604..c70b6dd4 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -562,6 +562,21 @@ jobslots process a large amount of data. See B<--pipe> and B<--pipepart> for use of this. +=item B<--blocktimeout> I + +=item B<--bt> I + +Time out for reading block when using B<--pipe>. If it takes longer +than I to generate a full block, use the partial block read +so far. + +I must be in whole seconds, but can be expressed as floats +postfixed with B, B, B, or B which would multiply the +float by 1, 60, 3600, or 86400. Thus these are equivalent: B<--timeout +100000> and B<--timeout 1d3.5h16.6m4s>. + + + =item B<--cat> Create a temporary file with content. Normally B<--pipe>/B<--pipepart> @@ -5157,8 +5172,8 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk -Copyright (C) 2010-2019 Ole Tange, -http://ole.tange.dk and Free Software Foundation, Inc. +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free +Software Foundation, Inc. Parts of the manual concerning B compatibility is inspired by the manual of B from GNU findutils 4.4.2. diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index ec7c5dea..74219629 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -2290,10 +2290,69 @@ B parallel -j 3 ::: ls df "echo hi" (Last checked: 2019-08) +=head2 DIFFERENCES BETWEEN rargs AND GNU Parallel + +Summary table (see legend above): +I1 - - - - - I7 +- - M3 M4 - - +- O2 O3 - O5 O6 - O8 - +E1 - - E4 - - - +- - - - - - - - - +- - + +B has elegant ways of doing named regexp capture and field ranges. + +With GNU B you can use B<--rpl> to get a similar +functionality as regexp capture gives, and use B and B<@arg> to +get the field ranges. But the syntax is longer. This: + + --rpl '{r(\d+)\.\.(\d+)} $_=join"$opt::colsep",@arg[$$1..$$2]' + +would make it possible to use: + + {1r3..6} + +for field 3..6. + +For full support of {n..m:s} including negative numbers use a dynamic +replacement string like this: + + +PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)} + $a = defined $$2 ? $$2 < 0 ? 1+$#arg+$$2 : $$2 : 1; + $b = defined $$4 ? $$4 < 0 ? 1+$#arg+$$4 : $$4 : $#arg+1; + $s = defined $$6 ? $$7 : " "; + $_ = join $s,@arg[$a..$b]'\' +export PARALLEL + +You can then do: + + head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} 4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} ..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..} + + +=head3 EXAMPLES FROM rargs MANUAL + + ls *.bak | rargs -p '(.*)\.bak' mv {0} {1} + ls *.bak | parallel mv {} {.} + + cat download-list.csv | rargs -p '(?P.*),(?P.*)' wget {url} -O {filename} + cat download-list.csv | parallel --csv wget {1} -O {2} + # or use regexps: + cat download-list.csv | + parallel --rpl '{url} s/,.*//' --rpl '{filename} s/.*?,//' wget {url} -O {filename} + + cat /etc/passwd | rargs -d: echo -e 'id: "{1}"\t name: "{5}"\t rest: "{6..::}"' + cat /etc/passwd | + parallel -q --colsep : echo -e 'id: "{1}"\t name: "{5}"\t rest: "{=6 $_=join":",@arg[6..$#arg]=}"' + +https://github.com/lotabout/rargs (Last checked: 2020-01) + =head2 Todo -Url for spread +https://github.com/jreisinger/runp + +https://github.com/JeiKeiLim/simple_distribute_job https://github.com/reggi/pkgrun @@ -2557,7 +2616,7 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk -Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. Parts of the manual concerning B compatibility is inspired by diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 2b518196..db33a9c3 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -2998,9 +2998,9 @@ When asking for help, always report the full output of this: Output: - GNU parallel 20180122 - Copyright (C) 2007-2018 - Ole Tange and Free Software Foundation, Inc. + GNU parallel 20200122 + Copyright (C) 2007-2020 Ole Tange, http://ole.tange.dk and Free Software + Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. @@ -3160,7 +3160,7 @@ https://my.fsf.org/donate/ =back -(C) 2013-2019 Ole Tange, FDLv1.3 (See fdl.txt) +(C) 2013-2020 Ole Tange, FDLv1.3 (See fdl.txt) =cut diff --git a/src/parcat.pod b/src/parcat.pod index f2939b64..b558488d 100644 --- a/src/parcat.pod +++ b/src/parcat.pod @@ -60,7 +60,7 @@ GNU B is part of GNU B. Report bugs to =head1 AUTHOR -Copyright (C) 2016-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. =head1 LICENSE diff --git a/src/parset b/src/parset index 57ce8e0c..d1764056 100755 --- a/src/parset +++ b/src/parset @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Copyright (C) 2016-2019 Ole Tange, http://ole.tange.dk and -# Free Software Foundation, Inc. +# Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free +# Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/parset.pod b/src/parset.pod index acee72cf..f3f652fe 100644 --- a/src/parset.pod +++ b/src/parset.pod @@ -190,7 +190,7 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk -Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. diff --git a/src/sem.pod b/src/sem.pod index 3be621c0..c1b1c8c5 100644 --- a/src/sem.pod +++ b/src/sem.pod @@ -244,7 +244,7 @@ Report bugs to . =head1 AUTHOR -Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. diff --git a/src/sql b/src/sql index bccbec6f..a1f3036c 100755 --- a/src/sql +++ b/src/sql @@ -341,7 +341,7 @@ Databases Using DBURLs, ;login: The USENIX Magazine, April 2011:29-32. Copyright (C) 2008-2010 Ole Tange http://ole.tange.dk -Copyright (C) 2010-2019 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010-2020 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. =head1 LICENSE diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index 020c1b00..842cea3d 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -57,13 +57,16 @@ add_server_to_hosts() { fi } +shellsplus() { + shells="bash sh csh ash dash tcsh zsh ksh fish fizsh mksh posh rc sash yash nopathbash nopathcsh" + shellsplus="parallel $shells" + parallel -k echo ::: $shellsplus +} + create_shell_logins() { # SHELLS echo '# Create shell logins' touch ~/.zshrc - SSHPASS=`goodpasswd` - export SSHPASS - shells="bash sh csh ash dash tcsh zsh ksh fish fizsh mksh posh rc sash yash nopathbash nopathcsh" del_add_user() { shell="$1" append-if-not-exists() { @@ -77,23 +80,34 @@ create_shell_logins() { echo "$string" | sudo tee -a "$file" fi } + SSHPASS=`goodpasswd` + export SSHPASS append-if-not-exists /etc/shells $(which $shell || which ${shell#"nopath"}) sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM sudo groupdel $shell if echo $shell | grep -q parallel; then + # User parallel should have /bin/bash loginshell=/bin/bash else + # Other users should have `which $shell` with nopath removed loginshell=$(which $shell || which ${shell#"nopath"}) fi sudo adduser --shell $loginshell --disabled-password --gecos "$shell for parallel,,," $shell && - echo "$shell:$SSHPASS" | sudo chpasswd + echo "$shell:$SSHPASS" | sudo chpasswd && + sshpass -e ssh-copy-id $shell@lo && + echo "$shell created" } export -f del_add_user + echo '# (Re-)create user' + # Racecondition: if multiple adds a group it will the same group ID + shellsplus | parallel --timeout 15 --retries 5 --tag -j1 del_add_user +} + +copy_ssh_keys() { make_sshkey() { shell="$1" - sshpass -e ssh-copy-id $shell@lo - echo Add server keys for lo and server && + echo Add server keys for lo and server && ssh $shell@lo 'rm -f .ssh/id_rsa; ssh-keyscan lo >>.ssh/known_hosts; ssh-keyscan server >> .ssh/known_hosts' && echo Do ssh-keygen && echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa && @@ -109,21 +123,17 @@ create_shell_logins() { echo } export -f make_sshkey - . `which env_parallel.bash` - shellsplus="parallel $shells" - echo '# (Re-)create user' - # Racecondition: if multiple adds a group it will the same group ID - parallel --timeout 15 --retries 5 --tag -j1 del_add_user ::: $shellsplus echo '# Make and copy sshkey' - parallel --timeout 15 --retries 5 --tag -j5 make_sshkey ::: $shellsplus + shellsplus | parallel --timeout 20 --retries 5 --tag -j5 make_sshkey ssh_a_to_b() { ssh $1@lo ssh $2@lo echo OK || echo failed && false } export -f ssh_a_to_b - parallel -j8 --timeout 5 --tag ssh_a_to_b {} {} ::: $shellsplus + echo "# copy id from X to X" + shellsplus | parallel --timeout 10 --tag ssh_a_to_b {} {} ssh_copy_id() { from="$1" @@ -133,9 +143,9 @@ create_shell_logins() { export -f ssh_copy_id echo '# copy id from any X to any Y' - parallel -u --bar -j3 --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $shellsplus ::: $shellsplus + parallel -u --bar -j3 --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus) echo '# Test the copying went well' - parallel -j3 --timeout 3 --retries 10 --tag ssh_a_to_b ::: $shellsplus ::: $shellsplus + parallel -j2 --timeout 10 --retries 10 --tag ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus) echo '# change paths to no path' ( @@ -221,6 +231,7 @@ install_packages && setup_databases && add_server_to_hosts && create_shell_logins && + copy_ssh_keys && lsh_setup && add_freebsd && misc diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 1017c034..1fefdc2e 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -4,7 +4,6 @@ # Each should be taking 0.3-1s and be possible to run in parallel # I.e.: No race conditions, no logins - stdsort() { "$@" 2>&1 | LC_ALL=C sort; } diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 754873ad..6f9ce718 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -4,6 +4,201 @@ # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins +par_bug43654() { + echo "bug #43654: --bar with command not using {} - only last output line " + COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' +} + +par_replacement_rename() { + echo "### Test --basenamereplace" + parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test --bnr" + parallel -j1 -k -X --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + parallel -k --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test --extensionreplace" + parallel -j1 -k -X --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + parallel -k --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test --er" + parallel -j1 -k -X --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + parallel -k --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test --basenameextensionreplace" + parallel -j1 -k -X --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + parallel -k --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test --bner" + parallel -j1 -k -X --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + parallel -k --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b +} + +par_replacement_strings() { + echo "### Test {/}" + parallel -j1 -k -X echo {/} ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test {/.}" + parallel -j1 -k -X echo {/.} ::: /a/b.c a/b.c b.c /a/b a/b b + + echo "### Test {#/.}" + parallel -j1 -k -X echo {2/.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 + + echo "### Test {#/}" + parallel -j1 -k -X echo {2/} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 + + echo "### Test {#.}" + parallel -j1 -k -X echo {2.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 +} + +par_bug34241() { + echo "### bug #34241: --pipe should not spawn unneeded processes" + echo | parallel -r -j2 -N1 --pipe md5sum -c && echo OK +} + +par_test_gt_quoting() { + echo '### Test of quoting of > bug' + echo '>/dev/null' | parallel echo + + echo '### Test of quoting of > bug if line continuation' + (echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo +} + +par_trim_illegal_value() { + echo '### Test of --trim illegal' + stdout parallel --trim fj ::: echo +} + +par_eof_on_command_line_input_source() { + echo '### Test of eof string on :::' + parallel -k -E ole echo ::: foo ole bar +} + +par_empty_string_command_line() { + echo '### Test of ignore-empty string on :::' + parallel -k -r echo ::: foo '' ole bar +} + +par_trailing_space_line_continuation() { + echo '### Test of trailing space continuation' + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 echo + parallel -kr -L2 echo ::: foo '' 'ole ' bar quux + + echo '### Test of trailing space continuation with -E eof' + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 -E bar echo + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 -E bar echo + parallel -kr -L2 -E bar echo ::: foo '' 'ole ' bar quux +} + +par_colsep() { + echo '### Test of --colsep' + echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} + (echo 'a%c%b'; echo a%c%b%d) | parallel -k --colsep % echo {1} {3} {2} {4} + (echo a%c%b; echo d%f%e) | parallel -k --colsep % echo {1} {3} {2} + parallel -k --colsep % echo {1} {3} {2} ::: a%c%b d%f%e + parallel -k --colsep % echo {1} {3} {2} ::: a%c%b + parallel -k --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d + + + echo '### Test of tab as colsep' + printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1} + parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1} + + echo '### Test of multiple -a plus colsep' + parallel --xapply -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1} + + echo '### Test of multiple -a no colsep' + parallel --xapply -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1} + + echo '### Test of quoting after colsplit' + parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null' + + echo '### Test of --colsep as regexp' + (echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4} + parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' + + echo '### Test of -C' + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' + + echo '### Test of --trim n' + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' + parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' + + echo '### Test of bug: If input is empty string' + (echo ; echo abcbdbebf;echo abc) | parallel -k --colsep b -v echo {1}{2} +} + +par_mix_triple_colon_with_quad_colon() { + echo '### Test :::: mixed with :::' + echo '### Test :::: < ::: :::' + parallel -k echo {1} {2} {3} :::: <(seq 6 7) ::: 4 5 ::: 1 2 3 + + echo '### Test :::: < < :::: <' + parallel -k echo {1} {2} {3} :::: <(seq 6 7) <(seq 4 5) :::: <(seq 1 3) + + echo '### Test -a :::: < :::: <' + parallel -k -a <(seq 6 7) echo {1} {2} {3} :::: <(seq 4 5) :::: <(seq 1 3) + + echo '### Test -a -a :::' + parallel -k -a <(seq 6 7) -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 + + echo '### Test -a - -a :::' + seq 6 7 | parallel -k -a - -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 + + echo '### Test :::: < - :::' + seq 4 5 | parallel -k echo {1} {2} {3} :::: <(seq 6 7) - ::: 1 2 3 +} + +par_test_E() { + echo '### Test -E' + seq 1 100 | parallel -k -E 5 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) + + echo '### Test -E one empty' + seq 1 100 | parallel -k -E 3 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) + + echo '### Test -E 2 empty' + seq 1 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) + + echo '### Test -E all empty' + seq 3 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) +} + +par_test_job_number() { + echo '### Test {#}' + seq 1 10 | parallel -k echo {#} +} + +par_seqreplace_long_line() { + echo '### Test --seqreplace and line too long' + seq 1 1000 | stdout parallel -j1 -s 210 -k --seqreplace I echo IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII \|wc | uniq -c +} + +par_bug37042() { + echo '### bug #37042: -J foo is taken from the whole command line - not just the part before the command' + echo '--tagstring foo' > ~/.parallel/bug_37042_profile; + parallel -J bug_37042_profile echo ::: tag_with_foo; + parallel --tagstring a -J bug_37042_profile echo ::: tag_with_a; + parallel --tagstring a echo -J bug_37042_profile ::: print_-J_bug_37042_profile + + echo '### Bug introduce by fixing bug #37042' + parallel --xapply -a <(printf 'abc') --colsep '\t' echo {1} +} + +par_header() { + echo "### Test --header with -N" + (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '.*\n.*\n' echo Start\;cat \; echo Stop + + echo "### Test --header with --block 1k" + (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"}') | + parallel -j1 --pipe -k --block 1k --header '.*\n.*\n' echo Start\;cat \; echo Stop + + echo "### Test --header with multiple :::" + parallel --header : echo {a} {b} {1} {2} ::: b b1 ::: a a2 +} par_profiles_with_space() { echo '### bug #42902: profiles containing arguments with space' diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index 4ced1529..c4e70061 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -8,7 +8,8 @@ par_tee_with_premature_close() { echo '--tee --pipe should send all data to all commands' echo 'even if a command closes stdin before reading everything' echo 'tee with --output-error=warn-nopipe support' - seq 1000000 | parallel -k --tee --pipe ::: wc head tail 'sleep 1' + correct="$(seq 1000000 | parallel -k --tee --pipe ::: wc head tail 'sleep 1')" + echo "$correct" echo 'tee without --output-error=warn-nopipe support' cat > tmp/tee <<-EOF #!/usr/bin/perl @@ -24,8 +25,12 @@ par_tee_with_premature_close() { # * tee not supporting --output-error=warn-nopipe # * sleep closes stdin before EOF # Depending on tee it may provide partial output or no output - seq 1000000 | parallel -k --tee --pipe ::: wc head tail 'sleep 1' - echo + wrong="$(seq 1000000 | parallel -k --tee --pipe ::: wc head tail 'sleep 1')" + if diff <(echo "$correct") <(echo "$wrong") >/dev/null; then + echo Wrong: They should not give the same output + else + echo OK + fi } par_maxargs() { @@ -170,6 +175,7 @@ par_resume_failed_k() { parallel -k --resume-failed --joblog $tmp echo job{#} val {}\;exit {} ::: 0 1 2 3 0 1 echo with exit 0 parallel -k --resume-failed --joblog $tmp echo job{#} val {}\;exit 0 ::: 0 1 2 3 0 1 + sleep 0.5 echo try 2 again. Gives empty parallel -k --resume-failed --joblog $tmp echo job{#} val {}\;exit {} ::: 0 1 2 3 0 1 rm $tmp diff --git a/testsuite/tests-to-run/parallel-local-race01.sh b/testsuite/tests-to-run/parallel-local-race01.sh index a9d00749..a28e1c97 100755 --- a/testsuite/tests-to-run/parallel-local-race01.sh +++ b/testsuite/tests-to-run/parallel-local-race01.sh @@ -47,9 +47,9 @@ par_distribute_input_by_ability() { echo "### Distribute input to jobs that are ready" echo "Job-slot n is 50% slower than n+1, so the order should be 1..7" seq 20000000 | - parallel --tagstring {#} -j7 --block 300k --round-robin --pipe \ + parallel --tagstring {#} -j7 --block 300k --round-robin --pipe \ 'pv -qL{=$_=$job->seq()**3+9=}0000 |wc -c' | - sort -nk2 | field 1 + sort -nk2 | field 1 } par_print_before_halt_on_error() { @@ -69,6 +69,29 @@ par_print_before_halt_on_error() { parallel -j0 -k --tag mytest ::: -2 -1 0 1 2 } +par_bug56403() { + echo 'bug #56403: --pipe block by time.' + ( + sleep 2; # make sure parallel is ready + echo job1a; + sleep 2; + echo job2b; + echo -n job3c; + sleep 2; + echo job3d + ) | parallel --blocktimeout 1 -j1 --pipe --tagstring {#} -k cat + ( + sleep 2; # make sure parallel is ready + echo job1a; + sleep 1; + echo job1b; + echo -n job2c; + sleep 4; + echo job2d + ) | parallel --blocktimeout 3 -j1 --pipe --tagstring {#} -k cat + +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | # parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 4247c4d5..f67f6c27 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -1707,14 +1707,16 @@ _EOF par_bash_environment_too_big() { myscript=$(cat <<'_EOF' echo 'bug #50815: env_parallel should warn if the environment is too big' - len_var=100+50-25-12+6-3+2-1 - len_var_remote=100-50+25+12-6+3 - len_var_quote=100-50+25-12-6+3-2 - len_var_quote_remote=100-50-25+12+6-3 - len_fun=100+50-25-12+6-3 - len_fun_remote=100-50+25+12-6+3 - len_fun_quote=100+50-25-12 - len_fun_quote_remote=100-50+25+12-6 + len_functions=-$(typeset -f | wc -c)/1000 + len_variables=-$(typeset -p | wc -c)/1000 + len_var=$len_variables+100 + len_var_remote=$len_variables+100-50+25-12+6 + len_var_quote=$len_variables+100-50 + len_var_quote_remote=$len_variables+100-50-25+12 + len_fun=$len_functions+100 + len_fun_remote=$len_functions+100-50+25-12+6 + len_fun_quote=$len_functions+100 + len_fun_quote_remote=$len_functions+100-50 . `which env_parallel.bash`; @@ -1856,14 +1858,16 @@ par_fish_environment_too_big() { par_ksh_environment_too_big() { myscript=$(cat <<'_EOF' echo 'bug #50815: env_parallel should warn if the environment is too big' - len_var=100+50-25-12 - len_var_remote=100-50+25+12-6-3 - len_var_quote=100+50-25-12 - len_var_quote_remote=100-50+25+12-6 - len_fun=100+12+6-3 - len_fun_remote=100-50+25+12-6+3 - len_fun_quote=100+50-25-12 - len_fun_quote_remote=100-50+25+12-6 + len_functions=-$(functions|wc -c)/1000 + len_variables=-$(typeset -p | wc -c)/1000 + len_var=$len_variables+100 + len_var_remote=$len_variables+100-50+25-12+6 + len_var_quote=$len_variables+100 + len_var_quote_remote=$len_variables+100-50+25 + len_fun=$len_functions+100 + len_fun_remote=$len_functions+100-50+25-12+6 + len_fun_quote=$len_functions+100 + len_fun_quote_remote=$len_functions+100-50 . `which env_parallel.ksh`; diff --git a/testsuite/tests-to-run/parallel-local-ssh8.sh b/testsuite/tests-to-run/parallel-local-ssh8.sh index 3ebedaaa..f74b2abd 100644 --- a/testsuite/tests-to-run/parallel-local-ssh8.sh +++ b/testsuite/tests-to-run/parallel-local-ssh8.sh @@ -88,25 +88,34 @@ par_bar_m() { uniq } +retries() { + retries=$1 + min=$2 + max=$3 + export PARALLEL="--retries $retries -S 12/localhost,1/:,parallel@lo -uq" + tries=$(seq 0 12 | + parallel perl -e 'print "job{}\n";exit({})' 2>/dev/null | + wc -l) + # Dont care if they are off by one + if [ $min -le $tries -o $tries -le $max ] ; then + echo OK + fi +} +export -f retries + par_retries_1() { echo '### Test of --retries - it should run 13 jobs in total'; - export PARALLEL="--retries 1 -S 12/localhost,1/:,parallel@lo -uq" - seq 0 12 | - parallel perl -e 'print "job{}\n";exit({})' 2>/dev/null | wc -l + retries 1 11 13 } par_retries_2() { echo '### Test of --retries - it should run 25 jobs in total'; - export PARALLEL="--retries 2 -S 12/localhost,1/:,parallel@lo -uq" - seq 0 12 | - parallel perl -e 'print "job{}\n";exit({})' 2>/dev/null | wc -l + retries 2 24 25 } par_retries_4() { echo '### Test of --retries - it should run 49 jobs in total'; - export PARALLEL="--retries 4 -S 12/localhost,1/:,parallel@lo -uq" - seq 0 12 | - parallel perl -e 'print "job{}\n";exit({})' 2>/dev/null | wc -l + retries 4 48 49 } par_retries_bug_from_2010() { diff --git a/testsuite/tests-to-run/parallel-local1.sh b/testsuite/tests-to-run/parallel-local1.sh index 295bbd50..6d7842eb 100644 --- a/testsuite/tests-to-run/parallel-local1.sh +++ b/testsuite/tests-to-run/parallel-local1.sh @@ -1,167 +1,7 @@ #!/bin/bash cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r -echo "bug #43654: --bar with command not using {} - only last output line " - COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' - -echo "### Test --basenamereplace" - parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test --bnr" - parallel -j1 -k -X --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - parallel -k --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test --extensionreplace" - parallel -j1 -k -X --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - parallel -k --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test --er" - parallel -j1 -k -X --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - parallel -k --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test --basenameextensionreplace" - parallel -j1 -k -X --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - parallel -k --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test --bner" - parallel -j1 -k -X --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - parallel -k --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test {/}" - parallel -j1 -k -X echo {/} ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test {/.}" - parallel -j1 -k -X echo {/.} ::: /a/b.c a/b.c b.c /a/b a/b b - -echo "### Test {#/.}" - parallel -j1 -k -X echo {2/.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 - -echo "### Test {#/}" - parallel -j1 -k -X echo {2/} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 - -echo "### Test {#.}" - parallel -j1 -k -X echo {2.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 - -echo "### bug #34241: --pipe should not spawn unneeded processes" - echo | parallel -r -j2 -N1 --pipe md5sum -c && echo OK - -echo '### Test of quoting of > bug' - echo '>/dev/null' | parallel echo - -echo '### Test of quoting of > bug if line continuation' - (echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo - -echo '### Test of --trim illegal' - stdout parallel --trim fj ::: echo - -echo '### Test of eof string on :::' - parallel -k -E ole echo ::: foo ole bar - -echo '### Test of ignore-empty string on :::' - parallel -k -r echo ::: foo '' ole bar - -echo '### Test of trailing space continuation' - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 echo - parallel -kr -L2 echo ::: foo '' 'ole ' bar quux - -echo '### Test of trailing space continuation with -E eof' - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 -E bar echo - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 -E bar echo - parallel -kr -L2 -E bar echo ::: foo '' 'ole ' bar quux - -echo '### Test of --colsep' - echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} - (echo 'a%c%b'; echo a%c%b%d) | parallel -k --colsep % echo {1} {3} {2} {4} - (echo a%c%b; echo d%f%e) | parallel -k --colsep % echo {1} {3} {2} - parallel -k --colsep % echo {1} {3} {2} ::: a%c%b d%f%e - parallel -k --colsep % echo {1} {3} {2} ::: a%c%b - parallel -k --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d - -echo '### Test of tab as colsep' - printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1} - parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1} - -echo '### Test of multiple -a plus colsep' - parallel --xapply -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1} - -echo '### Test of multiple -a no colsep' - parallel --xapply -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1} - -echo '### Test of quoting after colsplit' - parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null' - -echo '### Test of --colsep as regexp' - (echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4} - parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' - -echo '### Test of -C' - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' - -echo '### Test of --trim n' - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' - parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' - -echo '### Test of bug: If input is empty string' - (echo ; echo abcbdbebf;echo abc) | parallel -k --colsep b -v echo {1}{2} - -echo '### Test :::: mixed with :::' -echo '### Test :::: < ::: :::' - parallel -k echo {1} {2} {3} :::: <(seq 6 7) ::: 4 5 ::: 1 2 3 - -echo '### Test :::: < < :::: <' - parallel -k echo {1} {2} {3} :::: <(seq 6 7) <(seq 4 5) :::: <(seq 1 3) - -echo '### Test -a :::: < :::: <' - parallel -k -a <(seq 6 7) echo {1} {2} {3} :::: <(seq 4 5) :::: <(seq 1 3) - -echo '### Test -a -a :::' - parallel -k -a <(seq 6 7) -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 - -echo '### Test -a - -a :::' - seq 6 7 | parallel -k -a - -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 - -echo '### Test :::: < - :::' - seq 4 5 | parallel -k echo {1} {2} {3} :::: <(seq 6 7) - ::: 1 2 3 - -echo '### Test -E' - seq 1 100 | parallel -k -E 5 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) - -echo '### Test -E one empty' - seq 1 100 | parallel -k -E 3 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) - -echo '### Test -E 2 empty' - seq 1 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) - -echo '### Test -E all empty' - seq 3 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) - -echo '### Test {#}' - seq 1 10 | parallel -k echo {#} - -echo '### Test --seqreplace and line too long' - seq 1 1000 | stdout parallel -j1 -s 210 -k --seqreplace I echo IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII \|wc | uniq -c - -echo '### bug #37042: -J foo is taken from the whole command line - not just the part before the command' - echo '--tagstring foo' > ~/.parallel/bug_37042_profile; - parallel -J bug_37042_profile echo ::: tag_with_foo; - parallel --tagstring a -J bug_37042_profile echo ::: tag_with_a; - parallel --tagstring a echo -J bug_37042_profile ::: print_-J_bug_37042_profile - -echo '### Bug introduce by fixing bug #37042' - parallel --xapply -a <(printf 'abc') --colsep '\t' echo {1} - -echo "### Test --header with -N" - (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '.*\n.*\n' echo Start\;cat \; echo Stop - -echo "### Test --header with --block 1k" - (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '.*\n.*\n' echo Start\;cat \; echo Stop - -echo "### Test --header with multiple :::" - parallel --header : echo {a} {b} {1} {2} ::: b b1 ::: a a2 +echo moved to parallel-local-1s.sh EOF diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index 564f4b08..9b26187a 100644 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -206,3 +206,14 @@ env_parallel -u -S$MASTER doit ::: 1 | # env_parallel myfunc ::: a | wc # OK # eval 'myfunc2() { echo '$(perl -e 'print "x"x120000')'; }' # env_parallel myfunc ::: a | wc # Fail too big env + +# Supported keylength: +# 16300: +# debian-ppc netbsd openbsd qnx aix centos freebsd hpux hpux-ia64 +# macosx mandriva miros raspbian redhat scosysv suse tru64 unixware +# 8000: +# solaris-x86 +# 4000: +# openindiana solaris + + diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index fb83232a..b7e178b7 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1019,7 +1019,7 @@ par_verbose_t echo bar par_verbose_t echo car par_verbose_t echo far par_version ### Test --version: Version output (just check we get the same amount of lines) -par_version 10 +par_version 11 par_wd_3dot_local bug #45993: --wd ... should also work when run locally par_wd_3dot_local parallel: Error: Cannot change into non-executable dir /bi: No such file or directory par_wd_3dot_local /bin diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index a3948ff7..a1b0d5c9 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -743,9 +743,9 @@ par_shellquote tcsh \\\\\\\\ "\ par_shellquote tcsh "\ \ \ par_shellquote tcsh \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_sockets_cores_threads ### Test --number-of-sockets/cores/threads par_sockets_cores_threads 1 +par_sockets_cores_threads 2 par_sockets_cores_threads 4 -par_sockets_cores_threads 8 -par_sockets_cores_threads 4 +par_sockets_cores_threads 2 par_sockets_cores_threads ### Test --use-sockets-instead-of-threads par_sockets_cores_threads threads done par_sockets_cores_threads sockets done diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 6c6d1749..2bd83b88 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -1,5 +1,63 @@ par_block_negative_prefix ### This should generate 10*2 jobs par_block_negative_prefix 20 40 720 +par_bug34241 ### bug #34241: --pipe should not spawn unneeded processes +par_bug34241 OK +par_bug37042 ### bug #37042: -J foo is taken from the whole command line - not just the part before the command +par_bug37042 foo tag_with_foo +par_bug37042 a tag_with_a +par_bug37042 a -J bug_37042_profile print_-J_bug_37042_profile +par_bug37042 ### Bug introduce by fixing bug #37042 +par_bug37042 abc +par_bug43654 bug #43654: --bar with command not using {} - only last output line +par_bug43654 par_bug43654 100% 1:0=0s 1  +par_colsep ### Test of --colsep +par_colsep a b c +par_colsep a b c +par_colsep a b c d +par_colsep a b c +par_colsep d e f +par_colsep a b c +par_colsep d e f +par_colsep a b c +par_colsep a b c +par_colsep a b c d +par_colsep ### Test of tab as colsep +par_colsep abc def +par_colsep ghi jkl +par_colsep abc def +par_colsep ghi jkl +par_colsep ### Test of multiple -a plus colsep +par_colsep abc def +par_colsep mno jkl +par_colsep ### Test of multiple -a no colsep +par_colsep abc def ghi +par_colsep jkl mno pqr +par_colsep ### Test of quoting after colsplit +par_colsep >/tmp/null >/dev/null +par_colsep ### Test of --colsep as regexp +par_colsep a b c +par_colsep a b c d +par_colsep a b c +par_colsep a b c d +par_colsep a b c +par_colsep a b c d +par_colsep a_b_c_ +par_colsep a_b_c_d +par_colsep ### Test of -C +par_colsep a_b_c_ +par_colsep a_b_c_d +par_colsep ### Test of --trim n +par_colsep a_b_' c '_ +par_colsep a_' b '_c_d +par_colsep a_b_c_ +par_colsep a_b_c_d +par_colsep ### Test of bug: If input is empty string +par_colsep echo '' +par_colsep +par_colsep echo ac +par_colsep ac +par_colsep echo ac +par_colsep ac par_compress_prg_fails ### bug #44546: If --compress-program fails: fail par_compress_prg_fails 1 par_compress_prg_fails parallel: Error: false failed. @@ -25,6 +83,12 @@ par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. par_compress_prg_fails --line-buffer --tag --files 1 par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. +par_empty_string_command_line ### Test of ignore-empty string on ::: +par_empty_string_command_line foo +par_empty_string_command_line ole +par_empty_string_command_line bar +par_eof_on_command_line_input_source ### Test of eof string on ::: +par_eof_on_command_line_input_source foo par_failing_compressor Compress with failing (de)compressor par_failing_compressor Test --tag/--line-buffer/--files in all combinations par_failing_compressor Test working/failing compressor/decompressor in all combinations @@ -155,6 +219,61 @@ par_fifo_under_csh 1 par_fifo_under_csh 868832 par_fifo_under_csh 1 par_fifo_under_csh exit 22 +par_header ### Test --header with -N +par_header Start +par_header h1 +par_header h2 +par_header 1a +par_header 1b +par_header Stop +par_header Start +par_header h1 +par_header h2 +par_header 2a +par_header 2b +par_header Stop +par_header Start +par_header h1 +par_header h2 +par_header 3a +par_header Stop +par_header ### Test --header with --block 1k +par_header Start +par_header h1 +par_header h2 +par_header 1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 3xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 4xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 5xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 6xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 7xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 8xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header Stop +par_header Start +par_header h1 +par_header h2 +par_header 9xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 10xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 11xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 12xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 13xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 14xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 15xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 16xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header Stop +par_header Start +par_header h1 +par_header h2 +par_header 17xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 18xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 19xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 20xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 21xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header 22xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +par_header Stop +par_header ### Test --header with multiple ::: +par_header a2 b1 b1 a2 par_header_parens bug #49538: --header and {= =} par_header_parens OK par_header_parens OK @@ -174,6 +293,85 @@ par_incomplete_linebuffer 7 par_incomplete_linebuffer 8 par_incomplete_linebuffer 9 par_incomplete_linebuffer 10 +par_mix_triple_colon_with_quad_colon ### Test :::: mixed with ::: +par_mix_triple_colon_with_quad_colon ### Test :::: < ::: ::: +par_mix_triple_colon_with_quad_colon 6 4 1 +par_mix_triple_colon_with_quad_colon 6 4 2 +par_mix_triple_colon_with_quad_colon 6 4 3 +par_mix_triple_colon_with_quad_colon 6 5 1 +par_mix_triple_colon_with_quad_colon 6 5 2 +par_mix_triple_colon_with_quad_colon 6 5 3 +par_mix_triple_colon_with_quad_colon 7 4 1 +par_mix_triple_colon_with_quad_colon 7 4 2 +par_mix_triple_colon_with_quad_colon 7 4 3 +par_mix_triple_colon_with_quad_colon 7 5 1 +par_mix_triple_colon_with_quad_colon 7 5 2 +par_mix_triple_colon_with_quad_colon 7 5 3 +par_mix_triple_colon_with_quad_colon ### Test :::: < < :::: < +par_mix_triple_colon_with_quad_colon 6 4 1 +par_mix_triple_colon_with_quad_colon 6 4 2 +par_mix_triple_colon_with_quad_colon 6 4 3 +par_mix_triple_colon_with_quad_colon 6 5 1 +par_mix_triple_colon_with_quad_colon 6 5 2 +par_mix_triple_colon_with_quad_colon 6 5 3 +par_mix_triple_colon_with_quad_colon 7 4 1 +par_mix_triple_colon_with_quad_colon 7 4 2 +par_mix_triple_colon_with_quad_colon 7 4 3 +par_mix_triple_colon_with_quad_colon 7 5 1 +par_mix_triple_colon_with_quad_colon 7 5 2 +par_mix_triple_colon_with_quad_colon 7 5 3 +par_mix_triple_colon_with_quad_colon ### Test -a :::: < :::: < +par_mix_triple_colon_with_quad_colon 6 4 1 +par_mix_triple_colon_with_quad_colon 6 4 2 +par_mix_triple_colon_with_quad_colon 6 4 3 +par_mix_triple_colon_with_quad_colon 6 5 1 +par_mix_triple_colon_with_quad_colon 6 5 2 +par_mix_triple_colon_with_quad_colon 6 5 3 +par_mix_triple_colon_with_quad_colon 7 4 1 +par_mix_triple_colon_with_quad_colon 7 4 2 +par_mix_triple_colon_with_quad_colon 7 4 3 +par_mix_triple_colon_with_quad_colon 7 5 1 +par_mix_triple_colon_with_quad_colon 7 5 2 +par_mix_triple_colon_with_quad_colon 7 5 3 +par_mix_triple_colon_with_quad_colon ### Test -a -a ::: +par_mix_triple_colon_with_quad_colon 6 4 1 +par_mix_triple_colon_with_quad_colon 6 4 2 +par_mix_triple_colon_with_quad_colon 6 4 3 +par_mix_triple_colon_with_quad_colon 6 5 1 +par_mix_triple_colon_with_quad_colon 6 5 2 +par_mix_triple_colon_with_quad_colon 6 5 3 +par_mix_triple_colon_with_quad_colon 7 4 1 +par_mix_triple_colon_with_quad_colon 7 4 2 +par_mix_triple_colon_with_quad_colon 7 4 3 +par_mix_triple_colon_with_quad_colon 7 5 1 +par_mix_triple_colon_with_quad_colon 7 5 2 +par_mix_triple_colon_with_quad_colon 7 5 3 +par_mix_triple_colon_with_quad_colon ### Test -a - -a ::: +par_mix_triple_colon_with_quad_colon 6 4 1 +par_mix_triple_colon_with_quad_colon 6 4 2 +par_mix_triple_colon_with_quad_colon 6 4 3 +par_mix_triple_colon_with_quad_colon 6 5 1 +par_mix_triple_colon_with_quad_colon 6 5 2 +par_mix_triple_colon_with_quad_colon 6 5 3 +par_mix_triple_colon_with_quad_colon 7 4 1 +par_mix_triple_colon_with_quad_colon 7 4 2 +par_mix_triple_colon_with_quad_colon 7 4 3 +par_mix_triple_colon_with_quad_colon 7 5 1 +par_mix_triple_colon_with_quad_colon 7 5 2 +par_mix_triple_colon_with_quad_colon 7 5 3 +par_mix_triple_colon_with_quad_colon ### Test :::: < - ::: +par_mix_triple_colon_with_quad_colon 6 4 1 +par_mix_triple_colon_with_quad_colon 6 4 2 +par_mix_triple_colon_with_quad_colon 6 4 3 +par_mix_triple_colon_with_quad_colon 6 5 1 +par_mix_triple_colon_with_quad_colon 6 5 2 +par_mix_triple_colon_with_quad_colon 6 5 3 +par_mix_triple_colon_with_quad_colon 7 4 1 +par_mix_triple_colon_with_quad_colon 7 4 2 +par_mix_triple_colon_with_quad_colon 7 4 3 +par_mix_triple_colon_with_quad_colon 7 5 1 +par_mix_triple_colon_with_quad_colon 7 5 2 +par_mix_triple_colon_with_quad_colon 7 5 3 par_null_resume ### --null --resume --jl par_null_resume a par_null_resume b @@ -223,6 +421,64 @@ par_pxz_complains bug #44250: pxz complains File format not recognized but decom par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory par_pxz_complains can not seek in input: Illegal seek par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory +par_replacement_rename ### Test --basenamereplace +par_replacement_rename b.c b.c b.c b b b +par_replacement_rename b.c +par_replacement_rename b.c +par_replacement_rename b.c +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename ### Test --bnr +par_replacement_rename b.c b.c b.c b b b +par_replacement_rename b.c +par_replacement_rename b.c +par_replacement_rename b.c +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename ### Test --extensionreplace +par_replacement_rename /a/b a/b b /a/b a/b b +par_replacement_rename /a/b +par_replacement_rename a/b +par_replacement_rename b +par_replacement_rename /a/b +par_replacement_rename a/b +par_replacement_rename b +par_replacement_rename ### Test --er +par_replacement_rename /a/b a/b b /a/b a/b b +par_replacement_rename /a/b +par_replacement_rename a/b +par_replacement_rename b +par_replacement_rename /a/b +par_replacement_rename a/b +par_replacement_rename b +par_replacement_rename ### Test --basenameextensionreplace +par_replacement_rename b b b b b b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename ### Test --bner +par_replacement_rename b b b b b b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_rename b +par_replacement_strings ### Test {/} +par_replacement_strings b.c b.c b.c b b b +par_replacement_strings ### Test {/.} +par_replacement_strings b b b b b b +par_replacement_strings ### Test {#/.} +par_replacement_strings number2 +par_replacement_strings ### Test {#/} +par_replacement_strings number2.c +par_replacement_strings ### Test {#.} +par_replacement_strings a/number2 par_result ### Test --results par_result I III par_result I IIII @@ -479,6 +735,42 @@ par_result_replace /tmp/par__49983-baz C par_result_replace /tmp/par__49983-baz C/seq par_result_replace /tmp/par__49983-baz C/stderr par_result_replace /tmp/par__49983-baz C/stdout +par_seqreplace_long_line ### Test --seqreplace and line too long +par_seqreplace_long_line 9 1 1 101 +par_seqreplace_long_line 90 1 1 201 +par_seqreplace_long_line 1 parallel: Error: Command line too long (309 >= 210) at input 0: 100 +par_test_E ### Test -E +par_test_E 1 2 3 +par_test_E 1 2 4 +par_test_E 1 3 3 +par_test_E 1 3 4 +par_test_E 1 4 3 +par_test_E 1 4 4 +par_test_E 2 2 3 +par_test_E 2 2 4 +par_test_E 2 3 3 +par_test_E 2 3 4 +par_test_E 2 4 3 +par_test_E 2 4 4 +par_test_E 3 2 3 +par_test_E 3 2 4 +par_test_E 3 3 3 +par_test_E 3 3 4 +par_test_E 3 4 3 +par_test_E 3 4 4 +par_test_E 4 2 3 +par_test_E 4 2 4 +par_test_E 4 3 3 +par_test_E 4 3 4 +par_test_E 4 4 3 +par_test_E 4 4 4 +par_test_E ### Test -E one empty +par_test_E 1 2 +par_test_E 2 2 +par_test_E ### Test -E 2 empty +par_test_E 1 +par_test_E 2 +par_test_E ### Test -E all empty par_test_XI_mI ### Test -I par_test_XI_mI 1 1 par_test_XI_mI 2 1 @@ -565,6 +857,34 @@ par_test_cpu_detection 2-24-48-24 2 24 48 24 par_test_cpu_detection 1-2-2-2 1 2 2 2 par_test_cpu_detection 1-8-8-8 1 8 8 8 par_test_cpu_detection 1-4-4-4 1 4 4 4 +par_test_gt_quoting ### Test of quoting of > bug +par_test_gt_quoting >/dev/null +par_test_gt_quoting ### Test of quoting of > bug if line continuation +par_test_gt_quoting > > > +par_test_job_number ### Test {#} +par_test_job_number 1 +par_test_job_number 2 +par_test_job_number 3 +par_test_job_number 4 +par_test_job_number 5 +par_test_job_number 6 +par_test_job_number 7 +par_test_job_number 8 +par_test_job_number 9 +par_test_job_number 10 par_too_long_line_X bug #54869: Long lines break par_too_long_line_X 1 29302 131012 par_too_long_line_X 1 12698 63490 +par_trailing_space_line_continuation ### Test of trailing space continuation +par_trailing_space_line_continuation foo ole bar +par_trailing_space_line_continuation quux +par_trailing_space_line_continuation foo ole bar +par_trailing_space_line_continuation quux +par_trailing_space_line_continuation foo ole bar +par_trailing_space_line_continuation quux +par_trailing_space_line_continuation ### Test of trailing space continuation with -E eof +par_trailing_space_line_continuation foo ole +par_trailing_space_line_continuation foo ole +par_trailing_space_line_continuation foo ole +par_trim_illegal_value ### Test of --trim illegal +par_trim_illegal_value parallel: Error: --trim must be one of: r l rl lr. diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index cd3f84e4..19516b3e 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -569,6 +569,8 @@ par_groupby --block 20 C1 cssv [\s,]+ 3,,2,,4,, par_groupby --block 20 C1 cssv [\s,]+ NewRec par_groupby --block 20 C1 cssv [\s,]+ a1, b1, C1, par_groupby --block 20 C1 cssv [\s,]+ 3 ,,, 2 ,,, 2 ,,, +par_groupby --block 20 C1 cssv [\s,]+ NewRec +par_groupby --block 20 C1 cssv [\s,]+ a1, b1, C1, par_groupby --block 20 C1 cssv [\s,]+ 3 2 3 par_groupby --block 20 C1 cssv [\s,]+ parallel: Warning: A record was longer than 20. Increasing to --blocksize 27. par_groupby --block 20 C1 cssv [\s,]+ parallel: Warning: A record was longer than 27. Increasing to --blocksize 37. @@ -1386,27 +1388,7 @@ par_tee_with_premature_close 999998 par_tee_with_premature_close 999999 par_tee_with_premature_close 1000000 par_tee_with_premature_close tee without --output-error=warn-nopipe support -par_tee_with_premature_close 1859 1860 8192 -par_tee_with_premature_close 1 -par_tee_with_premature_close 2 -par_tee_with_premature_close 3 -par_tee_with_premature_close 4 -par_tee_with_premature_close 5 -par_tee_with_premature_close 6 -par_tee_with_premature_close 7 -par_tee_with_premature_close 8 -par_tee_with_premature_close 9 -par_tee_with_premature_close 10 -par_tee_with_premature_close 1851 -par_tee_with_premature_close 1852 -par_tee_with_premature_close 1853 -par_tee_with_premature_close 1854 -par_tee_with_premature_close 1855 -par_tee_with_premature_close 1856 -par_tee_with_premature_close 1857 -par_tee_with_premature_close 1858 -par_tee_with_premature_close 1859 -par_tee_with_premature_close 1860 +par_tee_with_premature_close OK par_test_X_with_multiple_source ### Test {} multiple times in different commands par_test_X_with_multiple_source echo 1 2 3 4 5 6 7 8 9 10 ; echo 1 2 3 4 5 6 7 8 9 10 par_test_X_with_multiple_source 1 2 3 4 5 6 7 8 9 10 diff --git a/testsuite/wanted-results/parallel-local-race01 b/testsuite/wanted-results/parallel-local-race01 index 7adb96ae..d199a9e0 100644 --- a/testsuite/wanted-results/parallel-local-race01 +++ b/testsuite/wanted-results/parallel-local-race01 @@ -1,3 +1,10 @@ +par_bug56403 bug #56403: --pipe block by time. +par_bug56403 1 job1a +par_bug56403 2 job2b +par_bug56403 3 job3cjob3d +par_bug56403 1 job1a +par_bug56403 1 job1b +par_bug56403 2 job2cjob2d par_distribute_input_by_ability ### bug #48290: round-robin does not distribute data based on business par_distribute_input_by_ability ### Distribute input to jobs that are ready par_distribute_input_by_ability Job-slot n is 50% slower than n+1, so the order should be 1..7 diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 513170f9..27734413 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -1229,59 +1229,14 @@ par_ksh_environment_too_big OK_bigfunc_remote par_ksh_environment_too_big OK_bigfunc_quote par_ksh_environment_too_big OK_bigfunc_quote_remote par_ksh_environment_too_big Rest should fail -par_ksh_environment_too_big ksh[XXX]: perl: /usr/bin/perl: cannot execute [Argument list too long] -par_ksh_environment_too_big env_parallel: Error: Your environment is too big. -par_ksh_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_ksh_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_ksh_environment_too_big env_parallel: Error: variables or define functions. -par_ksh_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_ksh_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env -par_ksh_environment_too_big env_parallel: Error: And then use '--env _' -par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_remote -par_ksh_environment_too_big ksh[XXX]: perl: /usr/bin/perl: cannot execute [Argument list too long] -par_ksh_environment_too_big env_parallel: Error: Your environment is too big. -par_ksh_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_ksh_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_ksh_environment_too_big env_parallel: Error: variables or define functions. -par_ksh_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_ksh_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env -par_ksh_environment_too_big env_parallel: Error: And then use '--env _' -par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_quote par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_quote_remote -par_ksh_environment_too_big ksh[XXX]: perl: /usr/bin/perl: cannot execute [Argument list too long] -par_ksh_environment_too_big env_parallel: Error: Your environment is too big. -par_ksh_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_ksh_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_ksh_environment_too_big env_parallel: Error: variables or define functions. -par_ksh_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_ksh_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env -par_ksh_environment_too_big env_parallel: Error: And then use '--env _' -par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_remote -par_ksh_environment_too_big ksh[XXX]: perl: /usr/bin/perl: cannot execute [Argument list too long] -par_ksh_environment_too_big env_parallel: Error: Your environment is too big. -par_ksh_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_ksh_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_ksh_environment_too_big env_parallel: Error: variables or define functions. -par_ksh_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_ksh_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env -par_ksh_environment_too_big env_parallel: Error: And then use '--env _' -par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel -par_ksh_environment_too_big ksh[XXX]: perl: /usr/bin/perl: cannot execute [Argument list too long] -par_ksh_environment_too_big env_parallel: Error: Your environment is too big. -par_ksh_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_ksh_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_ksh_environment_too_big env_parallel: Error: variables or define functions. -par_ksh_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_ksh_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env -par_ksh_environment_too_big env_parallel: Error: And then use '--env _' -par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_quote +par_ksh_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_quote_remote par_ksh_env_parallel_session ### Test env_parallel --session par_ksh_env_parallel_session ### level0 should be hidden, level1 should be transferred par_ksh_env_parallel_session @@ -2087,59 +2042,14 @@ par_bash_environment_too_big OK_bigfunc_remote par_bash_environment_too_big OK_bigfunc_quote par_bash_environment_too_big OK_bigfunc_quote_remote par_bash_environment_too_big Rest should fail -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /usr/bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_remote -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /usr/bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_quote par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_quote_remote -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /usr/bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_remote -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /usr/bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /usr/bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_quote +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_quote_remote par_bash_env_parallel_session ### Test env_parallel --session / --end-session par_bash_env_parallel_session ### level0 should be hidden, level1 should be transferred par_bash_env_parallel_session diff --git a/testsuite/wanted-results/parallel-local-ssh8 b/testsuite/wanted-results/parallel-local-ssh8 index 1dabfbe0..5534180d 100644 --- a/testsuite/wanted-results/parallel-local-ssh8 +++ b/testsuite/wanted-results/parallel-local-ssh8 @@ -29,11 +29,11 @@ par_path_remote_csh ^^^^^^^^ Not found is OK par_path_remote_csh /bin:/usr/bin:/tmp OK par_path_remote_csh Done par_retries_1 ### Test of --retries - it should run 13 jobs in total -par_retries_1 13 +par_retries_1 OK par_retries_2 ### Test of --retries - it should run 25 jobs in total -par_retries_2 25 +par_retries_2 OK par_retries_4 ### Test of --retries - it should run 49 jobs in total -par_retries_4 49 +par_retries_4 OK par_retries_bug_from_2010 ### Bug with --retries par_retries_bug_from_2010 8 par_retries_bug_from_2010 8 diff --git a/testsuite/wanted-results/parallel-local1 b/testsuite/wanted-results/parallel-local1 index 381d3a5b..7000cc93 100644 --- a/testsuite/wanted-results/parallel-local1 +++ b/testsuite/wanted-results/parallel-local1 @@ -1,433 +1,2 @@ -echo "bug #43654: --bar with command not using {} - only last output line " -bug #43654: --bar with command not using {} - only last output line - COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' - 100% 1:0=0s 1  -echo "### Test --basenamereplace" -### Test --basenamereplace - parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b.c b.c b.c b b b - parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b.c -b.c -b.c -b -b -b -echo "### Test --bnr" -### Test --bnr - parallel -j1 -k -X --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b.c b.c b.c b b b - parallel -k --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b.c -b.c -b.c -b -b -b -echo "### Test --extensionreplace" -### Test --extensionreplace - parallel -j1 -k -X --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -/a/b a/b b /a/b a/b b - parallel -k --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -/a/b -a/b -b -/a/b -a/b -b -echo "### Test --er" -### Test --er - parallel -j1 -k -X --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -/a/b a/b b /a/b a/b b - parallel -k --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -/a/b -a/b -b -/a/b -a/b -b -echo "### Test --basenameextensionreplace" -### Test --basenameextensionreplace - parallel -j1 -k -X --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b b b b b b - parallel -k --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b -b -b -b -b -b -echo "### Test --bner" -### Test --bner - parallel -j1 -k -X --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b b b b b b - parallel -k --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b -b -b -b -b -b -b -echo "### Test {/}" -### Test {/} - parallel -j1 -k -X echo {/} ::: /a/b.c a/b.c b.c /a/b a/b b -b.c b.c b.c b b b -echo "### Test {/.}" -### Test {/.} - parallel -j1 -k -X echo {/.} ::: /a/b.c a/b.c b.c /a/b a/b b -b b b b b b -echo "### Test {#/.}" -### Test {#/.} - parallel -j1 -k -X echo {2/.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 -number2 -echo "### Test {#/}" -### Test {#/} - parallel -j1 -k -X echo {2/} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 -number2.c -echo "### Test {#.}" -### Test {#.} - parallel -j1 -k -X echo {2.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 -a/number2 -echo "### bug #34241: --pipe should not spawn unneeded processes" -### bug #34241: --pipe should not spawn unneeded processes - echo | parallel -r -j2 -N1 --pipe md5sum -c && echo OK -OK -echo '### Test of quoting of > bug' -### Test of quoting of > bug - echo '>/dev/null' | parallel echo ->/dev/null -echo '### Test of quoting of > bug if line continuation' -### Test of quoting of > bug if line continuation - (echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo -> > > -echo '### Test of --trim illegal' -### Test of --trim illegal - stdout parallel --trim fj ::: echo -parallel: Error: --trim must be one of: r l rl lr. -echo '### Test of eof string on :::' -### Test of eof string on ::: - parallel -k -E ole echo ::: foo ole bar -foo -echo '### Test of ignore-empty string on :::' -### Test of ignore-empty string on ::: - parallel -k -r echo ::: foo '' ole bar -foo -ole -bar -echo '### Test of trailing space continuation' -### Test of trailing space continuation - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo -foo ole bar -quux - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 echo -foo ole bar -quux - parallel -kr -L2 echo ::: foo '' 'ole ' bar quux -foo ole bar -quux -echo '### Test of trailing space continuation with -E eof' -### Test of trailing space continuation with -E eof - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 -E bar echo -foo ole - (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 -E bar echo -foo ole - parallel -kr -L2 -E bar echo ::: foo '' 'ole ' bar quux -foo ole -echo '### Test of --colsep' -### Test of --colsep - echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} -a b c - (echo 'a%c%b'; echo a%c%b%d) | parallel -k --colsep % echo {1} {3} {2} {4} -a b c -a b c d - (echo a%c%b; echo d%f%e) | parallel -k --colsep % echo {1} {3} {2} -a b c -d e f - parallel -k --colsep % echo {1} {3} {2} ::: a%c%b d%f%e -a b c -d e f - parallel -k --colsep % echo {1} {3} {2} ::: a%c%b -a b c - parallel -k --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d -a b c -a b c d -echo '### Test of tab as colsep' -### Test of tab as colsep - printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1} -abc def -ghi jkl - parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1} -abc def -ghi jkl -echo '### Test of multiple -a plus colsep' -### Test of multiple -a plus colsep - parallel --xapply -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1} -abc def -mno jkl -echo '### Test of multiple -a no colsep' -### Test of multiple -a no colsep - parallel --xapply -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1} -abc def ghi -jkl mno pqr -echo '### Test of quoting after colsplit' -### Test of quoting after colsplit - parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null' ->/tmp/null >/dev/null -echo '### Test of --colsep as regexp' -### Test of --colsep as regexp - (echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4} -a b c -a b c d - parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d -a b c -a b c d - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} -a b c -a b c d - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' -a_b_c_ -a_b_c_d -echo '### Test of -C' -### Test of -C - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' -a_b_c_ -a_b_c_d -echo '### Test of --trim n' -### Test of --trim n - (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' -a_b_' c '_ -a_' b '_c_d - parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' -a_b_c_ -a_b_c_d -echo '### Test of bug: If input is empty string' -### Test of bug: If input is empty string - (echo ; echo abcbdbebf;echo abc) | parallel -k --colsep b -v echo {1}{2} -echo '' - -echo ac -ac -echo ac -ac -echo '### Test :::: mixed with :::' -### Test :::: mixed with ::: -echo '### Test :::: < ::: :::' -### Test :::: < ::: ::: - parallel -k echo {1} {2} {3} :::: <(seq 6 7) ::: 4 5 ::: 1 2 3 -6 4 1 -6 4 2 -6 4 3 -6 5 1 -6 5 2 -6 5 3 -7 4 1 -7 4 2 -7 4 3 -7 5 1 -7 5 2 -7 5 3 -echo '### Test :::: < < :::: <' -### Test :::: < < :::: < - parallel -k echo {1} {2} {3} :::: <(seq 6 7) <(seq 4 5) :::: <(seq 1 3) -6 4 1 -6 4 2 -6 4 3 -6 5 1 -6 5 2 -6 5 3 -7 4 1 -7 4 2 -7 4 3 -7 5 1 -7 5 2 -7 5 3 -echo '### Test -a :::: < :::: <' -### Test -a :::: < :::: < - parallel -k -a <(seq 6 7) echo {1} {2} {3} :::: <(seq 4 5) :::: <(seq 1 3) -6 4 1 -6 4 2 -6 4 3 -6 5 1 -6 5 2 -6 5 3 -7 4 1 -7 4 2 -7 4 3 -7 5 1 -7 5 2 -7 5 3 -echo '### Test -a -a :::' -### Test -a -a ::: - parallel -k -a <(seq 6 7) -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 -6 4 1 -6 4 2 -6 4 3 -6 5 1 -6 5 2 -6 5 3 -7 4 1 -7 4 2 -7 4 3 -7 5 1 -7 5 2 -7 5 3 -echo '### Test -a - -a :::' -### Test -a - -a ::: - seq 6 7 | parallel -k -a - -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 -6 4 1 -6 4 2 -6 4 3 -6 5 1 -6 5 2 -6 5 3 -7 4 1 -7 4 2 -7 4 3 -7 5 1 -7 5 2 -7 5 3 -echo '### Test :::: < - :::' -### Test :::: < - ::: - seq 4 5 | parallel -k echo {1} {2} {3} :::: <(seq 6 7) - ::: 1 2 3 -6 4 1 -6 4 2 -6 4 3 -6 5 1 -6 5 2 -6 5 3 -7 4 1 -7 4 2 -7 4 3 -7 5 1 -7 5 2 -7 5 3 -echo '### Test -E' -### Test -E - seq 1 100 | parallel -k -E 5 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) -1 2 3 -1 2 4 -1 3 3 -1 3 4 -1 4 3 -1 4 4 -2 2 3 -2 2 4 -2 3 3 -2 3 4 -2 4 3 -2 4 4 -3 2 3 -3 2 4 -3 3 3 -3 3 4 -3 4 3 -3 4 4 -4 2 3 -4 2 4 -4 3 3 -4 3 4 -4 4 3 -4 4 4 -echo '### Test -E one empty' -### Test -E one empty - seq 1 100 | parallel -k -E 3 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) -1 2 -2 2 -echo '### Test -E 2 empty' -### Test -E 2 empty - seq 1 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) -1 -2 -echo '### Test -E all empty' -### Test -E all empty - seq 3 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) -echo '### Test {#}' -### Test {#} - seq 1 10 | parallel -k echo {#} -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -echo '### Test --seqreplace and line too long' -### Test --seqreplace and line too long - seq 1 1000 | stdout parallel -j1 -s 210 -k --seqreplace I echo IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII \|wc | uniq -c - 9 1 1 101 - 90 1 1 201 - 1 parallel: Error: Command line too long (309 >= 210) at input 0: 100 -echo '### bug #37042: -J foo is taken from the whole command line - not just the part before the command' -### bug #37042: -J foo is taken from the whole command line - not just the part before the command - echo '--tagstring foo' > ~/.parallel/bug_37042_profile; parallel -J bug_37042_profile echo ::: tag_with_foo; parallel --tagstring a -J bug_37042_profile echo ::: tag_with_a; parallel --tagstring a echo -J bug_37042_profile ::: print_-J_bug_37042_profile -foo tag_with_foo -a tag_with_a -a -J bug_37042_profile print_-J_bug_37042_profile -echo '### Bug introduce by fixing bug #37042' -### Bug introduce by fixing bug #37042 - parallel --xapply -a <(printf 'abc') --colsep '\t' echo {1} -abc -echo "### Test --header with -N" -### Test --header with -N - (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '.*\n.*\n' echo Start\;cat \; echo Stop -Start -h1 -h2 -1a -1b -Stop -Start -h1 -h2 -2a -2b -Stop -Start -h1 -h2 -3a -Stop -echo "### Test --header with --block 1k" -### Test --header with --block 1k - (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '.*\n.*\n' echo Start\;cat \; echo Stop -Start -h1 -h2 -1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -3xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -4xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -5xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -6xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -7xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -8xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -Stop -Start -h1 -h2 -9xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -10xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -11xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -12xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -13xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -14xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -15xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -16xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -Stop -Start -h1 -h2 -17xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -18xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -19xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -20xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -21xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -22xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -Stop -echo "### Test --header with multiple :::" -### Test --header with multiple ::: - parallel --header : echo {a} {b} {1} {2} ::: b b1 ::: a a2 -a2 b1 b1 a2 +echo moved to parallel-local-1s.sh +moved to parallel-local-1s.sh diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 3b580721..c2e72af2 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -39,7 +39,7 @@ https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice To silence this citation notice: run 'parallel --citation' once. 10 files to edit -[?2004h[?1049h[?1h=[?2004h[?12h[?12l[?25l"file1" [New File] [>c]10;?]11;?~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ 0,0-1All[?25h[?25lE173: 9 more files to edit[?2004h0,0-1All[?25h[?2004l[?2004l[?1l>[?1049lAcademic tradition requires you to cite works you base your article on. +[?2004h[?1049h[?1h=[?2004h[?12h[?12l[?25l"file1" [New File]~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ [?25h[?25lE173: 9 more files to edit[?2004h[?25h[?2004l[?2004l[?1l>[?1049lAcademic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 6710081f..a87f43ea 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -61,7 +61,7 @@ echo '### Check that 4 processes are really used' echo '### --version must have higher priority than retired options' ### --version must have higher priority than retired options $NICEPAR --version -g -Y -U -W -T | tail -n9 -Copyright (C) 2007-2019 Ole Tange and Free Software Foundation, Inc. +Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty.