diff --git a/doc/citation-notice-faq.txt b/doc/citation-notice-faq.txt index 3d0046fd..45ab00fb 100644 --- a/doc/citation-notice-faq.txt +++ b/doc/citation-notice-faq.txt @@ -121,7 +121,7 @@ contribution to the research is big enough to warrant a citation. The citation is also needed for reproducibility: Let us assume a bug in GNU Parallel skews the results. People replicating the research -needs to have the information, so they can replicate the (possibly +need to have the information, so they can replicate the (possibly wrong) results. diff --git a/doc/haikus b/doc/haikus index c275e13d..3815e9a3 100644 --- a/doc/haikus +++ b/doc/haikus @@ -4,6 +4,9 @@ Quote of the month: + parallel might be one of the best utilities out there + -- @ThePrimeagen ThePrimeagen + Recently learned how to use GNU parallel (from ChatGPT, no less!) and I've gone mad with power -- Mark, Anthropomorphic Anuran @reject_resubmit@twitter diff --git a/doc/release_new_version b/doc/release_new_version index 9fede752..e556a0b8 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -262,29 +262,25 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20230322 ('Arrest Warrant') released +Subject: GNU Parallel 20230422 ('Grand Jury<<>>') released -GNU Parallel 20230322 ('Arrest Warrant') has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20230422 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4 Quote of the month: - GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis - -- Best Catboy Key Grip @alamogordoglass@twitter + <<>> New in this release: -* Better support for wide characters in --latest-line. - -* Support for rsync 3.2.7. - +<<>> * Bug fixes and man page updates. News about GNU Parallel: -* Analyzing multi-gigabyte JSON files locally https://thenybble.de/posts/json-analysis/ - -* 5 great Perl scripts to keep in your sysadmin toolbox https://www.redhat.com/sysadmin/perl-scripts +https://bash-prompt.net/guides/parallell-bash/ +https://medium.com/linuxstories/bash-parallel-command-execution-d4bd7c7cc1d6 +<<>> GNU Parallel - For people who live life in the parallel lane. diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 49a23597..fe27f84e 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -393,7 +393,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 56d63e27..5eaa3060 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -395,7 +395,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 7063e17c..918cd89d 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -393,7 +393,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index f3195b29..51200422 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -376,7 +376,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index cb485d64..08e0d6f9 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -378,7 +378,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 2d6f262d..f1cd6637 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -393,7 +393,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index dede480a..f90385a0 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -368,7 +368,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20230322 (GNU parallel `parallel --minversion 1`)" + echo "parset 20230323 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index a25b20da..9aa4011e 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20230322; +$Global::version = 20230323; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 18cb1227..8eec6d73 100755 --- a/src/parallel +++ b/src/parallel @@ -1799,10 +1799,6 @@ sub options_completion_hash() { "[Use the replacement string replace-str instead of {%} for job ". "slot number]:replace-str" => \$opt::slotreplace), - ("jobs|j=s". - "[(Add +N to/Subtract -N from/Multiply N%) the number of CPU ". - "threads or read parameter from file]:_files" - => \$opt::jobs), ("delay=s". "[Delay starting next job by duration]:duration" => \$opt::delay), ("ssh-delay|sshdelay=f". @@ -1985,7 +1981,7 @@ sub options_completion_hash() { ("term-seq|termseq=s". "[Termination sequence]:sequence" => \$opt::termseq), # xargs-compatibility - implemented, man, testsuite - ("max-procs|maxprocs|P=s". + ("max-procs|maxprocs|P|jobs|j=s". "[Add N to/Subtract N from/Multiply N% with/ the number of CPU ". "threads or read parameter from file]:+N/-N/N%/N/procfile:_files" => \$opt::jobs), @@ -2766,7 +2762,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20230322; + $Global::version = 20230323; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -8245,28 +8241,7 @@ sub user_requested_processes($) { my $opt_P = shift; my $processes; if(defined $opt_P) { - if($opt_P =~ /^\+(\d+)$/) { - # E.g. -P +2 - my $j = $1; - $processes = - $self->ncpus() + $j; - } elsif ($opt_P =~ /^-(\d+)$/) { - # E.g. -P -2 - my $j = $1; - $processes = - $self->ncpus() - $j; - } elsif ($opt_P =~ /^(\d+(\.\d+)?)\%$/) { - # E.g. -P 10.5% - my $j = $1; - $processes = - $self->ncpus() * $j / 100; - } elsif ($opt_P =~ /^(\d+)$/) { - $processes = $1; - if($processes == 0) { - # -P 0 = infinity (or at least close) - $processes = $Global::infinity; - } - } elsif (-f $opt_P) { + if (-f $opt_P) { $Global::max_procs_file = $opt_P; if(open(my $in_fh, "<", $Global::max_procs_file)) { my $opt_P_file = join("",<$in_fh>); @@ -8277,8 +8252,15 @@ sub user_requested_processes($) { ::wait_and_exit(255); } } else { - ::error("Parsing of --jobs/-j/--max-procs/-P failed."); - ::die_usage(); + # -P +3 and -P -1 + $opt_P =~ s/^([-+])/\$self->ncpus()$1/; + # -P 40% + $opt_P =~ s:%$:*\$self->ncpus()/100:; + $processes = eval $opt_P; + if($processes == 0) { + # -P 0 = infinity (or at least close) + $processes = $Global::infinity; + } } $processes = ::ceil($processes); } @@ -11452,11 +11434,11 @@ sub print_files($) { if(::mbswidth($str) == length($str)) { $str = substr($str,0,$len); } else { - # Highly inefficient truncator + $str=~s/(.{$len}).*/$1/; + my $rlen = (::mbswidth($str) - $len)/2; + $str=~s/.{$rlen}$//; while(::mbswidth($str) > $len) { - do { - chop $str; - } while(::mbswidth($str) < 0); + $str=~s/.$//; } } return $str; @@ -11479,8 +11461,9 @@ sub print_files($) { } my ($color,$reset_color) = $self->color(); my $termcol = ::terminal_columns(); - my $untabify_tag = $self->untabtag(); - my $untabify_str = ::untabify($self->{$out_fh,'latestline'}); + my $untabify_tag = ::decode_utf8($self->untabtag()); + my $untabify_str = + ::untabify(::decode_utf8($self->{$out_fh,'latestline'})); # -1 to make space for $truncated_str my $maxtaglen = $termcol - 1; $untabify_tag = truncate_mbs($untabify_tag,$maxtaglen); @@ -11505,8 +11488,8 @@ sub print_files($) { "%s" # down ), "$up"x($currow - $row), "\n"x($row - $currow), "\r", $eol, - ::decode_utf8($untabify_tag),$truncated_tag, - $color, ::decode_utf8($untabify_str), $truncated_str, $reset_color, + $untabify_tag,$truncated_tag, + $color, $untabify_str, $truncated_str, $reset_color, "\n"x($maxrow - $row + 1)); $currow = $maxrow + 1; } @@ -15128,8 +15111,6 @@ sub main() { # Parallel check all hosts are up. Remove hosts that are down filter_hosts(); } - - if($opt::sqlmaster and $opt::sqlworker) { # Start a real --sqlworker in the background later $Global::start_sqlworker = 1; @@ -15184,7 +15165,6 @@ sub main() { # Compute $Global::max_jobs_running # Must be done after ungetting any --pipepart jobs. max_jobs_running(); - init_run_jobs(); my $sem; if($Global::semaphore) { diff --git a/src/parallel.pod b/src/parallel.pod index da245b6f..d31e4a62 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1375,19 +1375,27 @@ If the host is long, you can use B to pretty print it: See also: B<--resume> B<--resume-failed> -=item B<--jobs> I +=item B<--jobs> I (alpha testing) -=item B<-j> I +=item B<-j> I (alpha testing) -=item B<--max-procs> I +=item B<--max-procs> I (alpha testing) -=item B<-P> I +=item B<-P> I (alpha testing) Number of jobslots on each machine. -Run up to N jobs in parallel. 0 means as many as possible (this can -take a while to determine). Default is 100% which will run one job per -CPU thread on each machine. +Run up to I jobs in parallel. Default is 100%. + +=over 7 + +=item I + +Run up to I jobs in parallel. + +=item Z<>0 + +Run as many as possible (this can take a while to determine). Due to a bug B<-j 0> will also evaluate replacement strings twice up to the number of joblots: @@ -1397,81 +1405,49 @@ to the number of joblots: # This will count from 1 seq 10000 | parallel -j100 echo '{= $_ = $foo++; =}' | head -If B<--semaphore> is set, the default is 1 thus making a mutex. +=item I% -See also: B<--use-cores-instead-of-threads> -B<--use-sockets-instead-of-threads> +Multiply the number of CPU threads by I percent. E.g. 100% means +one job per CPU thread on each machine. +=item +I +Add I to the number of CPU threads. -=item B<--jobs> I<+N> +=item -I -=item B<-j> I<+N> +Subtract I from the number of CPU threads. -=item B<--max-procs> I<+N> +=item I -=item B<-P> I<+N> +Evaluate I. E.g. '12/2' to get 6, '+25%' gives the same as +'125%', or complex expressions like '+3*log(55)%' which means: +multiply 3 by log(55), multiply that by the number of CPU threads and +divide by 100, add this to the number of CPU threads. -Add N to the number of CPU threads. - -Run this many jobs in parallel. - -See also: B<--number-of-threads> B<--number-of-cores> -B<--number-of-sockets> - - -=item B<--jobs> I<-N> - -=item B<-j> I<-N> - -=item B<--max-procs> I<-N> - -=item B<-P> I<-N> - -Subtract N from the number of CPU threads. - -Run this many jobs in parallel. If the evaluated number is less than -1 then 1 will be used. - -See also: B<--number-of-threads> B<--number-of-cores> -B<--number-of-sockets> - - -=item B<--jobs> I% - -=item B<-j> I% - -=item B<--max-procs> I% - -=item B<-P> I% - -Multiply N% with the number of CPU threads. - -Run this many jobs in parallel. - -See also: B<--number-of-threads> B<--number-of-cores> -B<--number-of-sockets> - - -=item B<--jobs> I - -=item B<-j> I - -=item B<--max-procs> I - -=item B<-P> I +=item I Read parameter from file. Use the content of I as parameter for -I<-j>. E.g. I could contain the string 100% or +2 or 10. If -I is changed when a job completes, I is read again -and the new number of jobs is computed. If the number is lower than -before, running jobs will be allowed to finish but new jobs will not -be started until the wanted number of jobs has been reached. This +I<-j>. E.g. I could contain the string 100% or +2 or 10. + +If I is changed when a job completes, I is read +again and the new number of jobs is computed. If the number is lower +than before, running jobs will be allowed to finish but new jobs will +not be started until the wanted number of jobs has been reached. This makes it possible to change the number of simultaneous running jobs while GNU B is running. +=back + +If the evaluated number is less than 1 then 1 will be used. + +If B<--semaphore> is set, the default is 1 thus making a mutex. + +See also: B<--use-cores-instead-of-threads> +B<--use-sockets-instead-of-threads> + =item B<--keep-order> diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 8bcaa429..5fbc8acb 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -3917,6 +3917,10 @@ https://github.com/jkool702/forkrun =head2 Todo +https://www.npmjs.com/package/concurrently + +https://github.com/thilinaba/bash-parallel + http://code.google.com/p/push/ (cannot compile) https://github.com/krashanoff/parallel diff --git a/src/parallel_examples.pod b/src/parallel_examples.pod index a0307e2c..5f800154 100644 --- a/src/parallel_examples.pod +++ b/src/parallel_examples.pod @@ -161,7 +161,7 @@ line number and the URL. cat urlfile | parallel "wget {} 2>/dev/null || grep -n {} urlfile" -Create a mirror directory with the same filenames except all files and +Create a mirror directory with the same file names except all files and symlinks are empty files. cp -rs /the/source/dir mirror_dir @@ -288,6 +288,21 @@ Put all converted in the same directory: parallel lame {} -o mydir/{/.}.mp3 +=head2 EXAMPLE: Replacing parts of file names + +If you deal with paired end reads, you will have files like +barcode1_R1.fq.gz, barcode1_R2.fq.gz, barcode2_R1.fq.gz, and +barcode2_R2.fq.gz. + +You want barcodeI_R1 to be processed with barcodeI_R2. + + parallel --plus myprocess {} {/_R1.fq.gz/_R2.fq.gz} ::: *_R1.fq.gz + +If the barcode does not contain '_R1', you can do: + + parallel --plus myprocess {} {/_R1/_R2} ::: *_R1.fq.gz + + =head2 EXAMPLE: Removing strings from the argument If you have directory with tar.gz files and want these extracted in @@ -1719,7 +1734,7 @@ Here B: print "exclusively"; -=head2 EXAMPLE: Start editor with filenames from stdin (standard input) +=head2 EXAMPLE: Start editor with file names from stdin (standard input) You can use GNU B to start interactive programs like emacs or vi: diff --git a/src/parsort b/src/parsort index d308db40..14cbae8f 100755 --- a/src/parsort +++ b/src/parsort @@ -137,7 +137,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20230322; +$Global::version = 20230323; if($opt::version) { version(); exit 0; } # Remove -D and --parallel=N my @s = (grep { ! /^-D$|^--parallel=\S+$/ } diff --git a/src/sql b/src/sql index 7c406fee..9dfaa74b 100755 --- a/src/sql +++ b/src/sql @@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20230322; + $Global::version = 20230323; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index 4cf187eb..aa826c66 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -39,7 +39,7 @@ linebuffer_matters() { # Random data because it does not compress well # forcing the compress tool to spit out compressed blocks perl -pe 'y/[A-Za-z]//cd; $t++ % 1000 or print "\n"' < /dev/urandom | - head -c 10000000 > $randomfile + head -c 10000000 > "$randomfile" export randomfile testfunc() { @@ -49,10 +49,10 @@ linebuffer_matters() { # generate some incompressible ascii # with lines starting with the same string id=$1 - shuf $randomfile | perl -pe 's/^/'$id' /' + shuf "$randomfile" | perl -pe 's/^/'$id' /' # Sleep to give time to linebuffer-print the first part sleep 10 - shuf $randomfile | perl -pe 's/^/'$id' /' + shuf "$randomfile" | perl -pe 's/^/'$id' /' echo } export -f incompressible_ascii @@ -72,15 +72,15 @@ linebuffer_matters() { } # These can run in parallel if there are enough ressources - testfunc > $nolbfile - testfunc > $controlfile - testfunc --linebuffer > $lbfile + testfunc > "$nolbfile" + testfunc > "$controlfile" + testfunc --linebuffer > "$lbfile" wait - nolb="$(cat $nolbfile)" - control="$(cat $controlfile)" - lb="$(cat $lbfile)" - rm $nolbfile $lbfile $controlfile $randomfile + nolb="$(cat "$nolbfile")" + control="$(cat "$controlfile")" + lb="$(cat "$lbfile")" + rm "$nolbfile" "$lbfile" "$controlfile" "$randomfile" if [ "$nolb" == "$control" ] ; then if [ "$lb" == "$nolb" ] ; then diff --git a/testsuite/tests-to-run/parallel-local-300s.sh b/testsuite/tests-to-run/parallel-local-300s.sh index 3e7eef57..d8320642 100644 --- a/testsuite/tests-to-run/parallel-local-300s.sh +++ b/testsuite/tests-to-run/parallel-local-300s.sh @@ -18,38 +18,55 @@ par_compare_exit_codes() { echo '### compare the exit codes' echo 'directly from shells, shells called from parallel,' echo 'killed with different signals' - echo + # run: + # * exit $val + # * kill -$1 $$' + # both: + # * under different shells + # * from parallel running under different shells + # record: + # * exit value + # * signal echo sig=joblog_sig shell=parallel=joblog + # The quoting of \n is wrong for csh, so just use a sane TMPDIR here + TMPDIR=/tmp/par-300s + export TMPDIR + mkdir -p "$TMPDIR" selfkill=$(mktemp) - export selfkill - echo 'kill -$1 $$' >$selfkill + qselfkill=$(parallel -0 --shellquote ::: "$selfkill") + qqselfkill=$(parallel -0 --shellquote --shellquote ::: "$selfkill") + export selfkill qselfkill qqselfkill + echo 'kill -$1 $$' >"$selfkill" exit=$(mktemp) - export exit - echo 'exit $1' >$exit + qexit=$(parallel -0 --shellquote ::: "$exit") + qqexit=$(parallel -0 --shellquote --shellquote ::: "$exit") + export exit qexit qqexit + echo 'exit $1' >"$exit" doit() { shell=$1 sig=$2 sig128=$(( sig + 128 )) - sh -c "$shell $selfkill $sig" 2>/dev/null + sh -c "$shell $qselfkill $sig" 2>/dev/null raw=$? - sh -c "$shell $exit $sig128" 2>/dev/null + sh -c "$shell $qexit $sig128" 2>/dev/null raw128=$? log=$(mktemp) - $shell -c "parallel --halt now,done=1 --jl $log $shell $selfkill ::: $sig" 2>/dev/null + qlog=$(parallel -0 --shellquote ::: "$log") + qqlog=$(parallel -0 --shellquote --shellquote ::: "$log") + $shell -c "parallel --halt now,done=1 --jl $qlog $shell $qqselfkill ::: $sig" 2>/dev/null #echo parallel $shell $sig = $? parallel=$? - joblog_exit=$(field 7 < $log | tail -n1) - joblog_signal=$(field 8 < $log | tail -n1) - $shell -c "parallel --halt now,done=1 --jl $log $shell $exit ::: $sig128" 2>/dev/null + joblog_exit=$(field 7 < "$log" | tail -n1) + joblog_signal=$(field 8 < "$log" | tail -n1) + $shell -c "parallel --halt now,done=1 --jl $qlog $shell $qqexit ::: $sig128" 2>/dev/null parallel128=$? - joblog_exit128=$(field 7 < $log | tail -n1) - joblog_signal128=$(field 8 < $log | tail -n1) - - #echo joblog p $shell $sig $(field 8,7 < $log | tail -n1) - rm $log + joblog_exit128=$(field 7 < "$log" | tail -n1) + joblog_signal128=$(field 8 < "$log" | tail -n1) + + rm "$log" echo $shell sig' ' $sig=$joblog_signal $raw=$parallel=$joblog_exit echo $shell exit $sig128=$joblog_signal128 $raw128=$parallel128=$joblog_exit128 @@ -60,17 +77,17 @@ par_compare_exit_codes() { OK="ash csh dash fish fizsh posh rc sash sh tcsh" # These do not give the same exit code prepended with 'true;' or not BAD="bash ksh93 mksh static-sh yash zsh" - ( # Most block on signals: 19+20+21+22 ulimit -n `ulimit -Hn` parallel -j1000% -k doit ::: $OK $BAD ::: {1..18} {23..64} # fdsh blocks on a lot more signals parallel -j1000% -k doit ::: fdsh ::: 2 {9..12} {14..18} {20..23} 26 27 29 30 {32..64} - ) | - # Ignore where the exit codes are the same + ) | + # Ignore lines where the exit codes are the same perl -ne '/(\d+)=\1=\1/ or print' - rm $selfkill + rm "$selfkill" "$exit" + rmdir "$TMPDIR" } par_retries_unreachable() { diff --git a/testsuite/tests-to-run/parallel-manual.sh b/testsuite/tests-to-run/parallel-manual.sh index 5c53e330..1b2e4e65 100755 --- a/testsuite/tests-to-run/parallel-manual.sh +++ b/testsuite/tests-to-run/parallel-manual.sh @@ -10,14 +10,25 @@ par_ll_asian() { echo '### --ll with Asian wide chars mess up display' echo 'bug #63878: Wide East Asian chars in --latest-line' p="parallel --ll --color --tag" - COLUMNS=80 $p echo tag fits, line fits a{}b{}c \ - ::: ヌー平行 - COLUMNS=80 $p echo tag fits, line too long a{}b{}c \ - ::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー - COLUMNS=80 $p echo tag too long a{}b{}c \ + echo Oops: the first adds '>' too early + COLUMNS=50 $p echo tag fits, line fits a{}b{}c \ + ::: ヌー平 + COLUMNS=50 $p echo tag fits, line too long a{}b{}c \ + ::: ヌー平行ヌー平行ヌー平行ヌー平行ヌ + COLUMNS=50 $p echo tag too long a{}b{}c \ ::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行a } +par_mbswidth() { + echo '### characters with screen width > 1' + perl -e '@a=qw(ヌ ー 平 行.); + print map { + (join"",map{ $a[$_% $#a] } (1..$_))."\n". + "a".(join"",map{ $a[$_% $#a] } (1..$_))."\n" + } (1..40)' | + COLUMNS=50 parallel -k --ll --color --tag echo +} + par_ll_tag() { parallel --tag --ll -q printf "a\n{}\n" ::: should-be-tagged-A parallel --tag --ll -q printf "a\n\r{}\n" ::: should-be-tagged-B diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index fa3668cb..5f0ea5ba 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -83,6 +83,9 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' "$testsuit s/(std(out|err)|seq): Permission denied/$1: No such file or directory/; # Race condition s/^4-(middle|end)\n//; + # Race condition + s/^parallel: This job failed:\n//; + s/^echo .; exit .\n//; # Base 64 string with quotes s:['"'"'"\\+/a-z.0-9=]{50,}(\s['"'"'"\\+/a-z.0-9=]*)*:BASE64:ig; # Timings are often off diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 2a3c96b5..9f20e8b3 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -976,10 +976,10 @@ par_sem_quote ### sem --quote should not add empty argument par_sem_quote echo par_sem_quote par_shellcompletion ### --shellcompletion -par_shellcompletion 0097e98f494e6288b33f3201dda7ee11 - -par_shellcompletion 0097e98f494e6288b33f3201dda7ee11 - -par_shellcompletion 1655cb29ff14d0008866108c6b5262f7 - -par_shellcompletion 1655cb29ff14d0008866108c6b5262f7 - +par_shellcompletion 70960cbdbc411e041161ae228f029d70 - +par_shellcompletion 70960cbdbc411e041161ae228f029d70 - +par_shellcompletion aa125ab894780611a20bad4a52d7a58d - +par_shellcompletion aa125ab894780611a20bad4a52d7a58d - par_slow_pipe_regexp ### bug #53718: --pipe --regexp -N blocks par_slow_pipe_regexp This should take a few ms, but took more than 2 hours par_slow_pipe_regexp 0 1 1 diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 9c49f0f1..924e64f0 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -3,99 +3,115 @@ par_linebuffer_files bug #48658: --linebuffer --files par_linebuffer_files zstd normal par_linebuffer_files zstd 100000 par_linebuffer_files zstd --files -par_linebuffer_files zstd 1 +par_linebuffer_files zstd 4 par_linebuffer_files zstd --results par_linebuffer_files zstd 1 +par_linebuffer_files zstd parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files pzstd normal par_linebuffer_files pzstd 100000 par_linebuffer_files pzstd --files -par_linebuffer_files pzstd 1 +par_linebuffer_files pzstd 4 par_linebuffer_files pzstd --results par_linebuffer_files pzstd 1 +par_linebuffer_files pzstd parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files clzip normal par_linebuffer_files clzip 100000 par_linebuffer_files clzip --files -par_linebuffer_files clzip 1 +par_linebuffer_files clzip 4 par_linebuffer_files clzip --results par_linebuffer_files clzip 1 +par_linebuffer_files clzip parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files lz4 normal par_linebuffer_files lz4 100000 par_linebuffer_files lz4 --files -par_linebuffer_files lz4 1 +par_linebuffer_files lz4 4 par_linebuffer_files lz4 --results par_linebuffer_files lz4 1 +par_linebuffer_files lz4 parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files lzop normal par_linebuffer_files lzop 100000 par_linebuffer_files lzop --files -par_linebuffer_files lzop 1 +par_linebuffer_files lzop 4 par_linebuffer_files lzop --results par_linebuffer_files lzop 1 +par_linebuffer_files lzop parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files pigz normal par_linebuffer_files pigz 100000 par_linebuffer_files pigz --files -par_linebuffer_files pigz 1 +par_linebuffer_files pigz 4 par_linebuffer_files pigz --results par_linebuffer_files pigz 1 +par_linebuffer_files pigz parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files pxz normal par_linebuffer_files pxz 100000 par_linebuffer_files pxz --files -par_linebuffer_files pxz 1 +par_linebuffer_files pxz 4 par_linebuffer_files pxz --results par_linebuffer_files pxz 1 +par_linebuffer_files pxz parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files gzip normal par_linebuffer_files gzip 100000 par_linebuffer_files gzip --files -par_linebuffer_files gzip 1 +par_linebuffer_files gzip 4 par_linebuffer_files gzip --results par_linebuffer_files gzip 1 +par_linebuffer_files gzip parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files plzip normal par_linebuffer_files plzip 100000 par_linebuffer_files plzip --files -par_linebuffer_files plzip 1 +par_linebuffer_files plzip 4 par_linebuffer_files plzip --results par_linebuffer_files plzip 1 +par_linebuffer_files plzip parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files pbzip2 normal par_linebuffer_files pbzip2 100000 par_linebuffer_files pbzip2 --files -par_linebuffer_files pbzip2 1 +par_linebuffer_files pbzip2 4 par_linebuffer_files pbzip2 --results par_linebuffer_files pbzip2 1 +par_linebuffer_files pbzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files lzma normal par_linebuffer_files lzma 100000 par_linebuffer_files lzma --files -par_linebuffer_files lzma 1 +par_linebuffer_files lzma 4 par_linebuffer_files lzma --results par_linebuffer_files lzma 1 +par_linebuffer_files lzma parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files xz normal par_linebuffer_files xz 100000 par_linebuffer_files xz --files -par_linebuffer_files xz 1 +par_linebuffer_files xz 4 par_linebuffer_files xz --results par_linebuffer_files xz 1 +par_linebuffer_files xz parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files lzip normal par_linebuffer_files lzip 100000 par_linebuffer_files lzip --files -par_linebuffer_files lzip 1 +par_linebuffer_files lzip 4 par_linebuffer_files lzip --results par_linebuffer_files lzip 1 +par_linebuffer_files lzip parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files bzip2 normal par_linebuffer_files bzip2 100000 par_linebuffer_files bzip2 --files -par_linebuffer_files bzip2 1 +par_linebuffer_files bzip2 4 par_linebuffer_files bzip2 --results par_linebuffer_files bzip2 1 +par_linebuffer_files bzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files lbzip2 normal par_linebuffer_files lbzip2 100000 par_linebuffer_files lbzip2 --files -par_linebuffer_files lbzip2 1 +par_linebuffer_files lbzip2 4 par_linebuffer_files lbzip2 --results par_linebuffer_files lbzip2 1 +par_linebuffer_files lbzip2 parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_files lrz normal par_linebuffer_files lrz 100000 par_linebuffer_files lrz --files -par_linebuffer_files lrz 1 +par_linebuffer_files lrz 4 par_linebuffer_files lrz --results par_linebuffer_files lrz 1 +par_linebuffer_files lrz parallel: Warning: Use --files0 when $TMPDIR contains newline. par_linebuffer_matters_compress ### (--linebuffer) --compress should give different output par_linebuffer_matters_compress OK: --linebuffer makes a difference par_linebuffer_matters_compress_tag ### (--linebuffer) --compress --tag should give different output diff --git a/testsuite/wanted-results/parallel-local-300s b/testsuite/wanted-results/parallel-local-300s index 528b1d3c..4cb9fae9 100644 --- a/testsuite/wanted-results/parallel-local-300s +++ b/testsuite/wanted-results/parallel-local-300s @@ -318,7 +318,7 @@ par_test_build_and_install make distdir-am par_test_build_and_install make[0]: Entering directory '~/privat/parallel/src' par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src' par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src' -par_test_build_and_install test -n "" \ +par_test_build_and_install test -n " \ par_test_build_and_install || find "parallel-00000000" -type d ! -perm -755 \ par_test_build_and_install -exec chmod u+rwx,go+rx {} \; -o \ par_test_build_and_install ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \ diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index bbd1cda7..0ea19bcf 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -606,29 +606,16 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 3 : TIMESTAMP 9.999 0 0 3 0 exit 3 9 -parallel: This job failed: -echo X; exit X parallel: Starting no more jobs. Waiting for 1 jobs to finish. -parallel: This job failed: -echo X; exit X parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3 9 -parallel: This job failed: -echo X; exit X parallel -j2 --halt soon,fail=20% echo {}\; exit {} \ ::: 0 1 2 3 4 5 6 7 8 9 9 -parallel: This job failed: -echo X; exit X -parallel: This job failed: -echo X; exit X parallel: Starting no more jobs. Waiting for 1 jobs to finish. -parallel: This job failed: -echo X; exit X parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6 9 parallel: This job succeeded: -echo X; exit X parallel -k --retries 3 \ 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0 cat /tmp/runs