parallel: --pipe --csv: Pass only full CSV-records.

README

@@ -44,9 +44,9 @@ document.
 
 Full installation of GNU Parallel is as simple as:
 
-    wget https://ftpmirror.gnu.org/parallel/parallel-20180322.tar.bz2
+    wget https://ftpmirror.gnu.org/parallel/parallel-20180422.tar.bz2
-    bzip2 -dc parallel-20180322.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20180422.tar.bz2 | tar xvf -
-    cd parallel-20180322
+    cd parallel-20180422
     ./configure && make && sudo make install
 
 
@@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as:
 If you are not root you can add ~/bin to your path and install in
 ~/bin and ~/share:
 
-    wget https://ftpmirror.gnu.org/parallel/parallel-20180322.tar.bz2
+    wget https://ftpmirror.gnu.org/parallel/parallel-20180422.tar.bz2
-    bzip2 -dc parallel-20180322.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20180422.tar.bz2 | tar xvf -
-    cd parallel-20180322
+    cd parallel-20180422
     ./configure --prefix=$HOME && make && make install
 
 Or if your system lacks 'make' you can simply copy src/parallel

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20180322], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20180422], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/release_new_version

@@ -207,18 +207,29 @@ GNU Parallel 20180422 ('Trèbes') <<[stable]>> has been released. It is availabl
 
 Quote of the month:
 
-<<>>
+Today I discovered GNU Parallel, and I don’t know what to do with all this spare time.
-  -- 
+  --Ryan Booker
 
 New in this release:
 
-Launch at DIKU
+* --csv makes GNU Parallel parse the input sources as CSV. When used with --pipe it only passes full CSV-records.
 
-https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently
+* Time in --bar is printed as 1d02h03m04s.
 
-http://fliplinux.com/gnu-parallel-6.html
+* Optimization of --tee: It spawns a process less per value.
+
+* The GNU Parallel 2018 book is now available: http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
+
+* Modern pentest tricks for faster, wider, greater engagement (15) https://conference.hitb.org/hitbsecconf2018ams/materials/D1%20COMMSEC%20-%20Thomas%20Debize%20-%20Modern%20Pentest%20Tricks%20for%20Faster,%20Wider,%20Greater%20Engagements.pdf
+
+* Running many serial jobs efficiently https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently
+
+* GNU Parallel: как сохранить результаты нескольких команд для переменной? http://fliplinux.com/gnu-parallel-6.html
+
+* Running Multiple Commands in Parallel on a GNU Linux https://www.youtube.com/watch?v=sHpTywpb4_4
+
+* Klaatu covers the documentation of GNU parallel in episode 12x15 http://gnuworldorder.info/
 
-https://www.youtube.com/watch?v=sHpTywpb4_4
 
 <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
 

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20180323;
+$Global::version = 20180422;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -392,7 +392,8 @@ sub find_split_positions {
 		    }
 		} else {
 		    # If match $recend$recstart => Record position
-		    # TODO optimize to only look at the appended $dd_block_size + len $recendrecstart
+		    # TODO optimize to only look at the appended
+		    #   $dd_block_size + len $recendrecstart
 		    # TODO increase $dd_block_size to optimize for longer records
 		    my $i = index64(\$buf,$recendrecstart);
 		    if($i != -1) {
@@ -434,7 +435,7 @@ sub cat_partial {
 		 sysseek(STDIN,shift,0) || die;
 		 $left = shift;
 		 while($read =
-		       sysread(STDIN,$buf, $left > 1048524 ? 1048524 : $left)){
+		       sysread(STDIN,$buf, $left > 131072 ? 131072 : $left)){
 		     $left -= $read;
 		     syswrite(STDOUT,$buf);
 		 }
@@ -500,6 +501,7 @@ sub spreadstdin {
 	  # Read n-line records
 	  my $n_lines = $buf =~ tr/\n/\n/;
 	  my $last_newline_pos = rindex64(\$buf,"\n");
+	  # Go backwards until there are full n-line records
 	  while($n_lines % $Global::max_lines) {
 	      $n_lines--;
 	      $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1);
@@ -533,6 +535,21 @@ sub spreadstdin {
 					   $recstart,$recend,length $1);
 	      }
 	  }
+      } elsif($opt::csv) {
+	  # Read a full CSV record
+	  # even number of " + end of line
+	  my $last_newline_pos = length $buf;
+	  do {
+	      # find last EOL
+	      $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1);
+	      # While uneven "
+	  } while((substr($buf,0,$last_newline_pos) =~ y/"/"/)%2
+	      and $last_newline_pos >= 0);
+	  # Chop at $last_newline_pos as that is where CSV record ends
+	  $anything_written +=
+	      write_record_to_pipe($chunk_number++,\$header,\$buf,
+				   $recstart,$recend,$last_newline_pos+1);
+	  shorten(\$buf,$last_newline_pos+1);
       } else {
 	  if($Global::max_number_of_args) {
 	      # -N => (start..*?end){n}
@@ -1427,7 +1444,7 @@ sub check_invalid_option_combinations {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20180323;
+    $Global::version = 20180422;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -10817,7 +10834,7 @@ sub read_arg_from_fh {
     my $fh = shift;
     my $prepend;
     my $arg;
-    my $double_quotes = 0;
+    my $half_record = 0;
     do {{
 	# This makes 10% faster
 	if(not defined ($arg = <$fh>)) {
@@ -10829,16 +10846,20 @@ sub read_arg_from_fh {
 	}
 	if($opt::csv) {
 	    # We need to read a full CSV line.
-	    $double_quotes += ($arg =~ y/"/"/);
+	    if(($arg =~ y/"/"/) % 2 ) {
-	    if($double_quotes % 2) {
+		# The number of " on the line is uneven:
-		# CSV halflines with quoting:
+		# If we were in a half_record => we have a full record now
+		# If we were ouside a half_record => we are in a half record now
+		$half_record = not $half_record;
+	    }
+	    if($half_record) {
+		# CSV half-record with quoting:
 		#   col1,"col2 2""x3"" board newline  <-this one
 		#   cont",col3
 		$prepend .= $arg;
 		redo;
 	    } else {
-		# Now we have a full CSV line
+		# Now we have a full CSV record
-		$double_quotes = 0;
 	    }
 	}
 	# Remove delimiter

src/parallel.pod

@@ -607,6 +607,8 @@ Even quoted newlines are parsed correctly:
     echo 'Line 2 in field 1";value 2') |
      parallel --csv --colsep ';' echo Field 1: {1} Field 2: {2}
 
+When used with B<--pipe> only pass full CSV-records.
+
 =item B<--delimiter> I<delim>
 
 =item B<-d> I<delim>

src/sql

@@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20180323;
+    $Global::version = 20180422;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -521,23 +521,6 @@ par_pipe_record_size_in_lines() {
     seq 10 | parallel -k --pipe -l 4 cat\;echo bug 34958-2
 }
 
-par_pipe_compress_blocks() {
-    echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
-    seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat
-    echo echo 1-4 + 1-4
-    seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo
-    echo 4 times wc to stderr to stdout
-    (seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
-    echo 1 2 3 4
-    seq 4 | parallel -k -j3 --compress echo
-    echo 1 2 3 4
-    seq 4 | parallel -k -j1 --compress echo
-    echo 1 2
-    seq 2 | parallel -k -j1 --compress echo
-    echo 1 2 3
-    seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat
-}
-
 par_pipe_no_command() {
     echo '### --pipe without command'
 
@@ -570,18 +553,6 @@ par_tricolonplus() {
     parallel -k echo :::: <(seq 3) :::: <(seq 21 23) :::+ a b c ::: aa bb cc
 }
 
-par_header_parens() {
-    echo 'bug #49538: --header and {= =}'
-
-    parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
-    parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
-    parallel --header : echo {var/.} ::: var sub/dir/file.ext
-    parallel --header : echo {var//} ::: var sub/dir/file.ext
-    parallel --header : echo {var/.} ::: var sub/dir/file.ext
-    parallel --header : echo {var/} ::: var sub/dir/file.ext
-    parallel --header : echo {var.} ::: var sub/dir/file.ext
-}
-
 par_colsep_0() {
     echo 'bug --colsep 0'
 
@@ -656,21 +627,6 @@ par_link_files_as_only_arg() {
     parallel -k echo ::::+ <(seq 10) <(seq 3) <(seq 4)
 }
 
-par_macron() {
-    print_it() {
-	parallel ::: "echo $1"
-	parallel echo ::: "$1"
-	parallel echo "$1" ::: "$1"
-	parallel echo \""$1"\" ::: "$1"
-	parallel -q echo ::: "$1"
-	parallel -q echo "$1" ::: "$1"
-	parallel -q echo \""$1"\" ::: "$1"
-    }
-    print_it "$(perl -e 'print "\257"')"
-    print_it "$(perl -e 'print "\257\256"')"
-    print_it "$(perl -e 'print "\257<\257<\257>\257>"')"
-}
-
 par_basic_halt() {
     cpuburn=$(tempfile)
     cpuburn2=$(tempfile)
@@ -747,51 +703,6 @@ par_linebuffer_files() {
     stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
 }
 
-par_parset() {
-    . `which env_parallel.bash`
-    echo '### parset into array'
-    parset arr1 echo ::: foo bar baz
-    echo ${arr1[0]} ${arr1[1]} ${arr1[2]}
-
-    echo '### parset into vars with comma'
-    parset comma3,comma2,comma1 echo ::: baz bar foo
-    echo $comma1 $comma2 $comma3
-
-    echo '### parset into vars with space'
-    parset 'space3 space2 space1' echo ::: baz bar foo
-    echo $space1 $space2 $space3
-
-    echo '### parset with newlines'
-    parset 'newline3 newline2 newline1' seq ::: 3 2 1
-    echo "$newline1"
-    echo "$newline2"
-    echo "$newline3"
-
-    echo '### parset into indexed array vars'
-    parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo
-    echo ${myarray[*]}
-    echo ${myarray[4]} ${myarray[5]} ${myarray[5]}
-
-    echo '### env_parset'
-    alias myecho='echo myecho "$myvar" "${myarr[1]}"'
-    myvar="myvar"
-    myarr=("myarr  0" "myarr  1" "myarr  2")
-    mynewline="`echo newline1;echo newline2;`"
-    env_parset arr1 myecho ::: foo bar baz
-    echo "${arr1[0]} ${arr1[1]} ${arr1[2]}"
-    env_parset comma3,comma2,comma1 myecho ::: baz bar foo
-    echo "$comma1 $comma2 $comma3"
-    env_parset 'space3 space2 space1' myecho ::: baz bar foo
-    echo "$space1 $space2 $space3"
-    env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1
-    echo "$newline1"
-    echo "$newline2"
-    echo "$newline3"
-    env_parset 'myarray[6],myarray[5],myarray[4]' myecho ::: baz bar foo
-    echo "${myarray[*]}"
-    echo "${myarray[4]} ${myarray[5]} ${myarray[5]}"
-}
-
 par_halt_one_job() {
     echo '# Halt soon if there is a single job'
     echo should run 0 1 = job 1 2
@@ -823,13 +734,6 @@ par_pipepart_recend_recstart() {
     rm $tmp1 2>/dev/null
 }
 
-par_parset_v() {
-    echo 'bug #52507: parset arr1 -v echo ::: fails'
-    . `which env_parallel.bash`
-    parset arr1 -v seq ::: 1 2 3
-    echo "${arr1[2]}"
-}
-
 par_pipe_tag_v() {
     echo 'pipe with --tag -v'
     seq 3 | parallel -v --pipe --tagstring foo cat
@@ -861,6 +765,23 @@ par_csv() {
 	parallel --csv echo {1}-{2}-{3}
 }
 
+par_csv_pipe() {
+    echo 'Only pass full records to tail'
+    echo 'Too small block size'
+    perl -e 'for $b(1..10) {
+           print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000;
+           print "\n"
+         }' |
+	stdout parallel --pipe --csv -k --block 10k tail -n1
+
+    echo 'More records in single block'
+    perl -e 'for $b(1..10) {
+           print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000;
+           print "\n"
+         }' |
+	stdout parallel --pipe --csv -k --block 100k tail -n1
+}
+    
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |
     parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

testsuite/tests-to-run/parallel-local-1s.sh

@@ -237,6 +237,99 @@ _EOF
     ps aux | grep parallel[-]-lb-test
 }
 
+par_macron() {
+    print_it() {
+	parallel ::: "echo $1"
+	parallel echo ::: "$1"
+	parallel echo "$1" ::: "$1"
+	parallel echo \""$1"\" ::: "$1"
+	parallel -q echo ::: "$1"
+	parallel -q echo "$1" ::: "$1"
+	parallel -q echo \""$1"\" ::: "$1"
+    }
+    print_it "$(perl -e 'print "\257"')"
+    print_it "$(perl -e 'print "\257\256"')"
+    print_it "$(perl -e 'print "\257<\257<\257>\257>"')"
+}
+
+par_header_parens() {
+    echo 'bug #49538: --header and {= =}'
+
+    parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
+    parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
+    parallel --header : echo {var/.} ::: var sub/dir/file.ext
+    parallel --header : echo {var//} ::: var sub/dir/file.ext
+    parallel --header : echo {var/.} ::: var sub/dir/file.ext
+    parallel --header : echo {var/} ::: var sub/dir/file.ext
+    parallel --header : echo {var.} ::: var sub/dir/file.ext
+}
+
+par_parset2() {
+    . `which env_parallel.bash`
+    echo '### parset into array'
+    parset arr1 echo ::: foo bar baz
+    echo ${arr1[0]} ${arr1[1]} ${arr1[2]}
+
+    echo '### parset into vars with comma'
+    parset comma3,comma2,comma1 echo ::: baz bar foo
+    echo $comma1 $comma2 $comma3
+
+    echo '### parset into vars with space'
+    parset 'space3 space2 space1' echo ::: baz bar foo
+    echo $space1 $space2 $space3
+
+    echo '### parset with newlines'
+    parset 'newline3 newline2 newline1' seq ::: 3 2 1
+    echo "$newline1"
+    echo "$newline2"
+    echo "$newline3"
+
+    echo '### parset into indexed array vars'
+    parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo
+    echo ${myarray[*]}
+    echo ${myarray[4]} ${myarray[5]} ${myarray[5]}
+
+    echo '### env_parset'
+    alias myecho='echo myecho "$myvar" "${myarr[1]}"'
+    myvar="myvar"
+    myarr=("myarr  0" "myarr  1" "myarr  2")
+    mynewline="`echo newline1;echo newline2;`"
+    env_parset arr1 myecho ::: foo bar baz
+    echo "${arr1[0]} ${arr1[1]} ${arr1[2]}"
+    env_parset comma3,comma2,comma1 myecho ::: baz bar foo
+    echo "$comma1 $comma2 $comma3"
+    env_parset 'space3 space2 space1' myecho ::: baz bar foo
+    echo "$space1 $space2 $space3"
+    env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1
+    echo "$newline1"
+    echo "$newline2"
+    echo "$newline3"
+    env_parset 'myarray[6],myarray[5],myarray[4]' myecho ::: baz bar foo
+    echo "${myarray[*]}"
+    echo "${myarray[4]} ${myarray[5]} ${myarray[5]}"
+
+    echo 'bug #52507: parset arr1 -v echo ::: fails'
+    parset arr1 -v seq ::: 1 2 3
+    echo "${arr1[2]}"
+}
+
+par_pipe_compress_blocks() {
+    echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
+    seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat
+    echo echo 1-4 + 1-4
+    seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo
+    echo 4 times wc to stderr to stdout
+    (seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
+    echo 1 2 3 4
+    seq 4 | parallel -k -j3 --compress echo
+    echo 1 2 3 4
+    seq 4 | parallel -k -j1 --compress echo
+    echo 1 2
+    seq 2 | parallel -k -j1 --compress echo
+    echo 1 2 3
+    seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat
+}
+
 
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |

testsuite/wanted-results/parallel-local-0.3s

@@ -1366,6 +1366,23 @@ par_csv	col1"x3"-new
 par_csv	line col2-new2
 par_csv	line col3-col 4
 par_csv	2"x3" board-Value with ,-Column 3
+par_csv_pipe	Only pass full records to tail
+par_csv_pipe	Too small block size
+par_csv_pipe	parallel: Warning: A record was longer than 10000. Increasing to --blocksize 13001.
+par_csv_pipe	2000"
+par_csv_pipe	3000"
+par_csv_pipe	4000"
+par_csv_pipe	5000"
+par_csv_pipe	6000"
+par_csv_pipe	7000"
+par_csv_pipe	8000"
+par_csv_pipe	9000"
+par_csv_pipe	parallel: Warning: A record was longer than 13001. Increasing to --blocksize 16903.
+par_csv_pipe	10000"
+par_csv_pipe	11000"
+par_csv_pipe	More records in single block
+par_csv_pipe	9000"
+par_csv_pipe	11000"
 par_dryrun_append_joblog	--dry-run should not append to joblog
 par_dryrun_append_joblog	1
 par_dryrun_append_joblog	2
@@ -1408,14 +1425,6 @@ par_halt_one_job	1
 par_halt_one_job	parallel: This job failed:
 par_halt_one_job	echo 1;exit 1
 par_halt_one_job	parallel: Starting no more jobs. Waiting for 0 jobs to finish.
-par_header_parens	bug #49538: --header and {= =}
-par_header_parens	OK
-par_header_parens	OK
-par_header_parens	file
-par_header_parens	sub/dir
-par_header_parens	file
-par_header_parens	file.ext
-par_header_parens	sub/dir/file
 par_inefficient_L	bug #37325: Inefficiency of --pipe -L
 par_inefficient_L	    276     276     996
 par_inefficient_L	FOO
@@ -1441,28 +1450,6 @@ par_link_files_as_only_arg	bug #50685: single ::::+ does not work
 par_link_files_as_only_arg	1 1 1
 par_link_files_as_only_arg	2 2 2
 par_link_files_as_only_arg	3 3 3
-par_macron	¯
-par_macron	¯
-par_macron	¯ ¯
-par_macron	¯ ¯
-par_macron	¯
-par_macron	¯ ¯
-par_macron	"¯" ¯
-par_macron	¯®
-par_macron	¯®
-par_macron	¯® ¯®
-par_macron	¯® ¯®
-par_macron	¯®
-par_macron	¯® ¯
-par_macron	"¯®" ¯
-par_macron	/bin/bash: -c: line 0: syntax error near unexpected token `newline'
-par_macron	/bin/bash: -c: line 0: `echo ¯<¯<¯>¯>'
-par_macron	¯<¯<¯>¯>
-par_macron	/bin/bash: ¯: No such file or directory
-par_macron	¯<¯<¯>¯> ¯<¯<¯>¯>
-par_macron	¯<¯<¯>¯>
-par_macron	¯<¯<¯>¯> ¯<¯<¯>¯>
-par_macron	"¯<¯<¯>¯>" ¯<¯<¯>¯>
 par_newline_in_command	Command with newline and positional replacement strings
 par_newline_in_command	O K
 par_no_command_given	### Test bugfix if no command given
@@ -1475,78 +1462,6 @@ par_parcat_args_stdin	OK2
 par_parcat_rm	bug #51691: parcat --rm remove fifo when opened
 par_parcat_rm	OK1
 par_parcat_rm	OK file removed
-par_parset	### parset into array
-par_parset	foo bar baz
-par_parset	### parset into vars with comma
-par_parset	foo bar baz
-par_parset	### parset into vars with space
-par_parset	foo bar baz
-par_parset	### parset with newlines
-par_parset	1
-par_parset	1
-par_parset	2
-par_parset	1
-par_parset	2
-par_parset	3
-par_parset	### parset into indexed array vars
-par_parset	foo bar baz
-par_parset	foo bar bar
-par_parset	### env_parset
-par_parset	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
-par_parset	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
-par_parset	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
-par_parset	newline1
-par_parset	newline2
-par_parset	1
-par_parset	newline1
-par_parset	newline2
-par_parset	1
-par_parset	2
-par_parset	newline1
-par_parset	newline2
-par_parset	1
-par_parset	2
-par_parset	3
-par_parset	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
-par_parset	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 bar
-par_parset_v	bug #52507: parset arr1 -v echo ::: fails
-par_parset_v	seq 3
-par_parset_v	1
-par_parset_v	2
-par_parset_v	3
-par_pipe_compress_blocks	### bug #41482: --pipe --compress blocks at different -j/seq combinations
-par_pipe_compress_blocks	1
-par_pipe_compress_blocks	echo 1-4 + 1-4
-par_pipe_compress_blocks	echo 1
-par_pipe_compress_blocks	1
-par_pipe_compress_blocks	echo 2
-par_pipe_compress_blocks	2
-par_pipe_compress_blocks	echo 3
-par_pipe_compress_blocks	3
-par_pipe_compress_blocks	echo 4
-par_pipe_compress_blocks	4
-par_pipe_compress_blocks	4 times wc to stderr to stdout
-par_pipe_compress_blocks	      1       1       2
-par_pipe_compress_blocks	      1       1       2
-par_pipe_compress_blocks	      1       1       2
-par_pipe_compress_blocks	      1       1       2
-par_pipe_compress_blocks	1 2 3 4
-par_pipe_compress_blocks	1
-par_pipe_compress_blocks	2
-par_pipe_compress_blocks	3
-par_pipe_compress_blocks	4
-par_pipe_compress_blocks	1 2 3 4
-par_pipe_compress_blocks	1
-par_pipe_compress_blocks	2
-par_pipe_compress_blocks	3
-par_pipe_compress_blocks	4
-par_pipe_compress_blocks	1 2
-par_pipe_compress_blocks	1
-par_pipe_compress_blocks	2
-par_pipe_compress_blocks	1 2 3
-par_pipe_compress_blocks	1
-par_pipe_compress_blocks	2
-par_pipe_compress_blocks	3
 par_pipe_no_command	### --pipe without command
 par_pipe_no_command	parallel: Error: --pipe/--pipepart must have a command to pipe into (e.g. 'cat').
 par_pipe_record_size_in_lines	bug #34958: --pipe with record size measured in lines

testsuite/wanted-results/parallel-local-1s

@@ -153,6 +153,14 @@ par_fifo_under_csh	1
 par_fifo_under_csh	868832
 par_fifo_under_csh	1
 par_fifo_under_csh	exit 22
+par_header_parens	bug #49538: --header and {= =}
+par_header_parens	OK
+par_header_parens	OK
+par_header_parens	file
+par_header_parens	sub/dir
+par_header_parens	file
+par_header_parens	file.ext
+par_header_parens	sub/dir/file
 par_incomplete_linebuffer	bug #51337: --lb does not kill jobs at sigpipe
 par_incomplete_linebuffer	1
 par_incomplete_linebuffer	2
@@ -261,6 +269,28 @@ par_linebuffer_files	lrz	--files
 par_linebuffer_files	lrz	1
 par_linebuffer_files	lrz	--results
 par_linebuffer_files	lrz	1
+par_macron	¯
+par_macron	¯
+par_macron	¯ ¯
+par_macron	¯ ¯
+par_macron	¯
+par_macron	¯ ¯
+par_macron	"¯" ¯
+par_macron	¯®
+par_macron	¯®
+par_macron	¯® ¯®
+par_macron	¯® ¯®
+par_macron	¯®
+par_macron	¯® ¯
+par_macron	"¯®" ¯
+par_macron	/bin/bash: -c: line 0: syntax error near unexpected token `newline'
+par_macron	/bin/bash: -c: line 0: `echo ¯<¯<¯>¯>'
+par_macron	¯<¯<¯>¯>
+par_macron	/bin/bash: ¯: No such file or directory
+par_macron	¯<¯<¯>¯> ¯<¯<¯>¯>
+par_macron	¯<¯<¯>¯>
+par_macron	¯<¯<¯>¯> ¯<¯<¯>¯>
+par_macron	"¯<¯<¯>¯>" ¯<¯<¯>¯>
 par_max_length_len_128k	### BUG: The length for -X is not close to max (131072)
 par_max_length_len_128k	      1   12817  131016
 par_max_length_len_128k	      1   10946  131032
@@ -322,6 +352,78 @@ par_parset	9
 par_parset	Commands with newline require -0
 par_parset	line1
 par_parset	line2
+par_parset2	### parset into array
+par_parset2	foo bar baz
+par_parset2	### parset into vars with comma
+par_parset2	foo bar baz
+par_parset2	### parset into vars with space
+par_parset2	foo bar baz
+par_parset2	### parset with newlines
+par_parset2	1
+par_parset2	1
+par_parset2	2
+par_parset2	1
+par_parset2	2
+par_parset2	3
+par_parset2	### parset into indexed array vars
+par_parset2	foo bar baz
+par_parset2	foo bar bar
+par_parset2	### env_parset
+par_parset2	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
+par_parset2	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
+par_parset2	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
+par_parset2	newline1
+par_parset2	newline2
+par_parset2	1
+par_parset2	newline1
+par_parset2	newline2
+par_parset2	1
+par_parset2	2
+par_parset2	newline1
+par_parset2	newline2
+par_parset2	1
+par_parset2	2
+par_parset2	3
+par_parset2	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 baz
+par_parset2	myecho myvar myarr  1 foo myecho myvar myarr  1 bar myecho myvar myarr  1 bar
+par_parset2	bug #52507: parset arr1 -v echo ::: fails
+par_parset2	seq 3
+par_parset2	1
+par_parset2	2
+par_parset2	3
+par_pipe_compress_blocks	### bug #41482: --pipe --compress blocks at different -j/seq combinations
+par_pipe_compress_blocks	1
+par_pipe_compress_blocks	echo 1-4 + 1-4
+par_pipe_compress_blocks	echo 1
+par_pipe_compress_blocks	1
+par_pipe_compress_blocks	echo 2
+par_pipe_compress_blocks	2
+par_pipe_compress_blocks	echo 3
+par_pipe_compress_blocks	3
+par_pipe_compress_blocks	echo 4
+par_pipe_compress_blocks	4
+par_pipe_compress_blocks	4 times wc to stderr to stdout
+par_pipe_compress_blocks	      1       1       2
+par_pipe_compress_blocks	      1       1       2
+par_pipe_compress_blocks	      1       1       2
+par_pipe_compress_blocks	      1       1       2
+par_pipe_compress_blocks	1 2 3 4
+par_pipe_compress_blocks	1
+par_pipe_compress_blocks	2
+par_pipe_compress_blocks	3
+par_pipe_compress_blocks	4
+par_pipe_compress_blocks	1 2 3 4
+par_pipe_compress_blocks	1
+par_pipe_compress_blocks	2
+par_pipe_compress_blocks	3
+par_pipe_compress_blocks	4
+par_pipe_compress_blocks	1 2
+par_pipe_compress_blocks	1
+par_pipe_compress_blocks	2
+par_pipe_compress_blocks	1 2 3
+par_pipe_compress_blocks	1
+par_pipe_compress_blocks	2
+par_pipe_compress_blocks	3
 par_pxz_complains	bug #44250: pxz complains File format not recognized but decompresses anyway
 par_pxz_complains	ls: cannot access '/OK-if-missing-file': No such file or directory
 par_pxz_complains	can not seek in input: Illegal seek

testsuite/wanted-results/parallel-local-ssh2

@@ -90,14 +90,14 @@ par_tee_ssh	2
 par_tee_ssh	3
 par_tee_ssh	3
 par_wd_no_such_dir	### --wd no-such-dir - csh
-par_wd_no_such_dir	mkdir: cannot create directory ‘/no-such-dir’: Permission denied
+par_wd_no_such_dir	mkdir: cannot create directory '/no-such-dir': Permission denied
 par_wd_no_such_dir	parallel: Cannot chdir to /no-such-dir
 par_wd_no_such_dir	Exit code 1
 par_wd_no_such_dir	### --wd no-such-dir - tcsh
-par_wd_no_such_dir	mkdir: cannot create directory ‘/no-such-dir’: Permission denied
+par_wd_no_such_dir	mkdir: cannot create directory '/no-such-dir': Permission denied
 par_wd_no_such_dir	parallel: Cannot chdir to /no-such-dir
 par_wd_no_such_dir	Exit code 1
 par_wd_no_such_dir	### --wd no-such-dir - bash
-par_wd_no_such_dir	mkdir: cannot create directory ‘/no-such-dir’: Permission denied
+par_wd_no_such_dir	mkdir: cannot create directory '/no-such-dir': Permission denied
 par_wd_no_such_dir	parallel: Cannot chdir to /no-such-dir
 par_wd_no_such_dir	Exit code 1

testsuite/wanted-results/parallel-local-ssh7

@@ -315,7 +315,7 @@ par_tcsh_man	Academic tradition requires you to cite works you base your article
 par_tcsh_man	If you use programs that use GNU Parallel to process data for an article in a
 par_tcsh_man	scientific publication, please cite:
 par_tcsh_man	
-par_tcsh_man	  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+par_tcsh_man	  O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
 par_tcsh_man	  DOI https://doi.org/10.5281/zenodo.1146014
 par_tcsh_man	
 par_tcsh_man	This helps funding further development; AND IT WON'T COST YOU A CENT.
@@ -754,15 +754,15 @@ par_fish_underscore	aliases and arrays in functions work
 par_fish_underscore	aliases functions work
 par_fish_underscore	aliases functions work
 par_fish_underscore	      ^
-par_fish_underscore	in function “myfunc”
+par_fish_underscore	in function 'myfunc'
 par_fish_underscore		called on standard input
-par_fish_underscore		with parameter list “work”
+par_fish_underscore		with parameter list 'work'
 par_fish_underscore	
 par_fish_underscore	OK if   ^^^^^^^^^^^^^^^^^ no myecho
 par_fish_underscore	      ^
-par_fish_underscore	in function “myfunc”
+par_fish_underscore	in function 'myfunc'
 par_fish_underscore		called on standard input
-par_fish_underscore		with parameter list “work”
+par_fish_underscore		with parameter list 'work'
 par_fish_underscore	
 par_fish_underscore	OK if   ^^^^^^^^^^^^^^^^^ no myecho
 par_fish_underscore	      ^
@@ -1020,7 +1020,7 @@ par_csh_man
 par_csh_man	
 par_csh_man	                {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
 par_csh_man	  DOI https://doi.org/10.5281/zenodo.1146014
-par_csh_man	  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+par_csh_man	  O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
 par_csh_man	### From man env_parallel
 par_csh_man	--colsep regexp Split input on regexp for positional replacements
 par_csh_man	--nonall        Run the given command with no arguments on all sshlogins

testsuite/wanted-results/parallel-local12

@@ -4,7 +4,7 @@ Academic tradition requires you to cite works you base your article on.
 If you use programs that use GNU Parallel to process data for an article in a
 scientific publication, please cite:
 
-  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+  O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
   DOI https://doi.org/10.5281/zenodo.1146014
 
 This helps funding further development; AND IT WON'T COST YOU A CENT.
@@ -27,7 +27,7 @@ Academic tradition requires you to cite works you base your article on.
 If you use programs that use GNU Parallel to process data for an article in a
 scientific publication, please cite:
 
-  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+  O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
   DOI https://doi.org/10.5281/zenodo.1146014
 
 This helps funding further development; AND IT WON'T COST YOU A CENT.
@@ -47,7 +47,7 @@ scientific publication, please cite:
       author       = {Tange, Ole},
       title        = {GNU Parallel 2018},
       publisher    = {Ole Tange},
-      month        = Apr,
+      month        = Mar,
       year         = 2018,
       ISBN         = {9781387509881},
       doi          = {10.5281/zenodo.1146014},

testsuite/wanted-results/parallel-local22

@@ -75,11 +75,11 @@ parallel: Error: --pipepart is incompatible with --max-replace-args, --max-lines
 echo '### bug #42893: --block should not cause decimals in cat_partial'
 ### bug #42893: --block should not cause decimals in cat_partial
   seq 100000 >/tmp/parallel-decimal;   parallel --dry-run -kvv --pipepart --block 0.12345M -a /tmp/parallel-decimal true;   rm /tmp/parallel-decimal
-</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true)
+</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true)
-</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true)
+</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true)
-</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true)
+</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true)
-</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true)
+</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true)
-</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true)
+</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true)
 echo '### bug #42902: profiles containing arguments with space'
 ### bug #42902: profiles containing arguments with space
   echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH;   parallel -JFULLPATH echo FULLPATH ::: $0

testsuite/wanted-results/parallel-local9

@@ -171,7 +171,7 @@ Academic tradition requires you to cite works you base your article on.
 If you use programs that use GNU Parallel to process data for an article in a
 scientific publication, please cite:
 
-  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+  O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
   DOI https://doi.org/10.5281/zenodo.1146014
 
 This helps funding further development; AND IT WON'T COST YOU A CENT.