env_parallel.{bash,ksh,pdksh,zsh}: Minimal --record-env.

src/env_parallel.bash

@@ -63,6 +63,13 @@ env_parallel() {
             ' -- "$@"
     )"
 
+    # --record-env
+    if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
+	(compgen -a; compgen -A function; compgen -A variable) |
+	    cat > $HOME/.parallel/ignored_vars
+	return 0
+    fi
+    
     # Grep alias names
     local _alias_NAMES="$(compgen -a |
         grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

src/env_parallel.ksh

@@ -63,6 +63,13 @@ env_parallel() {
             ' -- "$@"
     )"
 
+    # --record-env
+    if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
+	(compgen -a; compgen -A function; compgen -A variable) |
+	    cat > $HOME/.parallel/ignored_vars
+	return 0
+    fi
+    
     # Grep alias names
     _alias_NAMES="$(alias | perl -pe 's/=.*//' |
         grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

src/env_parallel.pdksh

@@ -63,6 +63,13 @@ env_parallel() {
             ' -- "$@"
     )"
 
+    # --record-env
+    if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
+	(compgen -a; compgen -A function; compgen -A variable) |
+	    cat > $HOME/.parallel/ignored_vars
+	return 0
+    fi
+    
     # Grep alias names
     _alias_NAMES="$(alias | perl -pe 's/=.*//' |
         grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

src/env_parallel.zsh

@@ -63,6 +63,13 @@ env_parallel() {
             ' -- "$@"
     )"
 
+    # --record-env
+    if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
+	(compgen -a; compgen -A function; compgen -A variable) |
+	    cat > $HOME/.parallel/ignored_vars
+	return 0
+    fi
+    
     # Grep alias names
     _alias_NAMES="$(print -l ${(k)aliases} |
         grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

src/parallel

@@ -74,40 +74,44 @@ if($opt::skip_first_line) {
     my $fh = $input_source_fh[0];
     <$fh>;
 }
-if($opt::header and not $opt::pipe) {
+
-    # split with colsep or \t
+set_input_source_header();
-    # $header force $colsep = \t if undef?
+
-    my $delimiter = $opt::colsep;
+sub set_input_source_header {
-    $delimiter ||= "\t";
+    if($opt::header and not $opt::pipe) {
-    # regexp for {=
+	# split with colsep or \t
-    my $left = "\Q$Global::parensleft\E";
+	# $header force $colsep = \t if undef?
-    my $l = $Global::parensleft;
+	my $delimiter = defined $opt::colsep ? $opt::colsep : "\t";
-    # regexp for =}
+	# regexp for {=
-    my $right = "\Q$Global::parensright\E";
+	my $left = "\Q$Global::parensleft\E";
-    my $r = $Global::parensright;
+	my $l = $Global::parensleft;
-    my $id = 1;
+	# regexp for =}
-    for my $fh (@input_source_fh) {
+	my $right = "\Q$Global::parensright\E";
-	my $line = <$fh>;
+	my $r = $Global::parensright;
-	chomp($line);
+	my $id = 1;
-	::debug("init", "Delimiter: '$delimiter'");
+	for my $fh (@input_source_fh) {
-	for my $s (split /$delimiter/o, $line) {
+	    my $line = <$fh>;
-	    ::debug("init", "Colname: '$s'");
+	    chomp($line);
-	    # Replace {colname} with {2}
+	    ::debug("init", "Delimiter: '$delimiter'");
-	    for(@command,@Global::ret_files,@Global::transfer_files) {
+	    for my $s (split /$delimiter/o, $line) {
-	      s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
+		::debug("init", "Colname: '$s'");
-		# {=header1 ... =}  =>  {=1 ... =}
+		# Replace {colname} with {2}
-	      s:$left $s (.*?) $right:$l$id$1$r:gx;
+		for(@command,@Global::ret_files,@Global::transfer_files) {
+		  s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
+		    # {=header1 ... =}  =>  {=1 ... =}
+		  s:$left $s (.*?) $right:$l$id$1$r:gx;
+		}
+		$Global::input_source_header{$id} = $s;
+		$id++;
 	    }
-	    $Global::input_source_header{$id} = $s;
+	}
+    } else {
+	my $id = 1;
+	for my $fh (@input_source_fh) {
+	    $Global::input_source_header{$id} = $id;
 	    $id++;
 	}
     }
-} else {
-    my $id = 1;
-    for my $fh (@input_source_fh) {
-	$Global::input_source_header{$id} = $id;
-	$id++;
-    }
 }
 
 if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
@@ -187,10 +191,8 @@ if($Global::semaphore) {
 }
 $SIG{TERM} = \&start_no_new_jobs;
 start_more_jobs();
-if(not $opt::pipepart) {
+if($opt::pipe and not $opt::pipepart) {
-    if($opt::pipe) {
+    spreadstdin();
-	spreadstdin();
-    }
 }
 ::debug("init", "Start draining\n");
 drain_job_queue();
@@ -206,14 +208,10 @@ if($opt::pipe and @opt::a) {
     }
 }
 ::debug("init", "Cleaning\n");
-cleanup();
 if($Global::semaphore) {
     $sem->release();
 }
-for(keys %Global::sshmaster) {
+cleanup();
-    # If 'ssh -M's are running: kill them
-    kill "TERM", $_;
-}
 ::debug("init", "Halt\n");
 if($opt::halt and $Global::halt_when ne "never") {
     if(not defined $Global::halt_exitstatus) {
@@ -1820,6 +1818,10 @@ sub cleanup {
     unlink keys %Global::unlink;
     map { rmdir $_ } keys %Global::unlink;
     if(@opt::basefile and $opt::cleanup) { cleanup_basefile(); }
+    for(keys %Global::sshmaster) {
+	# If 'ssh -M's are running: kill them
+	kill "TERM", $_;
+    }
 }
 
 sub __QUOTING_ARGUMENTS_FOR_SHELL__ {}
@@ -9459,7 +9461,7 @@ sub new {
     if($opt::sqlworker) {
 	# Open SQL table
 	$arg_sub_queue = SQLRecordQueue->new();
-    } elsif($colsep) {
+    } elsif(defined $colsep) {
 	# Open one file with colsep
 	$arg_sub_queue = RecordColQueue->new($fhs);
     } else {

src/parallel.pod

@@ -807,7 +807,9 @@ soon,fail=1-99%
 Use regexp as header. For normal usage the matched header (typically
 the first line: B<--header '.*\n'>) will be split using B<--colsep>
 (which will default to '\t') and column names can be used as
-replacement variables: B<{column name}>.
+replacement variables: B<{column name}>, B<{column name/}>, B<{column
+name//}>, B<{column name/.}>, B<{column name.}>, B<{=column name perl
+expression =}>, ..
 
 For B<--pipe> the matched header will be prepended to each output.
 

testsuite/tests-to-run/parallel-freebsd.sh

@@ -27,8 +27,8 @@ echo 'bug #40135: FreeBSD: sem --fg does not finish under /bin/sh'
   sem --fg 'sleep 1; echo The job finished'
 
 echo 'bug #40133: FreeBSD: --round-robin gives no output'
-  jot 1000000 | parallel --round-robin --pipe -kj3 wc | sort
+  jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc
-  jot 1000000 | parallel --round-robin --pipe -kj4 wc | sort
+  jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc
 
 echo 'bug #40134: FreeBSD: --shebang not working'
   (echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang; 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -647,10 +647,22 @@ echo '### bug #48658: --linebuffer --files'
 
 echo '**'
 
-bug #49538: --header and {= =}
+echo 'bug #49538: --header and {= =}'
 
   parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
   parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
+  parallel --header : echo {var/.} ::: var sub/dir/file.ext
+  parallel --header : echo {var//} ::: var sub/dir/file.ext
+  parallel --header : echo {var/.} ::: var sub/dir/file.ext
+  parallel --header : echo {var/} ::: var sub/dir/file.ext
+  parallel --header : echo {var.} ::: var sub/dir/file.ext
+
+echo '**'
+
+echo 'bug --colsep 0'
+
+  parallel --colsep 0 echo {2} ::: a0OK0c
+  parallel --header : --colsep 0 echo {ok} ::: A0ok0B a0OK0b
 
 echo '**'
 

testsuite/tests-to-run/parallel-local-sql.sh

@@ -26,7 +26,7 @@ p_wrapper() {
   $DEBUG && sort -u $T1 $T2; 
   rm $T1 $T2
   p_showsqlresult $SERVERURL $TABLE
-  $DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null
+  $DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
 }
 
 p_template() {

testsuite/wanted-results/parallel-freebsd

@@ -28,15 +28,10 @@ bug #40135: FreeBSD: sem --fg does not finish under /bin/sh
 The job finished
 echo 'bug #40133: FreeBSD: --round-robin gives no output'
 bug #40133: FreeBSD: --round-robin gives no output
-  jot 1000000 | parallel --round-robin --pipe -kj3 wc | sort
+  jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc
-  299592  299592 2097144
+ 1000000 1000000 6888896
-  299592  299592 2097144
+  jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc
-  400816  400816 2694608
+ 1000000 1000000 6888896
-  jot 1000000 | parallel --round-robin --pipe -kj4 wc | sort
-  149796  149796 1048572
-  235148  235148 1646037
-  299592  299592 2097144
-  315464  315464 2097143
 echo 'bug #40134: FreeBSD: --shebang not working'
 bug #40134: FreeBSD: --shebang not working
   (echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang;   chmod 755 ./shebang; ./shebang

testsuite/wanted-results/parallel-local-0.3s

@@ -1765,11 +1765,29 @@ echo '### bug #48658: --linebuffer --files'
 10
 echo '**'
 **
+echo 'bug #49538: --header and {= =}'
 bug #49538: --header and {= =}
-/bin/bash: bug: command not found
   parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
 OK
   parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
+OK
+  parallel --header : echo {var/.} ::: var sub/dir/file.ext
+file
+  parallel --header : echo {var//} ::: var sub/dir/file.ext
+sub/dir
+  parallel --header : echo {var/.} ::: var sub/dir/file.ext
+file
+  parallel --header : echo {var/} ::: var sub/dir/file.ext
+file.ext
+  parallel --header : echo {var.} ::: var sub/dir/file.ext
+sub/dir/file
+echo '**'
+**
+echo 'bug --colsep 0'
+bug --colsep 0
+  parallel --colsep 0 echo {2} ::: a0OK0c
+OK
+  parallel --header : --colsep 0 echo {ok} ::: A0ok0B a0OK0b
 OK
 echo '**'
 **