--eta was broken. Now fixed.

Race condition in sem fixed with setsid().
Passes unittest.

doc/FUTURE_IDEAS

@@ -1,16 +1,7 @@
+Bug: seq 1 10 | stdout parallel --eta echo
+
 # Examples of sem
 
-Weird bug - only interactive
-perlipc: Complete Dissociation of Child from Parent
-Setting $SIG{CHLD} to 'IGNORE'
-
-echo '### BUG: Test --fg followed by --bg'
-parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
-parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
-parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
-parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
-sem --wait
-
 find . -execdir sem -j100 'sleep 4; echo {}' \; ; sem --wait
 
 
@@ -18,46 +9,12 @@ fex syntax for splitting fields
 http://www.semicomplete.com/projects/fex/
 sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1}
 
-
-Unittest: --colsep + multiple -a
-
 Import sql
 
 inputfile tabel, Split colonner til {n}
 sql :foo 'select * from bar' | parallel --colsep '\|' do_stuff {4} {1}
 
 
-
-if(defined $::opt_colsep and defined @::opt_a and @::opt_a > 1) {
-   # must be done after converting :::: to -a -a
-   warn("--colsep incompatible with multiple argument files. Ignoring --colsep");
-   $::opt_colsep = undef;
-}
-
-if($::opt_colsep) {
-  # read input either from -a or from stdin
-  my $max_cols = 0;
-  my @table;
-  my $lineno = 0;
-  $Global::input_is_filename = 0; # cheat get_next_arg into not quoting
-  while get_next_arg {
-    my @cols = split /$::opt_colsep/o, $_;
-    if ($Global::trim) {
-       for(@cols) { s/^\s+//; s/\s+$//; }
-    }
-    $max_cols = max($#cols+1,$max_cols);
-    @table[$lineno++] = @cols;
-  }
-  $::opt_N = $max_cols;
-  for ($lineno = 0; $lineno <= $#table; $lineno++) {
-    if (not defined $table[$lineno][$max_col-1]) {
-      # Make sure the table has the same columns for all rows
-      $table[$lineno][$max_col-1] = "";
-    }
-    unget_arg(@table[$lineno]);
-  }
-}
-
 --autocolsep: Læs alle linjer.
 Prøv fastlængde: Find tegn, som står i alle linjer på de samme pladser. Risiko for falske pos
 Prøv fieldsep: Find eet tegn, som optræder det samme antal gange i alle linjer (tab sep)
@@ -117,6 +74,9 @@ find music-files -type f | parallel -j+0 lame {} -o {.}.mp3
 find music-files -type f | parallel -j+0 -S :,computer1.examle.com,computer2.example.com \
 --eta --trc {.}.mp3 lame {} -o {.}.mp3
 
+# Colsep
+# sem
+
 
 =head1 YouTube video
 

doc/release_new_version

@@ -91,7 +91,9 @@ download at: http://ftp.gnu.org/gnu/parallel/
 New in this release:
 
 * Counting semaphore functionality: start a job in the background. If
-  N jobs are already running, wait for one to complete.
+  N jobs are already running, wait for one to complete. Examples:
+  sem 'sleep 2; echo foo'; sem 'sleep 1; echo bar'; sem --wait
+  sem -j2 'sleep 2; echo foo'; sem -j2 'sleep 1; echo bar'; sem --wait
 
 * With --colsep a table can be used as input. Example:
   cat tab_sep_table | parallel --colsep '\t' echo col1 {1} col2 {2}

src/parallel

@@ -1265,11 +1265,11 @@ To run:
 
 you can run:
 
-  parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1}
+B<parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1}>
 
 Note: The default for GNU B<parallel> is to remove the spaces around the columns. To keep the spaces:
 
-  parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1}
+B<parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1}>
 
 
 =head1 EXAMPLE: Working as cat | sh. Ressource inexpensive jobs and evaluation
@@ -2077,7 +2077,7 @@ B<xjobs>(1), B<prll>(1), B<dxargs>(1), B<mdm>(1)
 use IPC::Open3;
 use Symbol qw(gensym);
 use IO::File;
-use POSIX ":sys_wait_h";
+use POSIX qw/:sys_wait_h setsid/;
 use File::Temp qw/tempfile tempdir/;
 use Getopt::Long;
 use strict;
@@ -2109,8 +2109,6 @@ sub acquire_semaphore {
     my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
     $sem->acquire();
     debug("run");
-    $Global::argfile = open_or_exit("/dev/null");
-    unget_arg("");
     if($Semaphore::fg) {
 	# skip
     } else {
@@ -2118,14 +2116,11 @@ sub acquire_semaphore {
 	# therefore release and re-acquire the semaphore
 	$sem->release();
 	if(fork()) {
-	    # TODO figure out the race condition that requires this sleep 1
-	    # sem --fg seq 21 30
-	    # sem --bg seq 31 40
-	    sleep(1);
 	    exit(0);
 	} else {
 	    # child
 	    # Get a semaphore for this pid
+	    die "Can't start a new session: $!" if setsid() == -1;
 	    $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
 	    $sem->acquire();
 	}
@@ -2316,6 +2311,25 @@ sub parse_options {
 	$Global::Xargs = 1;
     }
 
+    # Semaphore defaults
+    # Must be done before computing number of processes and max_line_length as no args
+    $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem'
+    if($Global::semaphore) {
+	# A semaphore does not take input from neither stdin nor file
+	$Global::argfile = open_or_exit("/dev/null");
+	unget_arg("");
+	$Semaphore::timeout = $::opt_semaphoretimeout || 0;
+	if(defined $::opt_semaphorename) {
+	    $Semaphore::name = $::opt_semaphorename;
+	} else {
+	    $Semaphore::name = `tty`;
+	    chomp $Semaphore::name;
+	}
+	$Semaphore::fg = $::opt_fg;
+	$Semaphore::wait = $::opt_wait;
+	$Global::default_simultaneous_sshlogins = 1;
+    }
+
     if(defined $::opt_eta) {
 	# must be done after opt_a
 	$::opt_progress = $::opt_eta;
@@ -2345,22 +2359,6 @@ sub parse_options {
 	print STDERR ("Warning: using -X or -m with --sshlogin may fail\n");
     }
 
-    # Semaphore defaults
-    # Must be done before computing number of processes
-    $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem'
-    if($Global::semaphore) {
-	$Semaphore::timeout = $::opt_semaphoretimeout || 0;
-	if(defined $::opt_semaphorename) {
-	    $Semaphore::name = $::opt_semaphorename;
-	} else {
-	    $Semaphore::name = `tty`;
-	    chomp $Semaphore::name;
-	}
-	$Semaphore::fg = $::opt_fg;
-	$Semaphore::wait = $::opt_wait;
-	$Global::default_simultaneous_sshlogins = 1;
-    }
-
     # Needs to be done after setting $Global::command and $Global::command_line_max_len
     # as '-m' influences the number of commands that needs to be run
     if(defined $::opt_P) {
@@ -2518,7 +2516,7 @@ sub trim {
     # lr|rl = both
     # Returns:
     #   string with white space removed as needed
-    my (@strings) = (@_);
+    my (@strings) = map { defined $_ ? $_ : "" } (@_);
     my $arg;
     if($Global::trim eq "n") {
 	# skip
@@ -3774,9 +3772,11 @@ sub more_arguments {
     # Returns:
     #   whether there are more arguments to be processed or not
     my $fh = shift || $Global::argfile;
-    return (@Global::unget_argv or @{$Global::unget_line{$fh}} or
+    return (
-	    @{$Global::unget_col{$fh}} or @Global::unget_arg or not
+	@Global::unget_argv or
-	    eof $fh);
+	(defined $Global::unget_line{$fh} and @{$Global::unget_line{$fh}}) or
+	(defined $Global::unget_col{$fh} and @{$Global::unget_col{$fh}})
+	or @Global::unget_arg or not eof $fh);
 }
 
 sub get_line_from_fh {

unittest/tests-to-run/sem01.sh

@@ -24,16 +24,17 @@ sem -u -j2  'echo job2; sleep 0.5; echo job2'
 sem --wait
 echo done
 
-echo '### Test similar example as from man page'
+echo '### Test similar example as from man page - run 2 jobs simultaneously'
-for i in 0.5 0.1 0.2 0.3 0.4 ; do
+echo 'Expect done: 1 2 5 3 4'
-  echo $i
+for i in 5 1 2 3 4 ; do
-  sem -j+0 sleep $i ";" echo done $i
+  echo Scheduling $i
+  sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i
 done
 sem --wait
 
-echo '### BUG: Test --fg followed by --bg'
+echo '### Test --fg followed by --bg'
-parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
+parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30
-parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
+parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30
-parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
+parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30
-parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
+parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30
 sem --wait

unittest/tests-to-run/test29.sh

@@ -37,6 +37,12 @@ echo '### Test of tab as colsep'
 printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1}
 parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1}
 
+echo '### Test of multiple -a plus colsep'
+parallel -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1}
+
+echo '### Test of multiple -a no colsep'
+parallel -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1}
+
 echo '### Test of quoting after colsplit'
 parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null'
 

unittest/tests-to-run/test30.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo '### Test of --eta'
+seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l
+
+echo '### Test of --progress'
+seq 1 10 | stdout parallel --progress "sleep 1; echo {}" | wc -l

unittest/wanted-results/sem01

@@ -54,18 +54,24 @@ job2
 job1
 job2
 done
-### Test similar example as from man page
+### Test similar example as from man page - run 2 jobs simultaneously
-0.5
+Expect done: 1 2 5 3 4
-0.1
+Scheduling 5
-0.2
+Scheduling 1
-done 0.1
+starting 5
-0.3
+Scheduling 2
-done 0.5
+starting 1
-0.4
+done 1
-done 0.2
+Scheduling 3
-done 0.3
+starting 2
-done 0.4
+done 2
-### BUG: Test --fg followed by --bg
+Scheduling 4
+starting 3
+done 5
+starting 4
+done 3
+done 4
+### Test --fg followed by --bg
 1
 2
 3

unittest/wanted-results/test29

@@ -38,6 +38,14 @@ abc def
 ghi jkl
 abc def
 ghi jkl
+### Test of multiple -a plus colsep
+abc def
+ghi jkl
+mno
+pqr
+### Test of multiple -a no colsep
+abc	def ghi
+jkl	mno pqr
 ### Test of quoting after colsplit
 >/tmp/null >/dev/null
 ### Test of --colsep as regexp

unittest/wanted-results/test30

@@ -0,0 +1,4 @@
+### Test of --eta
+16
+### Test of --progress
+16