parallel: Too slow spawning only gives a warning, not an error.

.parallelrc is now read along with .parallel/config.
Passes testsuite.

src/parallel

@@ -852,6 +852,16 @@ B<--silent>. See also B<-t>.
 
 Print the version GNU B<parallel> and exit.
 
+=item B<--workdir> I<dir> (unimplemented)
+
+=item B<-W> I<dir> (unimplemented)
+
+Files transferred using B<--transfer> and B<--return> will be relative
+to workdir on remote machines, and the command will be executed in
+that dir. The special workdir B<...> will create a workdir i
+B<~/.parallel/workdirs/> on the remote machines and will be removed if
+using B<--cleanup>.
+
 
 =item B<--wait> (beta testing)
 
@@ -1076,7 +1086,7 @@ B<seq 1 30 | parallel date -d '"today -{} days"' +%Y%m%d>
 
 Based on this we can let GNU B<parallel> generate 10 B<wget>s per day:
 
-B<I<the above> | parallel -I {o} seq -w 1 10 "|" parallel wget
+I<the above> B<| parallel -I {o} seq -w 1 10 "|" parallel wget
 http://www.website.com/path/to/{o}_{}.jpg>
 
 =head1 EXAMPLE: Rewriting a for-loop and a while-loop
@@ -1563,6 +1573,16 @@ B<-j> take an argument and thus both need to be at the end of a group.
 
 =head1 CONFIG FILE
 
+The file ~/.parallel/config (formerly known as .parallelrc) will be
+read if it exists.  Lines starting with '#' will be ignored. It can be
+formatted like the environment variable $PARALLEL, but it is often
+easier to simply put each option on its own line.
+
+Options on the command line takes precedence over the environment
+variable $PARALLEL which takes precedence over the file
+~/.parallel/config.
+
+
 The file ~/.parallel/config will be read if it exists. It should be
 formatted like the environment variable $PARALLEL. Lines starting with
 '#' will be ignored.
@@ -1703,7 +1723,7 @@ M1 -  M3 -  -  M6
 O1 -  -  x  -  -
 E1 E2 ?E3 E4 - -
 R1 R2 R3 R4 -  -  ?R7 ? ?
-- -
+-  -
 
 pexec:
 I1 I2 -  I4 I5 -  -
@@ -2387,7 +2407,7 @@ sub acquire_semaphore {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20100922;
+    $Global::version = 20101014;
     $Global::progname = 'parallel';
     $Global::debug = 0;
     $Global::verbose = 0;
@@ -2423,15 +2443,17 @@ sub parse_options {
     }
 
     # Add options from .parallel/config
-    my $parallel_config = $ENV{'HOME'}."/.parallel/config";
+    for my $parallel_config ($ENV{'HOME'}."/.parallel/config",
-    if(-r $parallel_config) {
+			     $ENV{'HOME'}."/.parallelrc") {
-	open (IN, "<", $parallel_config) || die;
+	if(-r $parallel_config) {
-	while(<IN>) {
+	    open (IN, "<", $parallel_config) || die;
-	    /^\s*\#/ and next;
+	    while(<IN>) {
-	    chomp;
+		/^\s*\#/ and next;
-	    unshift @ARGV, $_;
+		chomp;
+		unshift @ARGV, $_;
+	    }
+	    close IN;
 	}
-	close IN;
     }
     # Add options from shell variable $PARALLEL
     $ENV{'PARALLEL'} and unshift @ARGV, split/\n/, $ENV{'PARALLEL'};
@@ -2463,6 +2485,7 @@ sub parse_options {
 	       "transfer" => \$::opt_transfer,
 	       "cleanup" => \$::opt_cleanup,
 	       "basefile|B=s" => \@::opt_basefile,
+	       "workdir|W=s" => \$::opt_workdir,
 	       "halt-on-error|H=s" => \$::opt_halt_on_error,
 	       "retries=i" => \$::opt_retries,
 	       "progress" => \$::opt_progress,
@@ -3184,7 +3207,7 @@ sub processes_available_by_system_limit {
     my ($next_command_line, $args_ref);
     my $more_filehandles;
     my $max_system_proc_reached=0;
-    my $spawning_too_slow=0;
+    my $slow_spawining_warning_printed=0;
     my $time = time;
     my %fh;
     my @children;
@@ -3226,19 +3249,18 @@ sub processes_available_by_system_limit {
 	    $max_system_proc_reached = 1;
 	}
 	debug("Time to fork ten procs: ", time-$time, " (processes so far: ", $system_limit,")\n");
-	if(time-$time > 2) {
+	if(time-$time > 2 and not $slow_spawining_warning_printed) {
-	    # It took more than 2 second to fork ten processes. We should stop forking.
+	    # It took more than 2 second to fork ten processes.
-	    # Let us give the system a little slack
+	    # Give the user a warning. He can press Ctrl-C if this
-	    debug("\nLimiting processes to: $system_limit-10%=".
+	    # sucks.
-		  (int ($system_limit * 0.9)+1)."\n");
+	    print STDERR ("Warning: Starting 10 extra processes takes > 2 sec.\n",
-	    $system_limit = int ($system_limit * 0.9)+1;
+			  "Consider adjusting -j. Press CTRL-C to stop.\n");
-	    $spawning_too_slow = 1;
+	    $slow_spawining_warning_printed = 1;
 	}
     } while($system_limit < $wanted_processes
 	    and (defined $next_command_line or $Global::semaphore)
 	    and $more_filehandles
-	    and not $max_system_proc_reached
+	    and not $max_system_proc_reached);
-	    and not $spawning_too_slow);
     if($system_limit < $wanted_processes and not $more_filehandles) {
 	print STDERR ("Warning: Only enough filehandles to run ",
 		      $system_limit, " jobs in parallel. ",
@@ -3248,10 +3270,6 @@ sub processes_available_by_system_limit {
 	print STDERR ("Warning: Only enough available processes to run ",
 		      $system_limit, " jobs in parallel.\n");
     }
-    if($system_limit < $wanted_processes and $spawning_too_slow) {
-	print STDERR ("Warning: Starting 10 extra processes takes > 2 sec.\n",
-		      "Limiting to ", $system_limit, " jobs in parallel.\n");
-    }
     # Cleanup: Close the files
     for (values %fh) { close $_ }
     # Cleanup: Kill the children
@@ -4910,6 +4928,6 @@ sub unlock {
 
 # Keep perl -w happy
 
-$Private::control_path = $Semaphore::timeout = $Semaphore::wait =
+$::opt_workdir = $Private::control_path = $Semaphore::timeout = $Semaphore::wait =
 $::opt_skip_first_line = $::opt_shebang = 0 ;
 

testsuite/Makefile

@@ -11,6 +11,8 @@ testsuite: ../src/parallel tests-to-run/* wanted-results/*
 	echo | pv -qL 10 || (echo pv is required for testsuite; /bin/false)
 	echo | script -c echo -q /dev/null || (echo script is required for testsuite; /bin/false)
 	niceload true || (echo niceload is required for testsuite; /bin/false)
+	which burnP6 || (echo burnP6 is required for testsuite; /bin/false)
+	which timeout || (echo timeout is required for testsuite; /bin/false)
 	time sh Start.sh
 	date
 

testsuite/tests-to-run/test32.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 echo '### Test of --retries'
-seq 1 10 | parallel --retries 2 -v -S 4.3.2.1,: echo
+seq 1 10 | parallel -k --retries 2 -v -S 4.3.2.1,: echo
 
 echo '### Test of --retries - it should run 13 jobs in total'
 seq 0 12 | parallel --progress -kj100% --retries 1 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \

testsuite/tests-to-run/test34.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo '### Test too slow spawning'
+(sleep 0.3; seq 1 10 | parallel -j50 burnP6) &
+seq 1 500 | nice nice stdout timeout 10 \
+  parallel -j500 "killall -9 burnP6 2>/dev/null ; echo {} >/dev/null"
+killall -9 burnP6

testsuite/wanted-results/test34

@@ -0,0 +1,3 @@
+### Test too slow spawning
+Warning: Starting 10 extra processes takes > 2 sec.
+Consider adjusting -j. Press CTRL-C to stop.