parallel: Fixed bug #41409: -N + -S fails by halfing max command line length if sshlogin <> :

doc/release_new_version

@@ -204,17 +204,24 @@ cc:Sandro Cazzaniga <kharec@mandriva.org>,
    Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
    Jesse Alama <jesse.alama@gmail.com>
 
-Subject: GNU Parallel 20140222 ('') released
+Subject: GNU Parallel 20140222 ('Sochi') released
 
-GNU Parallel 20140222 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
+GNU Parallel 20140222 ('Sochi') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
 * --tollef has been retired.
 
+* GNU Parallel was cited in: Speaker Verification and Keyword Spotting
+  Systems for Forensic Applications
+  http://lantana.tenet.res.in/website_files/thesis/Phd/srikanth.pdf
+
 * GNU Parallel is co-distributed with RepeatExplorer
   http://www.vcru.wisc.edu/simonlab/bioinformatics/programs/repeatexplorer/README.txt
 
+* PHP wrapper class for the GNU Parallel tool
+  https://github.com/geopal-solutions/gnu-parallel-wrapper/tree/master
+
 * Bug fixes and man page updates.
 
 

src/parallel

@@ -1879,11 +1879,21 @@ sub parse_sshlogin {
             }
         }
     }
+    $Global::minimal_command_line_length = 8_000_000;
     for my $sshlogin_string (@login) {
         my $sshlogin = SSHLogin->new($sshlogin_string);
-        $sshlogin->set_maxlength(Limits::Command::max_length());
+	if($sshlogin_string eq ":") {
+	    $sshlogin->set_maxlength(Limits::Command::max_length());
+	} else {
+	    # If all chars needs to be quoted, every other character will be \
+	    $sshlogin->set_maxlength(Limits::Command::max_length()/2);
+	}
+	$Global::minimal_command_line_length =
+	    ::min($Global::minimal_command_line_length, $sshlogin->maxlength());
         $Global::host{$sshlogin->string()} = $sshlogin;
     }
+
+
     debug("sshlogin: ", my_dump(%Global::host),"\n");
     if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) {
         if(not remote_hosts()) {
@@ -2044,6 +2054,7 @@ sub filter_hosts {
     unlink $tmpfile;
     delete @Global::host{@down_hosts};
     @down_hosts and ::warning("Removed @down_hosts\n");
+    $Global::minimal_command_line_length = 8_000_000;
     while (my ($sshlogin, $obj) = each %Global::host) {
 	$ncpus{$sshlogin} or ::die_bug("ncpus missing: ".$obj->serverlogin());
 	$ncores{$sshlogin} or ::die_bug("ncores missing: ".$obj->serverlogin());
@@ -2056,8 +2067,12 @@ sub filter_hosts {
 	}
 	$obj->set_time_to_login($time_to_login{$sshlogin});
         $obj->set_maxlength($maxlen{$sshlogin});
+	$Global::minimal_command_line_length =
+	    ::min($Global::minimal_command_line_length,
+		  int($maxlen{$sshlogin}/2));
 	::debug("Timing from -S:$sshlogin ncpus:$ncpus{$sshlogin} ncores:$ncores{$sshlogin} ",
-		"time_to_login:$time_to_login{$sshlogin} maxlen:$maxlen{$sshlogin}\n");
+		"time_to_login:$time_to_login{$sshlogin} maxlen:$maxlen{$sshlogin} ",
+		"min_max_len:$Global::minimal_command_line_length\n");
     }
 }
 
@@ -5154,13 +5169,14 @@ sub populate {
     # Returns: N/A
     my $self = shift;
     my $next_arg;
+    my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
     while (not $self->{'arg_queue'}->empty()) {
 	$next_arg = $self->{'arg_queue'}->get();
 	if(not defined $next_arg) {
 	    next;
 	}
 	$self->push($next_arg);
-	if($self->len() >= Limits::Command::max_length()) {
+	if($self->len() >= $max_len) {
 	    # Command length is now > max_length
 	    # If there are arguments: remove the last
 	    # If there are no arguments: Error

testsuite/tests-to-run/parallel-local-ssh1.sh

@@ -5,6 +5,9 @@ mkdir tmp
 cd tmp
 
 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j8 -k -L1
+echo '### Deal with long command lines on remote servers'
+  perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
+
 echo '### Test bug #34241: --pipe should not spawn unneeded processes'
   seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
 

testsuite/wanted-results/parallel-local-ssh1

@@ -1,3 +1,5 @@
+### Deal with long command lines on remote servers
+      4      10  100010
 ### Test bug #34241: --pipe should not spawn unneeded processes
 1
 2

testsuite/wanted-results/parallel-local150

@@ -31,6 +31,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 parallel: Error: -g has been retired. Use --group.
 parallel: Error: -B has been retired. Use --bf.
 parallel: Error: -T has been retired. Use --tty.
@@ -38,6 +39,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 parallel: Error: -g has been retired. Use --group.
 parallel: Error: -B has been retired. Use --bf.
 parallel: Error: -T has been retired. Use --tty.
@@ -45,6 +47,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 parallel: Error: -g has been retired. Use --group.
 parallel: Error: -B has been retired. Use --bf.
 parallel: Error: -T has been retired. Use --tty.
@@ -52,6 +55,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 parallel: Error: -g has been retired. Use --group.
 parallel: Error: -B has been retired. Use --bf.
 parallel: Error: -T has been retired. Use --tty.
@@ -59,6 +63,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 parallel: Error: -g has been retired. Use --group.
 parallel: Error: -B has been retired. Use --bf.
 parallel: Error: -T has been retired. Use --tty.
@@ -66,6 +71,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 parallel: Error: -g has been retired. Use --group.
 parallel: Error: -B has been retired. Use --bf.
 parallel: Error: -T has been retired. Use --tty.
@@ -73,6 +79,7 @@ parallel: Error: -U has been retired. Use --er.
 parallel: Error: -W has been retired. Use --wd.
 parallel: Error: -Y has been retired. Use --shebang.
 parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 ### Test --joblog followed by --resume --joblog
       5      49
 ### Test --resume --joblog followed by --resume --joblog

testsuite/wanted-results/parallel-local152

@@ -1,33 +1,18 @@
 ### test global config - must run alone so the global config does not confuse others
-parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.
+parallel: Error: -g has been retired. Use --group.
-parallel: Warning: --tollef is obsolete and will be retired 20140222.
+parallel: Error: -B has been retired. Use --bf.
-parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html
+parallel: Error: -T has been retired. Use --tty.
-1
+parallel: Error: -U has been retired. Use --er.
-2
+parallel: Error: -W has been retired. Use --wd.
-3
+parallel: Error: -Y has been retired. Use --shebang.
-:::
+parallel: Error: -H has been retired. Use --halt.
-a
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
-b
+parallel: Error: -g has been retired. Use --group.
-c
+parallel: Error: -B has been retired. Use --bf.
-1
+parallel: Error: -T has been retired. Use --tty.
-2
+parallel: Error: -U has been retired. Use --er.
-3
+parallel: Error: -W has been retired. Use --wd.
---
+parallel: Error: -Y has been retired. Use --shebang.
-a
+parallel: Error: -H has been retired. Use --halt.
-b
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
-c
-1
-2
-3
---
-a
-b
-c
-1
-2
-3
---
-a
-b
-c
 <<< End test global config - must run alone so the global config does not confuse others

testsuite/wanted-results/parallel-local164

@@ -84,24 +84,23 @@ d
 bug #37956: --colsep does not default to '\t' as specified in the man page.
 one 1
 ### Test --tollef
-1
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
-2
+parallel: Error: -B has been retired. Use --bf.
-3
+parallel: Error: -H has been retired. Use --halt.
-:::
+parallel: Error: -T has been retired. Use --tty.
-a
+parallel: Error: -U has been retired. Use --er.
-b
+parallel: Error: -W has been retired. Use --wd.
-c
+parallel: Error: -Y has been retired. Use --shebang.
-parallel: Warning: --tollef is obsolete and will be retired 20140222.
+parallel: Error: -g has been retired. Use --group.
-parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html
-parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.
 ### Test --tollef --gnu
-1
+parallel: Error: -g has been retired. Use --group.
-2
+parallel: Error: -B has been retired. Use --bf.
-3
+parallel: Error: -T has been retired. Use --tty.
---
+parallel: Error: -U has been retired. Use --er.
-a
+parallel: Error: -W has been retired. Use --wd.
-b
+parallel: Error: -Y has been retired. Use --shebang.
-c
+parallel: Error: -H has been retired. Use --halt.
+parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
 ### Test --gnu
 1
 2