From 213e49c07354321eb5f03cbac54505c8faab3585 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 4 Feb 2014 20:08:36 +0100 Subject: [PATCH] parallel: Fixed bug #41409: -N + -S fails by halfing max command line length if sshlogin <> : --- doc/release_new_version | 11 ++++- src/parallel | 22 +++++++-- testsuite/tests-to-run/parallel-local-ssh1.sh | 3 ++ testsuite/wanted-results/parallel-local-ssh1 | 2 + testsuite/wanted-results/parallel-local150 | 7 +++ testsuite/wanted-results/parallel-local152 | 47 +++++++------------ testsuite/wanted-results/parallel-local164 | 33 +++++++------ 7 files changed, 72 insertions(+), 53 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 76778f62..2007d49f 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -204,17 +204,24 @@ cc:Sandro Cazzaniga , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20140222 ('') released +Subject: GNU Parallel 20140222 ('Sochi') released -GNU Parallel 20140222 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20140222 ('Sochi') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: * --tollef has been retired. +* GNU Parallel was cited in: Speaker Verification and Keyword Spotting + Systems for Forensic Applications + http://lantana.tenet.res.in/website_files/thesis/Phd/srikanth.pdf + * GNU Parallel is co-distributed with RepeatExplorer http://www.vcru.wisc.edu/simonlab/bioinformatics/programs/repeatexplorer/README.txt +* PHP wrapper class for the GNU Parallel tool + https://github.com/geopal-solutions/gnu-parallel-wrapper/tree/master + * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index 54a6973d..331ac738 100755 --- a/src/parallel +++ b/src/parallel @@ -1879,11 +1879,21 @@ sub parse_sshlogin { } } } + $Global::minimal_command_line_length = 8_000_000; for my $sshlogin_string (@login) { my $sshlogin = SSHLogin->new($sshlogin_string); - $sshlogin->set_maxlength(Limits::Command::max_length()); + if($sshlogin_string eq ":") { + $sshlogin->set_maxlength(Limits::Command::max_length()); + } else { + # If all chars needs to be quoted, every other character will be \ + $sshlogin->set_maxlength(Limits::Command::max_length()/2); + } + $Global::minimal_command_line_length = + ::min($Global::minimal_command_line_length, $sshlogin->maxlength()); $Global::host{$sshlogin->string()} = $sshlogin; } + + debug("sshlogin: ", my_dump(%Global::host),"\n"); if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) { if(not remote_hosts()) { @@ -2044,6 +2054,7 @@ sub filter_hosts { unlink $tmpfile; delete @Global::host{@down_hosts}; @down_hosts and ::warning("Removed @down_hosts\n"); + $Global::minimal_command_line_length = 8_000_000; while (my ($sshlogin, $obj) = each %Global::host) { $ncpus{$sshlogin} or ::die_bug("ncpus missing: ".$obj->serverlogin()); $ncores{$sshlogin} or ::die_bug("ncores missing: ".$obj->serverlogin()); @@ -2056,8 +2067,12 @@ sub filter_hosts { } $obj->set_time_to_login($time_to_login{$sshlogin}); $obj->set_maxlength($maxlen{$sshlogin}); + $Global::minimal_command_line_length = + ::min($Global::minimal_command_line_length, + int($maxlen{$sshlogin}/2)); ::debug("Timing from -S:$sshlogin ncpus:$ncpus{$sshlogin} ncores:$ncores{$sshlogin} ", - "time_to_login:$time_to_login{$sshlogin} maxlen:$maxlen{$sshlogin}\n"); + "time_to_login:$time_to_login{$sshlogin} maxlen:$maxlen{$sshlogin} ", + "min_max_len:$Global::minimal_command_line_length\n"); } } @@ -5154,13 +5169,14 @@ sub populate { # Returns: N/A my $self = shift; my $next_arg; + my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); while (not $self->{'arg_queue'}->empty()) { $next_arg = $self->{'arg_queue'}->get(); if(not defined $next_arg) { next; } $self->push($next_arg); - if($self->len() >= Limits::Command::max_length()) { + if($self->len() >= $max_len) { # Command length is now > max_length # If there are arguments: remove the last # If there are no arguments: Error diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 9a567db7..b1c9ac35 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -5,6 +5,9 @@ mkdir tmp cd tmp cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j8 -k -L1 +echo '### Deal with long command lines on remote servers' + perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc + echo '### Test bug #34241: --pipe should not spawn unneeded processes' seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 9a12e3ae..17f3c6a1 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -1,3 +1,5 @@ +### Deal with long command lines on remote servers + 4 10 100010 ### Test bug #34241: --pipe should not spawn unneeded processes 1 2 diff --git a/testsuite/wanted-results/parallel-local150 b/testsuite/wanted-results/parallel-local150 index 2112614e..0ebc784d 100644 --- a/testsuite/wanted-results/parallel-local150 +++ b/testsuite/wanted-results/parallel-local150 @@ -31,6 +31,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -38,6 +39,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -45,6 +47,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -52,6 +55,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -59,6 +63,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -66,6 +71,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -73,6 +79,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. ### Test --joblog followed by --resume --joblog 5 49 ### Test --resume --joblog followed by --resume --joblog diff --git a/testsuite/wanted-results/parallel-local152 b/testsuite/wanted-results/parallel-local152 index 2b826376..95432214 100644 --- a/testsuite/wanted-results/parallel-local152 +++ b/testsuite/wanted-results/parallel-local152 @@ -1,33 +1,18 @@ ### test global config - must run alone so the global config does not confuse others -parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu. -parallel: Warning: --tollef is obsolete and will be retired 20140222. -parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html -1 -2 -3 -::: -a -b -c -1 -2 -3 --- -a -b -c -1 -2 -3 --- -a -b -c -1 -2 -3 --- -a -b -c +parallel: Error: -g has been retired. Use --group. +parallel: Error: -B has been retired. Use --bf. +parallel: Error: -T has been retired. Use --tty. +parallel: Error: -U has been retired. Use --er. +parallel: Error: -W has been retired. Use --wd. +parallel: Error: -Y has been retired. Use --shebang. +parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. +parallel: Error: -g has been retired. Use --group. +parallel: Error: -B has been retired. Use --bf. +parallel: Error: -T has been retired. Use --tty. +parallel: Error: -U has been retired. Use --er. +parallel: Error: -W has been retired. Use --wd. +parallel: Error: -Y has been retired. Use --shebang. +parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. <<< End test global config - must run alone so the global config does not confuse others diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index 1c99f8c5..4dc22d20 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -84,24 +84,23 @@ d bug #37956: --colsep does not default to '\t' as specified in the man page. one 1 ### Test --tollef -1 -2 -3 -::: -a -b -c -parallel: Warning: --tollef is obsolete and will be retired 20140222. -parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html -parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. +parallel: Error: -B has been retired. Use --bf. +parallel: Error: -H has been retired. Use --halt. +parallel: Error: -T has been retired. Use --tty. +parallel: Error: -U has been retired. Use --er. +parallel: Error: -W has been retired. Use --wd. +parallel: Error: -Y has been retired. Use --shebang. +parallel: Error: -g has been retired. Use --group. ### Test --tollef --gnu -1 -2 -3 --- -a -b -c +parallel: Error: -g has been retired. Use --group. +parallel: Error: -B has been retired. Use --bf. +parallel: Error: -T has been retired. Use --tty. +parallel: Error: -U has been retired. Use --er. +parallel: Error: -W has been retired. Use --wd. +parallel: Error: -Y has been retired. Use --shebang. +parallel: Error: -H has been retired. Use --halt. +parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. ### Test --gnu 1 2