tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #41409: -N + -S fails by halfing max command line length if sshlogin <> :

Browse source
This commit is contained in:
Ole Tange 2014-02-04 20:08:36 +01:00
parent eede4d8670
commit 213e49c073
7 changed files with 72 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
doc/release_new_version
View file

@ -204,17 +204,24 @@ cc:Sandro Cazzaniga <kharec@mandriva.org>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20140222 ('') released
Subject: GNU Parallel 20140222 ('Sochi') released
GNU Parallel 20140222 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
GNU Parallel 20140222 ('Sochi') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
New in this release:
* --tollef has been retired.
* GNU Parallel was cited in: Speaker Verification and Keyword Spotting
Systems for Forensic Applications
http://lantana.tenet.res.in/website_files/thesis/Phd/srikanth.pdf
* GNU Parallel is co-distributed with RepeatExplorer
http://www.vcru.wisc.edu/simonlab/bioinformatics/programs/repeatexplorer/README.txt
* PHP wrapper class for the GNU Parallel tool
https://github.com/geopal-solutions/gnu-parallel-wrapper/tree/master
* Bug fixes and man page updates.

22
src/parallel
View file

@ -1879,11 +1879,21 @@ sub parse_sshlogin {
}
}
}
$Global::minimal_command_line_length = 8_000_000;
for my $sshlogin_string (@login) {
my $sshlogin = SSHLogin->new($sshlogin_string);
$sshlogin->set_maxlength(Limits::Command::max_length());
if($sshlogin_string eq ":") {
$sshlogin->set_maxlength(Limits::Command::max_length());
} else {
# If all chars needs to be quoted, every other character will be \
$sshlogin->set_maxlength(Limits::Command::max_length()/2);
}
$Global::minimal_command_line_length =
::min($Global::minimal_command_line_length, $sshlogin->maxlength());
$Global::host{$sshlogin->string()} = $sshlogin;
}
debug("sshlogin: ", my_dump(%Global::host),"\n");
if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) {
if(not remote_hosts()) {
@ -2044,6 +2054,7 @@ sub filter_hosts {
unlink $tmpfile;
delete @Global::host{@down_hosts};
@down_hosts and ::warning("Removed @down_hosts\n");
$Global::minimal_command_line_length = 8_000_000;
while (my ($sshlogin, $obj) = each %Global::host) {
$ncpus{$sshlogin} or ::die_bug("ncpus missing: ".$obj->serverlogin());
$ncores{$sshlogin} or ::die_bug("ncores missing: ".$obj->serverlogin());
@ -2056,8 +2067,12 @@ sub filter_hosts {
}
$obj->set_time_to_login($time_to_login{$sshlogin});
$obj->set_maxlength($maxlen{$sshlogin});
$Global::minimal_command_line_length =
::min($Global::minimal_command_line_length,
int($maxlen{$sshlogin}/2));
::debug("Timing from -S:$sshlogin ncpus:$ncpus{$sshlogin} ncores:$ncores{$sshlogin} ",
"time_to_login:$time_to_login{$sshlogin} maxlen:$maxlen{$sshlogin}\n");
"time_to_login:$time_to_login{$sshlogin} maxlen:$maxlen{$sshlogin} ",
"min_max_len:$Global::minimal_command_line_length\n");
}
}
@ -5154,13 +5169,14 @@ sub populate {
# Returns: N/A
my $self = shift;
my $next_arg;
my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
while (not $self->{'arg_queue'}->empty()) {
$next_arg = $self->{'arg_queue'}->get();
if(not defined $next_arg) {
next;
}
$self->push($next_arg);
if($self->len() >= Limits::Command::max_length()) {
if($self->len() >= $max_len) {
# Command length is now > max_length
# If there are arguments: remove the last
# If there are no arguments: Error

3
testsuite/tests-to-run/parallel-local-ssh1.sh
View file

@ -5,6 +5,9 @@ mkdir tmp
cd tmp
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j8 -k -L1
echo '### Deal with long command lines on remote servers'
perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end

2
testsuite/wanted-results/parallel-local-ssh1
View file

@ -1,3 +1,5 @@
### Deal with long command lines on remote servers
4 10 100010
### Test bug #34241: --pipe should not spawn unneeded processes
1
2

7
testsuite/wanted-results/parallel-local150
View file

@ -31,6 +31,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
@ -38,6 +39,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
@ -45,6 +47,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
@ -52,6 +55,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
@ -59,6 +63,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
@ -66,6 +71,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
@ -73,6 +79,7 @@ parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
### Test --joblog followed by --resume --joblog
5 49
### Test --resume --joblog followed by --resume --joblog

47
testsuite/wanted-results/parallel-local152
View file

@ -1,33 +1,18 @@
### test global config - must run alone so the global config does not confuse others
parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.
parallel: Warning: --tollef is obsolete and will be retired 20140222.
parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html
1
2
3
:::
a
b
c
1
2
3
--
a
b
c
1
2
3
--
a
b
c
1
2
3
--
a
b
c
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
<<< End test global config - must run alone so the global config does not confuse others

33
testsuite/wanted-results/parallel-local164
View file

@ -84,24 +84,23 @@ d
bug #37956: --colsep does not default to '\t' as specified in the man page.
one 1
### Test --tollef
1
2
3
:::
a
b
c
parallel: Warning: --tollef is obsolete and will be retired 20140222.
parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html
parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -g has been retired. Use --group.
### Test --tollef --gnu
1
2
3
--
a
b
c
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l.
### Test --gnu
1
2