From 3549c38b0f6ef59e0fb4ba2a534113b068b5ea92 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 1 Feb 2015 21:46:06 +0100 Subject: [PATCH] Fixed bug #44148: csh long vars fail exporting. --- src/parallel | 86 ++++++++++++------- testsuite/tests-to-run/parallel-local-ssh2.sh | 2 +- testsuite/tests-to-run/parallel-local-ssh3.sh | 4 - testsuite/tests-to-run/parallel-local-ssh4.sh | 27 ++++++ testsuite/tests-to-run/parallel-remote1.sh | 19 ++-- testsuite/wanted-results/parallel-local-ssh3 | 6 -- testsuite/wanted-results/parallel-local-ssh4 | 20 +++++ testsuite/wanted-results/parallel-remote1 | 24 +++--- 8 files changed, 123 insertions(+), 65 deletions(-) create mode 100644 testsuite/tests-to-run/parallel-local-ssh4.sh create mode 100644 testsuite/wanted-results/parallel-local-ssh4 diff --git a/src/parallel b/src/parallel index dfa9f0bd..e5edba68 100755 --- a/src/parallel +++ b/src/parallel @@ -6025,16 +6025,63 @@ sub sshlogin_wrap { my $self = shift; my $command = shift; + # TODO test that *sh -c 'parallel --env' use *sh if(not defined $self->{'sshlogin_wrap'}) { my $sshlogin = $self->sshlogin(); - my $sshcmd = $sshlogin->sshcommand(); my $serverlogin = $sshlogin->serverlogin(); - my ($pre,$post,$cleanup)=("","",""); + my $quoted_remote_command; + $ENV{'PARALLEL_SEQ'} = $self->seq(); + $ENV{'PARALLEL_PID'} = $$; if($serverlogin eq ":") { - # No transfer/environment neeeded - # TODO Copy env between shells (zsh to bash) - $self->{'sshlogin_wrap'} = $command; + if(@opt::env) { + # Prepend with environment setter, which sets functions in zsh + my ($csh_friendly,$envset,$bashfuncset) = env_as_eval(); + my $env_command = $envset.$bashfuncset. + '@ARGV="'.::perl_quote_scalar($command).'";'. + "exec\"$Global::shell\",\"-c\",\(\$bashfunc.\"\@ARGV\"\)\;die\"exec:\$\!\\n\"\;"; + if(length $env_command > 1000 + or + not $csh_friendly + or + $command =~ /\n/) { + # csh does not deal well with > 1000 chars in one word + # csh does not deal well with $ENV with \n + $env_command = "perl -e '".base64_zip_eval()."' ". + join" ",string_zip_base64($env_command); + $self->{'sshlogin_wrap'} = $env_command; + } else { + $self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($env_command); + } + } else { + $self->{'sshlogin_wrap'} = $command; + } } else { + my $pwd = ""; + if($opt::workdir) { + # Create remote workdir if needed. Then cd to it. + my $wd = $self->workdir(); + $pwd = qq{system("mkdir","-p","--","$wd"); chdir "$wd" ||}. + qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;}; + } + my ($csh_friendly,$envset,$bashfuncset) = env_as_eval(); + my $remote_command = $pwd.$envset.$bashfuncset. + '@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script(); + $quoted_remote_command = "perl -e ".::shell_quote_scalar($remote_command); + if(length $quoted_remote_command > 1000 + or + not $csh_friendly + or + $command =~ /\n/) { + # csh does not deal well with > 1000 chars in one word + # csh does not deal well with $ENV with \n + $quoted_remote_command = "perl -e \\''".base64_zip_eval()."'\\' ". + join" ",string_zip_base64($remote_command); + } else { + $quoted_remote_command = ::shell_quote_scalar($quoted_remote_command); + } + + my $sshcmd = $sshlogin->sshcommand(); + my ($pre,$post,$cleanup)=("","",""); # --transfer $pre .= $self->sshtransfer(); # --return @@ -6045,36 +6092,9 @@ sub sshlogin_wrap { # We need to save the exit status of the job $post = '_EXIT_status=$?; ' . $post . ' exit $_EXIT_status;'; } - # If the remote login shell is (t)csh then use 'setenv' - # otherwise use 'export' - # We cannot use parse_env_var(), as PARALLEL_SEQ changes - # for each command - $ENV{'PARALLEL_SEQ'} = $self->seq(); - $ENV{'PARALLEL_PID'} = $$; - my $pwd = ""; - if($opt::workdir) { - # Create remote workdir if needed. Then cd to it. - my $wd = $self->workdir(); - $pwd = qq{system("mkdir","-p","--","$wd"); chdir "$wd" ||}. - qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;}; - } - my ($csh_friendly,$envset,$bashfuncset) = env_as_eval(); - my $remote_command = $pwd.$envset.$bashfuncset. - '@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script(); - my $quoted_remote_command = "exec perl -e ".::shell_quote_scalar(::shell_quote_scalar($remote_command)); - if(length $quoted_remote_command > 1000 - or - not $csh_friendly - or - $command =~ "\n") { - # csh does not deal well with > 1000 chars in one word - # csh does not deal well with $ENV with \n - $quoted_remote_command = "exec perl -e '".::shell_quote_scalar(base64_zip_eval())."' ". - join" ",string_zip_base64($remote_command); - } $self->{'sshlogin_wrap'} = ($pre - . "$sshcmd $serverlogin " + . "$sshcmd $serverlogin exec " . $quoted_remote_command . ";" . $post); diff --git a/testsuite/tests-to-run/parallel-local-ssh2.sh b/testsuite/tests-to-run/parallel-local-ssh2.sh index 27b1e6dc..d49e957e 100644 --- a/testsuite/tests-to-run/parallel-local-ssh2.sh +++ b/testsuite/tests-to-run/parallel-local-ssh2.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj6 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj5 -k --joblog /tmp/jl-`basename $0` -L1 echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded" # Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed # Thus must take at least 25 sec to run diff --git a/testsuite/tests-to-run/parallel-local-ssh3.sh b/testsuite/tests-to-run/parallel-local-ssh3.sh index e99a3f95..1ec6364a 100644 --- a/testsuite/tests-to-run/parallel-local-ssh3.sh +++ b/testsuite/tests-to-run/parallel-local-ssh3.sh @@ -61,8 +61,4 @@ echo '### Uniq {=perlexpr=} in return - not used in command' # Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept? -echo '### zsh' - ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' - ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK' - EOF diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh new file mode 100644 index 00000000..dd0e48d3 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# SSH only allowed to localhost/lo +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj8 -k --joblog /tmp/jl-`basename $0` -L1 +echo '### zsh' + ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; + export fun; + parallel --env fun fun ::: OK' + + ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; + export fun; + parallel -S parallel@lo --env fun fun ::: OK' + +echo '### csh' + echo "3 big vars run remotely - length(base64) > 1000" + ssh csh@lo 'setenv A `seq 200|xargs`; + setenv B `seq 200 -1 1|xargs`; + setenv C `seq 300 -2 1|xargs`; + parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' + + echo "3 big vars run locally" + ssh csh@lo 'setenv A `seq 200|xargs`; + setenv B `seq 200 -1 1|xargs`; + setenv C `seq 300 -2 1|xargs`; + parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' + +EOF diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index 57382796..4a6158c2 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -11,9 +11,15 @@ echo 'TODO test ssh with > 9 simultaneous' echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1 echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 chmod 755 /tmp/myssh1 /tmp/myssh2 -seq 1 100 | parallel --sshdelay 0.01 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo +seq 1 100 | parallel --sshdelay 0.03 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo + +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj3 -k -L1 +echo '### test --timeout --retries' + parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 $SSHLOGIN1 $SSHLOGIN2 $SSHLOGIN3 + +echo '### test --filter-hosts with server w/o ssh, non-existing server' + parallel -S 192.168.1.197,8.8.8.8,$SSHLOGIN1,$SSHLOGIN2,$SSHLOGIN3 --filter-hosts --nonall -k --tag echo -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj1 -k -L1 echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & @@ -26,13 +32,8 @@ echo '### test --workdir . in $HOME' cd && mkdir -p parallel-test && cd parallel-test && echo OK > testfile && parallel --workdir . --transfer -S $SSHLOGIN1 cat {} ::: testfile -echo '### test --timeout --retries' - parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 $SSHLOGIN1 $SSHLOGIN2 $SSHLOGIN3 - -echo '### test --filter-hosts with server w/o ssh, non-existing server' - parallel -S 192.168.1.197,8.8.8.8,$SSHLOGIN1,$SSHLOGIN2,$SSHLOGIN3 --filter-hosts --nonall -k --tag echo - echo '### Missing: test --filter-hosts proxied through the one host' EOF -rm /tmp/myssh1 /tmp/myssh2 +rm /tmp/myssh1 /tmp/myssh2 /tmp/myssh1-run /tmp/myssh2-run + diff --git a/testsuite/wanted-results/parallel-local-ssh3 b/testsuite/wanted-results/parallel-local-ssh3 index 8409ef9c..f7cfaadc 100644 --- a/testsuite/wanted-results/parallel-local-ssh3 +++ b/testsuite/wanted-results/parallel-local-ssh3 @@ -101,9 +101,3 @@ echo '### Uniq {=perlexpr=} in return - not used in command' cd /tmp; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr; echo local > parallel_perlexpr; parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr; cat /tmp/parallel_perlexpr.2Parallel_PerlexPr; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr remote OK # Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept? -echo '### zsh' -### zsh - ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' -function from zsh to zsh OK - ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK' -function from zsh to bash OK diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 new file mode 100644 index 00000000..24782087 --- /dev/null +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -0,0 +1,20 @@ +echo '### zsh' +### zsh + ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' +function from zsh to zsh OK + ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK' +function from zsh to bash OK +echo '### csh' +### csh + echo "3 big vars run remotely - length(base64) > 1000" +3 big vars run remotely - length(base64) > 1000 + ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' + 1 200 692 + 1 200 692 + 1 150 547 + echo "3 big vars run locally" +3 big vars run locally + ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' + 1 200 692 + 1 200 692 + 1 150 547 diff --git a/testsuite/wanted-results/parallel-remote1 b/testsuite/wanted-results/parallel-remote1 index 4d8002b9..e8b2e96f 100644 --- a/testsuite/wanted-results/parallel-remote1 +++ b/testsuite/wanted-results/parallel-remote1 @@ -100,6 +100,18 @@ TODO test ssh with > 9 simultaneous 98 99 100 +echo '### test --timeout --retries' +### test --timeout --retries + parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 parallel@parallel-server3 parallel@lo parallel@parallel-server2 +parallel@parallel-server3 +parallel@lo +parallel@parallel-server2 +echo '### test --filter-hosts with server w/o ssh, non-existing server' +### test --filter-hosts with server w/o ssh, non-existing server + parallel -S 192.168.1.197,8.8.8.8,parallel@parallel-server3,parallel@lo,parallel@parallel-server2 --filter-hosts --nonall -k --tag echo +parallel@lo +parallel@parallel-server2 +parallel@parallel-server3 echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' ### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & wait @@ -113,17 +125,5 @@ echo '### test --workdir . in $HOME' ### test --workdir . in $HOME cd && mkdir -p parallel-test && cd parallel-test && echo OK > testfile && parallel --workdir . --transfer -S parallel@parallel-server3 cat {} ::: testfile OK -echo '### test --timeout --retries' -### test --timeout --retries - parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 parallel@parallel-server3 parallel@lo parallel@parallel-server2 -parallel@parallel-server3 -parallel@lo -parallel@parallel-server2 -echo '### test --filter-hosts with server w/o ssh, non-existing server' -### test --filter-hosts with server w/o ssh, non-existing server - parallel -S 192.168.1.197,8.8.8.8,parallel@parallel-server3,parallel@lo,parallel@parallel-server2 --filter-hosts --nonall -k --tag echo -parallel@lo -parallel@parallel-server2 -parallel@parallel-server3 echo '### Missing: test --filter-hosts proxied through the one host' ### Missing: test --filter-hosts proxied through the one host