Fixed bug #44148: csh long vars fail exporting.

src/parallel

@@ -6025,16 +6025,63 @@ sub sshlogin_wrap {
 
     my $self = shift;
     my $command = shift;
+    # TODO test that *sh -c 'parallel --env' use *sh
     if(not defined $self->{'sshlogin_wrap'}) {
 	my $sshlogin = $self->sshlogin();
-	my $sshcmd = $sshlogin->sshcommand();
 	my $serverlogin = $sshlogin->serverlogin();
-	my ($pre,$post,$cleanup)=("","","");
+	my $quoted_remote_command;
+	$ENV{'PARALLEL_SEQ'} = $self->seq();
+	$ENV{'PARALLEL_PID'} = $$;
 	if($serverlogin eq ":") {
-	    # No transfer/environment neeeded
+	    if(@opt::env) {
-	    # TODO Copy env between shells (zsh to bash)
+		# Prepend with environment setter, which sets functions in zsh
-	    $self->{'sshlogin_wrap'} = $command;
+		my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
+		my $env_command = $envset.$bashfuncset.
+		    '@ARGV="'.::perl_quote_scalar($command).'";'.
+		    "exec\"$Global::shell\",\"-c\",\(\$bashfunc.\"\@ARGV\"\)\;die\"exec:\$\!\\n\"\;";
+		if(length $env_command > 1000
+		   or
+		   not $csh_friendly
+		   or
+		   $command =~ /\n/) {
+		    # csh does not deal well with > 1000 chars in one word
+		    # csh does not deal well with $ENV with \n
+		    $env_command = "perl -e '".base64_zip_eval()."' ".
+			join" ",string_zip_base64($env_command);
+		    $self->{'sshlogin_wrap'} = $env_command;
+		} else {
+		    $self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($env_command);
+		}
+	    } else {
+		$self->{'sshlogin_wrap'} = $command;
+	    }
 	} else {
+	    my $pwd = "";
+	    if($opt::workdir) {
+		# Create remote workdir if needed. Then cd to it.
+	        my $wd = $self->workdir();
+		$pwd = qq{system("mkdir","-p","--","$wd"); chdir "$wd" ||}.
+		    qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;};
+	    }
+	    my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
+	    my $remote_command = $pwd.$envset.$bashfuncset.
+		'@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script();
+	    $quoted_remote_command = "perl -e ".::shell_quote_scalar($remote_command);
+	    if(length $quoted_remote_command > 1000
+	       or
+	       not $csh_friendly
+	       or
+	       $command =~ /\n/) {
+		# csh does not deal well with > 1000 chars in one word
+		# csh does not deal well with $ENV with \n
+		$quoted_remote_command = "perl -e \\''".base64_zip_eval()."'\\' ".
+		    join" ",string_zip_base64($remote_command);
+	    } else {
+		$quoted_remote_command = ::shell_quote_scalar($quoted_remote_command);
+	    }
+
+	    my $sshcmd = $sshlogin->sshcommand();
+	    my ($pre,$post,$cleanup)=("","","");
 	    # --transfer
 	    $pre .= $self->sshtransfer();
 	    # --return
@@ -6045,36 +6092,9 @@ sub sshlogin_wrap {
 		# We need to save the exit status of the job
 		$post = '_EXIT_status=$?; ' . $post . ' exit $_EXIT_status;';
 	    }
-	    # If the remote login shell is (t)csh then use 'setenv'
-	    # otherwise use 'export'
-	    # We cannot use parse_env_var(), as PARALLEL_SEQ changes
-	    # for each command
-	    $ENV{'PARALLEL_SEQ'} = $self->seq();
-	    $ENV{'PARALLEL_PID'} = $$;
-	    my $pwd = "";
-	    if($opt::workdir) {
-		# Create remote workdir if needed. Then cd to it.
-	        my $wd = $self->workdir();
-		$pwd = qq{system("mkdir","-p","--","$wd"); chdir "$wd" ||}.
-		    qq{print(STDERR "parallel: Cannot chdir to $wd\\n") && exit 255;};
-	    }
-	    my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
-	    my $remote_command = $pwd.$envset.$bashfuncset.
-	    '@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script();
-	    my $quoted_remote_command = "exec perl -e ".::shell_quote_scalar(::shell_quote_scalar($remote_command));
-	    if(length $quoted_remote_command > 1000
-	       or
-	       not $csh_friendly
-	       or
-	       $command =~ "\n") {
-		# csh does not deal well with > 1000 chars in one word
-		# csh does not deal well with $ENV with \n
-		$quoted_remote_command = "exec perl -e '".::shell_quote_scalar(base64_zip_eval())."' ".
-		    join" ",string_zip_base64($remote_command);
-	    }
 	    $self->{'sshlogin_wrap'} =
 		($pre
-		 . "$sshcmd $serverlogin "
+		 . "$sshcmd $serverlogin exec "
 		 . $quoted_remote_command
 		 . ";"
 		 . $post);

testsuite/tests-to-run/parallel-local-ssh2.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj6 -k --joblog /tmp/jl-`basename $0` -L1
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj5 -k --joblog /tmp/jl-`basename $0` -L1
 echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded"
   # Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed
   # Thus must take at least 25 sec to run

testsuite/tests-to-run/parallel-local-ssh3.sh

@@ -61,8 +61,4 @@ echo '### Uniq {=perlexpr=} in return - not used in command'
 
 #  Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
 
-echo '### zsh'
-  ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK'
-  ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK'
-
 EOF

testsuite/tests-to-run/parallel-local-ssh4.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# SSH only allowed to localhost/lo
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj8 -k --joblog /tmp/jl-`basename $0` -L1
+echo '### zsh'
+  ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; 
+              export fun; 
+              parallel --env fun fun ::: OK'
+
+  ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; 
+              export fun; 
+              parallel -S parallel@lo --env fun fun ::: OK'
+
+echo '### csh'
+  echo "3 big vars run remotely - length(base64) > 1000"
+  ssh csh@lo 'setenv A `seq 200|xargs`; 
+              setenv B `seq 200 -1 1|xargs`; 
+              setenv C `seq 300 -2 1|xargs`; 
+              parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+
+  echo "3 big vars run locally"
+  ssh csh@lo 'setenv A `seq 200|xargs`; 
+              setenv B `seq 200 -1 1|xargs`; 
+              setenv C `seq 300 -2 1|xargs`; 
+              parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+
+EOF

testsuite/tests-to-run/parallel-remote1.sh

@@ -11,9 +11,15 @@ echo 'TODO test ssh with > 9 simultaneous'
 echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1
 echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2
 chmod 755 /tmp/myssh1 /tmp/myssh2
-seq 1 100 | parallel --sshdelay 0.01 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
+seq 1 100 | parallel --sshdelay 0.03 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
+
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj3 -k -L1
+echo '### test --timeout --retries'
+  parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 $SSHLOGIN1 $SSHLOGIN2 $SSHLOGIN3
+
+echo '### test --filter-hosts with server w/o ssh, non-existing server'
+  parallel -S 192.168.1.197,8.8.8.8,$SSHLOGIN1,$SSHLOGIN2,$SSHLOGIN3 --filter-hosts --nonall -k --tag echo
 
-cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj1 -k -L1
 echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host'
   (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & 
   (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & 
@@ -26,13 +32,8 @@ echo '### test --workdir . in $HOME'
   cd && mkdir -p parallel-test && cd parallel-test && 
     echo OK > testfile && parallel --workdir . --transfer -S $SSHLOGIN1 cat {} ::: testfile
 
-echo '### test --timeout --retries'
-  parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 $SSHLOGIN1 $SSHLOGIN2 $SSHLOGIN3
-
-echo '### test --filter-hosts with server w/o ssh, non-existing server'
-  parallel -S 192.168.1.197,8.8.8.8,$SSHLOGIN1,$SSHLOGIN2,$SSHLOGIN3 --filter-hosts --nonall -k --tag echo
-
 echo '### Missing: test --filter-hosts proxied through the one host'
 
 EOF
-rm /tmp/myssh1 /tmp/myssh2
+rm /tmp/myssh1 /tmp/myssh2 /tmp/myssh1-run /tmp/myssh2-run
+

testsuite/wanted-results/parallel-local-ssh3

@@ -101,9 +101,3 @@ echo '### Uniq {=perlexpr=} in return - not used in command'
   cd /tmp;   rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr;   echo local > parallel_perlexpr;   parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr;   cat /tmp/parallel_perlexpr.2Parallel_PerlexPr;   rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
 remote OK
 #  Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
-echo '### zsh'
-### zsh
-  ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK'
-function from zsh to zsh OK
-  ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK'
-function from zsh to bash OK

testsuite/wanted-results/parallel-local-ssh4

@@ -0,0 +1,20 @@
+echo '### zsh'
+### zsh
+  ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }";               export fun;               parallel --env fun fun ::: OK'
+function from zsh to zsh OK
+  ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }";               export fun;               parallel -S parallel@lo --env fun fun ::: OK'
+function from zsh to bash OK
+echo '### csh'
+### csh
+  echo "3 big vars run remotely - length(base64) > 1000"
+3 big vars run remotely - length(base64) > 1000
+  ssh csh@lo 'setenv A `seq 200|xargs`;               setenv B `seq 200 -1 1|xargs`;               setenv C `seq 300 -2 1|xargs`;               parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+      1     200     692
+      1     200     692
+      1     150     547
+  echo "3 big vars run locally"
+3 big vars run locally
+  ssh csh@lo 'setenv A `seq 200|xargs`;               setenv B `seq 200 -1 1|xargs`;               setenv C `seq 300 -2 1|xargs`;               parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+      1     200     692
+      1     200     692
+      1     150     547

testsuite/wanted-results/parallel-remote1

@@ -100,6 +100,18 @@ TODO test ssh with > 9 simultaneous
 98
 99
 100
+echo '### test --timeout --retries'
+### test --timeout --retries
+  parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 parallel@parallel-server3 parallel@lo parallel@parallel-server2
+parallel@parallel-server3
+parallel@lo
+parallel@parallel-server2
+echo '### test --filter-hosts with server w/o ssh, non-existing server'
+### test --filter-hosts with server w/o ssh, non-existing server
+  parallel -S 192.168.1.197,8.8.8.8,parallel@parallel-server3,parallel@lo,parallel@parallel-server2 --filter-hosts --nonall -k --tag echo
+parallel@lo	
+parallel@parallel-server2	
+parallel@parallel-server3	
 echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host'
 ### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host
   (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) &   (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) &   wait
@@ -113,17 +125,5 @@ echo '### test --workdir . in $HOME'
 ### test --workdir . in $HOME
   cd && mkdir -p parallel-test && cd parallel-test &&     echo OK > testfile && parallel --workdir . --transfer -S parallel@parallel-server3 cat {} ::: testfile
 OK
-echo '### test --timeout --retries'
-### test --timeout --retries
-  parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 parallel@parallel-server3 parallel@lo parallel@parallel-server2
-parallel@parallel-server3
-parallel@lo
-parallel@parallel-server2
-echo '### test --filter-hosts with server w/o ssh, non-existing server'
-### test --filter-hosts with server w/o ssh, non-existing server
-  parallel -S 192.168.1.197,8.8.8.8,parallel@parallel-server3,parallel@lo,parallel@parallel-server2 --filter-hosts --nonall -k --tag echo
-parallel@lo	
-parallel@parallel-server2	
-parallel@parallel-server3	
 echo '### Missing: test --filter-hosts proxied through the one host'
 ### Missing: test --filter-hosts proxied through the one host