diff --git a/src/parallel b/src/parallel index c8984c65..b9765cf0 100755 --- a/src/parallel +++ b/src/parallel @@ -1124,7 +1124,6 @@ sub parse_options { parse_halt(); parse_sshlogin(); - parse_env_var(); if(remote_hosts() and ($opt::X or $opt::m or $opt::xargs)) { # As we do not know the max line length on the remote machine @@ -1195,7 +1194,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20160523; + $Global::version = 20160529; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1425,30 +1424,6 @@ sub record_env { } } -sub parse_env_var { - # Parse --env and set $Global::envvar, $Global::envwarn and $Global::envvarlen - # - # Bash functions must be parsed to export them remotely - # Pre-shellshock style bash function: - # myfunc=() {... - # Post-shellshock style bash function (v1): - # BASH_FUNC_myfunc()=() {... - # Post-shellshock style bash function (v2): - # BASH_FUNC_myfunc%%=() {... - # - # Uses: - # $Global::envvar = eval string that will set variables in both bash and csh - # $Global::envwarn = If functions are used: Give warning in csh - # $Global::envvarlen = length of $Global::envvar - # @opt::env - # $Global::shell - # %ENV - # Returns: N/A - $Global::envvar = ""; - - $Global::envvarlen = length $Global::envvar; -} - sub open_joblog { # Open joblog as specified by --joblog # Uses: @@ -2060,7 +2035,6 @@ sub __RUNNING_THE_JOBS_AND_PRINTING_PROGRESS__ {} # $Global::start_no_new_jobs = should more jobs be started? # $Global::original_stderr = file handle for STDERR when the program started # $Global::total_started = total number of jobs started -# $Global::envvar = string to set the shell environment variables # $Global::joblog = filehandle of joblog # $Global::debug = Is debugging on? # $Global::exitstatus = status code of GNU Parallel @@ -3113,7 +3087,6 @@ sub parse_host_filtering { sub parallelized_host_filtering { # Uses: - # $Global::envvar # %Global::host # Returns: # text entries with: @@ -3135,7 +3108,6 @@ sub parallelized_host_filtering { } my(@cores, @cpus, @maxline, @echo); - my $envvar = ::shell_quote_scalar($Global::envvar); while (my ($host, $sshlogin) = each %Global::host) { if($host eq ":") { next } # The 'true' is used to get the $host out later @@ -3150,28 +3122,34 @@ sub parallelized_host_filtering { $sshlogin->serverlogin(); push(@echo, $host."\t".$sshcmd." -- echo\n\0"); } - my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh"); - print $fh @cores, @cpus, @maxline, @echo; - close $fh; + # --timeout 10: Setting up an SSH connection and running a simple # command should never take > 10 sec. # --delay 0.1: If multiple sshlogins use the same proxy the delay # will make it less likely to overload the ssh daemon. # --retries 3: If the ssh daemon it overloaded, try 3 times - # -s 16000: Half of the max line on UnixWare - my $unlinkcmd = $Global::debug ? "true" : "rm $tmpfile"; - my $cmd = "($unlinkcmd; cat -) < $tmpfile | ". + my $cmd = "$0 -j0 --timeout 10 --joblog - --plain --delay 0.1 --retries 3 ". - "--tag --tagstring '{1}' -0 --colsep '\t' -k eval '{2}' "; + "--tag --tagstring '{1}' -0 --colsep '\t' -k eval '{2}' && true "; $cmd = $Global::shell." -c ".::shell_quote_scalar($cmd); ::debug("init", $cmd, "\n"); my @out; my $prepend = ""; - open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd"); + + my ($host_fh,$in,$err); + open3($in, $host_fh, $err, $cmd) || ::die_bug("parallel host check: $cmd"); + if(not fork()) { + # Give the commands to run to the $cmd + close $host_fh; + print $in @cores, @cpus, @maxline, @echo; + close $in; + exit(); + } + close $in; for(<$host_fh>) { if(/\'$/) { # if last char = ' then append next line - # This may be due to quoting of $Global::envvar + # This may be due to quoting of \n in environment var $prepend .= $_; next; } @@ -4559,8 +4537,7 @@ sub memfree_recompute { $perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}'; } $perlscript =~ s/[\t\n ]+/ /g; - $perlscript = "perl -e " . ::shell_quote_scalar($perlscript); - $script = $Global::envvar. " " .$perlscript; + $script = "perl -e " . ::shell_quote_scalar($perlscript); } return $script } @@ -4745,8 +4722,7 @@ sub swap_activity { $perlscript .= 'if($^O eq "'.$os.'") { print `'.$vmstat{$os}[0].' | awk "{print ' . $vmstat{$os}[1] . '}"` }'; } - $perlscript = "perl -e " . ::shell_quote_scalar($perlscript); - $script = $Global::envvar. " " .$perlscript; + $script = "perl -e " . ::shell_quote_scalar($perlscript); } return $script; } @@ -5337,12 +5313,11 @@ sub ncpus { } } else { my $ncpu; - my $sqe = ::shell_quote_scalar($Global::envvar); if($opt::use_cpus_instead_of_cores) { - $ncpu = ::qqx("echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cpus"); + $ncpu = ::qqx("echo|$sshcmd $serverlogin -- parallel --number-of-cpus"); } else { - ::debug("init",qq(echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores\n)); - $ncpu = ::qqx("echo|$sshcmd $serverlogin -- $sqe parallel --number-of-cores"); + ::debug("init",qq(echo|$sshcmd $serverlogin -- parallel --number-of-cores\n)); + $ncpu = ::qqx("echo|$sshcmd $serverlogin -- parallel --number-of-cores"); } chomp $ncpu; if($ncpu =~ /^\s*[0-9]+\s*$/s) { @@ -6815,7 +6790,6 @@ sub wrapped { # * --nice/--cat/--fifo should be done on the remote machine # * --pipepart/--pipe should be done on the local machine inside --tmux # Uses: - # $Global::envvar # $opt::shellquote # $opt::nice # $Global::shell @@ -8642,7 +8616,7 @@ sub len { $len *= 4; } # If we are using --env, add the prefix for that, too. - $len += $Global::envvarlen; + $len += 0; return $len; } diff --git a/testsuite/input-files/shebang/shebangwrap.gp b/testsuite/input-files/shebang/shebangwrap.gp index 84d56b0b..050cef56 100755 --- a/testsuite/input-files/shebang/shebangwrap.gp +++ b/testsuite/input-files/shebang/shebangwrap.gp @@ -1,4 +1,4 @@ -#!/usr/bin/parallel --shebang-wrap -k A={} /usr/bin/gnuplot +#!/usr/local/bin/parallel --shebang-wrap -k A={} /usr/bin/gnuplot name=system("echo $A") print name diff --git a/testsuite/input-files/shebang/shebangwrap.oct b/testsuite/input-files/shebang/shebangwrap.oct index e45bbfd2..99630f3d 100755 --- a/testsuite/input-files/shebang/shebangwrap.oct +++ b/testsuite/input-files/shebang/shebangwrap.oct @@ -1,4 +1,4 @@ -#!/usr/bin/parallel --shebang-wrap -k /usr/bin/octave -qf +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/octave -qf arg_list = argv (); filename = arg_list{1}; diff --git a/testsuite/input-files/shebang/shebangwrap.pl b/testsuite/input-files/shebang/shebangwrap.pl index 5970265e..b9fde22b 100755 --- a/testsuite/input-files/shebang/shebangwrap.pl +++ b/testsuite/input-files/shebang/shebangwrap.pl @@ -1,3 +1,3 @@ -#!/usr/bin/parallel --shebang-wrap -k /usr/bin/perl +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl print @ARGV,"\n"; diff --git a/testsuite/input-files/shebang/shebangwrap.py b/testsuite/input-files/shebang/shebangwrap.py index 07dbeeb6..b01b64d5 100755 --- a/testsuite/input-files/shebang/shebangwrap.py +++ b/testsuite/input-files/shebang/shebangwrap.py @@ -1,4 +1,4 @@ -#!/usr/bin/parallel --shebang-wrap -k /usr/bin/python +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/python import sys diff --git a/testsuite/input-files/shebang/shebangwrap.r b/testsuite/input-files/shebang/shebangwrap.r index 6b47943e..27cb1c9d 100755 --- a/testsuite/input-files/shebang/shebangwrap.r +++ b/testsuite/input-files/shebang/shebangwrap.r @@ -1,4 +1,4 @@ -#!/usr/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave options <- commandArgs(trailingOnly = TRUE) options diff --git a/testsuite/input-files/shebang/shebangwrap.rb b/testsuite/input-files/shebang/shebangwrap.rb index 77397c5b..297b1de6 100755 --- a/testsuite/input-files/shebang/shebangwrap.rb +++ b/testsuite/input-files/shebang/shebangwrap.rb @@ -1,3 +1,3 @@ -#!/usr/bin/parallel --shebang-wrap -k /usr/bin/ruby +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ruby p ARGV diff --git a/testsuite/input-files/shebang/shebangwrap.sh b/testsuite/input-files/shebang/shebangwrap.sh index df7bdc31..c2516531 100755 --- a/testsuite/input-files/shebang/shebangwrap.sh +++ b/testsuite/input-files/shebang/shebangwrap.sh @@ -1,3 +1,3 @@ -#!/usr/bin/parallel --shebang-wrap -k /bin/sh +#!/usr/local/bin/parallel --shebang-wrap -k /bin/sh echo "$@" diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 6b66e576..b6fb3fba 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -215,7 +215,7 @@ echo '**' echo 'bug #47002: --tagstring with -d \n\n' - (seq 3;echo;seq 4) | parallel -d '\n\n' --tagstring {%} echo ABC';'echo + (seq 3;echo;seq 4) | parallel -k -d '\n\n' --tagstring {%} echo ABC';'echo echo '**' diff --git a/testsuite/tests-to-run/parallel-local-ssh3.sh b/testsuite/tests-to-run/parallel-local-ssh3.sh index 7da05db8..25934939 100644 --- a/testsuite/tests-to-run/parallel-local-ssh3.sh +++ b/testsuite/tests-to-run/parallel-local-ssh3.sh @@ -3,7 +3,7 @@ # SSH only allowed to localhost/lo cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 echo '### --hostgroup force ncpu' - parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.3{} ::: {1..8} | sort + parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8} | sort echo '### --hostgroup two group arg' parallel -k --sshdelay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.3{} ::: {1..8}@g1+g2 | sort @@ -90,7 +90,7 @@ echo 'bug #47695: How to set $PATH on remote?' rm -rf /tmp/parallel cp /usr/local/bin/parallel /tmp - cat <<'_EOS' | stdout ssh nopathbash@lo -T + cat <<'_EOS' | stdout ssh nopathbash@lo -T | grep -v 'packages can be updated' echo BASH Path before: $PATH with no parallel parallel echo ::: 1 echo '^^^^^^^^ Not found is OK' @@ -103,7 +103,7 @@ echo 'bug #47695: How to set $PATH on remote?' _EOS echo - cat <<'_EOS' | stdout ssh nopathcsh@lo -T + cat <<'_EOS' | stdout ssh nopathcsh@lo -T | grep -v 'packages can be updated' echo CSH Path before: $PATH with no parallel which parallel >& /dev/stdout echo '^^^^^^^^ Not found is OK' diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index c5c6bd4a..b7c9d6f5 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -7,7 +7,7 @@ par_tmux_filter() { export -f par_tmux_filter par_tmux() { - (stdout parallel --timeout 3 --tmux --delay .4 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter + (stdout parallel --timeout 3 --tmux --delay .03 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter } export -f par_tmux cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj3 --timeout 60 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 0abfd92b..c2f84009 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -400,7 +400,7 @@ echo '**' ** echo 'bug #47002: --tagstring with -d \n\n' bug #47002: --tagstring with -d \n\n - (seq 3;echo;seq 4) | parallel -d '\n\n' --tagstring {%} echo ABC';'echo + (seq 3;echo;seq 4) | parallel -k -d '\n\n' --tagstring {%} echo ABC';'echo 1 ABC 1 1 1 2 diff --git a/testsuite/wanted-results/parallel-local-ssh3 b/testsuite/wanted-results/parallel-local-ssh3 index b9ea5744..26ae8d63 100644 --- a/testsuite/wanted-results/parallel-local-ssh3 +++ b/testsuite/wanted-results/parallel-local-ssh3 @@ -1,6 +1,6 @@ echo '### --hostgroup force ncpu' ### --hostgroup force ncpu - parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.3{} ::: {1..8} | sort + parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8} | sort parallel parallel tange @@ -128,7 +128,6 @@ Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64) Welcome to Linux Mint * Documentation: http://www.linuxmint.com -195 packages can be updated. 0 updates are security updates. BASH Path before: /bin:/usr/bin with no parallel @@ -141,7 +140,6 @@ Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64) Welcome to Linux Mint * Documentation: http://www.linuxmint.com -195 packages can be updated. 0 updates are security updates. Warning: no access to tty (Bad file descriptor).