tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-29 17:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #43567: --env + --filter_hosts + function fails

Browse source
This commit is contained in:
Ole Tange 2014-11-10 00:29:42 +01:00
parent ee6d07d49b
commit 8563f3f95a
6 changed files with 45 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/niceload.pod
View file

@ -7,7 +7,8 @@ niceload - slow down a program when the load average is above a certain limit
=head1 SYNOPSIS =head1 SYNOPSIS
B<niceload> [-v] [-h] [-n nice] [-I io] [-L load] [-M mem] [-N] B<niceload> [-v] [-h] [-n nice] [-I io] [-L load] [-M mem] [-N]
[--sensor program] [-t time] [-s time|-f factor] ( command | -p PID [-p PID ...] ) [--sensor program] [-t time] [-s time|-f factor]
( command | -p PID [-p PID ...] | --prg program )
=head1 DESCRIPTION =head1 DESCRIPTION
@ -31,6 +32,13 @@ run 1 second, suspend (3.00-1.00) seconds, run 1 second, suspend
=over 9 =over 9
=item B<-B> (alpha testing)
=item B<--battery> (alpha testing)
Suspend if the system is running on battery. Short hand for: -l -1 --sensor 'cat /sys/class/power_supply/BAT0/status /proc/acpi/battery/BAT0/state 2>/dev/null |grep -i -q discharging; echo $?'
=item B<-f> I<FACTOR> =item B<-f> I<FACTOR>
=item B<--factor> I<FACTOR> =item B<--factor> I<FACTOR>

20
src/parallel
View file

@ -2372,15 +2372,16 @@ sub cleanup_basefile {
sub filter_hosts { sub filter_hosts {
my(@cores, @cpus, @maxline, @echo); my(@cores, @cpus, @maxline, @echo);
my $envvar = ::shell_quote_scalar($Global::envvar);
while (my ($host, $sshlogin) = each %Global::host) { while (my ($host, $sshlogin) = each %Global::host) {
if($host eq ":") { next } if($host eq ":") { next }
# The 'true' is used to get the $host out later # The 'true' is used to get the $host out later
my $sshcmd = "true $host;" . $sshlogin->sshcommand()." ".$sshlogin->serverlogin(); my $sshcmd = "true $host;" . $sshlogin->sshcommand()." ".$sshlogin->serverlogin();
push(@cores, $host."\t".$sshcmd." ".$Global::envvar." parallel --number-of-cores\n"); push(@cores, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cores\n\0");
push(@cpus, $host."\t".$sshcmd." ".$Global::envvar." parallel --number-of-cpus\n"); push(@cpus, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cpus\n\0");
push(@maxline, $host."\t".$sshcmd." ".$Global::envvar." parallel --max-line-length-allowed\n"); push(@maxline, $host."\t".$sshcmd." ".$envvar." parallel --max-line-length-allowed\n\0");
# 'echo' is used to get the best possible value for an ssh login time # 'echo' is used to get the best possible value for an ssh login time
push(@echo, $host."\t".$sshcmd." echo\n"); push(@echo, $host."\t".$sshcmd." echo\n\0");
} }
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh"); my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh");
print $fh @cores, @cpus, @maxline, @echo; print $fh @cores, @cpus, @maxline, @echo;
@ -2391,11 +2392,20 @@ sub filter_hosts {
# will make it less likely to overload the ssh daemon. # will make it less likely to overload the ssh daemon.
# --retries 3: If the ssh daemon it overloaded, try 3 times # --retries 3: If the ssh daemon it overloaded, try 3 times
# -s 16000: Half of the max line on UnixWare # -s 16000: Half of the max line on UnixWare
my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} --colsep '\t' -k eval {2} 2>/dev/null"; my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null";
::debug("init", $cmd, "\n"); ::debug("init", $cmd, "\n");
open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd"); open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd");
my (%ncores, %ncpus, %time_to_login, %maxlen, %echo, @down_hosts); my (%ncores, %ncpus, %time_to_login, %maxlen, %echo, @down_hosts);
my $prepend = "";
while(<$host_fh>) { while(<$host_fh>) {
if(/\'$/) {
# if last char = ' then append next line
# This may be due to quoting of $Global::envvar
$prepend .= $_;
next;
}
$_ = $prepend . $_;
$prepend = "";
chomp; chomp;
my @col = split /\t/, $_; my @col = split /\t/, $_;
if(defined $col[6]) { if(defined $col[6]) {

7
testsuite/tests-to-run/niceload03.sh
View file

@ -9,16 +9,17 @@ int() {
export -f int export -f int
cat <<'EOF' | stdout parallel -k -vj0 -L1 cat <<'EOF' | stdout parallel -k -vj0 -L1
# The seq 10000000 should take > 1 cpu sec to run. # The seq 30000000 should take > 1 cpu sec to run.
echo '### --soft -f and test if child is actually suspended and thus takes longer' echo '### --soft -f and test if child is actually suspended and thus takes longer'
niceload --soft -f 0.5 'seq 20000000 | wc;echo This should finish last' & niceload --soft -f 0.5 'seq 30000000 | nice wc;echo This should finish last' &
(sleep 1; seq 20000000 | wc;echo This should finish first) & (sleep 1; seq 30000000 | nice wc;echo This should finish first) &
wait wait
echo '### niceload with no arguments should give no output' echo '### niceload with no arguments should give no output'
niceload niceload
echo '### Test -t and -s' echo '### Test -t and -s'
# This should sleep 2*1s and run 2*2s
niceload -v -t 1 -s 2 sleep 4.5 niceload -v -t 1 -s 2 sleep 4.5
echo 'bug #38908: niceload: Ctrl-C/TERM should resume jobs if using -p - Order may change, but not output' echo 'bug #38908: niceload: Ctrl-C/TERM should resume jobs if using -p - Order may change, but not output'

8
testsuite/tests-to-run/parallel-local-ssh1.sh
View file

@ -15,13 +15,13 @@ echo '### Stop if all hosts are filtered and there are no hosts left to run on'
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
echo '** test_zsh' echo '** test_zsh'
FOO=test_zsh parallel --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh parallel --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME'
echo '** test_zsh_filter' echo '** test_zsh_filter'
FOO=test_zsh_filter parallel --filter-hosts --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME'
echo '** test_csh' echo '** test_csh'
FOO=test_csh parallel --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh parallel --env FOO,HOME -S csh@lo env ::: "" |sort|egrep 'FOO|^HOME'
echo '** test_csh_filter' echo '** test_csh_filter'
FOO=test_csh_filter parallel --filter-hosts --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo env ::: "" |sort|egrep 'FOO|^HOME'
echo '** bug #41805 done' echo '** bug #41805 done'
echo '### Deal with long command lines on remote servers' echo '### Deal with long command lines on remote servers'

9
testsuite/wanted-results/niceload03
View file

@ -1,16 +1,17 @@
# The seq 10000000 should take > 1 cpu sec to run. # The seq 30000000 should take > 1 cpu sec to run.
echo '### --soft -f and test if child is actually suspended and thus takes longer' echo '### --soft -f and test if child is actually suspended and thus takes longer'
### --soft -f and test if child is actually suspended and thus takes longer ### --soft -f and test if child is actually suspended and thus takes longer
niceload --soft -f 0.5 'seq 20000000 | wc;echo This should finish last' & (sleep 1; seq 20000000 | wc;echo This should finish first) & wait niceload --soft -f 0.5 'seq 30000000 | nice wc;echo This should finish last' & (sleep 1; seq 30000000 | nice wc;echo This should finish first) & wait
20000000 20000000 168888897 30000000 30000000 258888897
This should finish first This should finish first
20000000 20000000 168888897 30000000 30000000 258888897
This should finish last This should finish last
echo '### niceload with no arguments should give no output' echo '### niceload with no arguments should give no output'
### niceload with no arguments should give no output ### niceload with no arguments should give no output
niceload niceload
echo '### Test -t and -s' echo '### Test -t and -s'
### Test -t and -s ### Test -t and -s
# This should sleep 2*1s and run 2*2s
niceload -v -t 1 -s 2 sleep 4.5 niceload -v -t 1 -s 2 sleep 4.5
Sleeping 1s Sleeping 1s
Running 2s Running 2s

16
testsuite/wanted-results/parallel-local-ssh1
View file

@ -12,24 +12,24 @@ echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
### bug #41805: Idea: propagate --env for parallel --number-of-cores ### bug #41805: Idea: propagate --env for parallel --number-of-cores
echo '** test_zsh' echo '** test_zsh'
** test_zsh ** test_zsh
FOO=test_zsh parallel --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh parallel --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_zsh FOO=test_zsh
PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin HOME=/home/tange
echo '** test_zsh_filter' echo '** test_zsh_filter'
** test_zsh_filter ** test_zsh_filter
FOO=test_zsh_filter parallel --filter-hosts --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_zsh_filter FOO=test_zsh_filter
PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin HOME=/home/tange
echo '** test_csh' echo '** test_csh'
** test_csh ** test_csh
FOO=test_csh parallel --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh parallel --env FOO,HOME -S csh@lo env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_csh FOO=test_csh
PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin HOME=/home/tange
echo '** test_csh_filter' echo '** test_csh_filter'
** test_csh_filter ** test_csh_filter
FOO=test_csh_filter parallel --filter-hosts --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_csh_filter FOO=test_csh_filter
PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin HOME=/home/tange
echo '** bug #41805 done' echo '** bug #41805 done'
** bug #41805 done ** bug #41805 done
echo '### Deal with long command lines on remote servers' echo '### Deal with long command lines on remote servers'