Fixed bug #42913: Dont use $SHELL but the shell currently running

This commit is contained in:
Ole Tange 2014-08-05 11:37:58 +02:00
parent f613b12721
commit d02f31b05c
4 changed files with 308 additions and 69 deletions

56
doc/boxplot-runtime Normal file
View file

@ -0,0 +1,56 @@
#!/bin/bash
# Create a boxplot of running 1000 jobs 10 times on all released
# versions of GNU Parallel
# test
# --cpus-as-cores (virker ikke)
# hvor stor forskel mlm 2 run
# hvor stor forskel mlm 1000/3000/10000
# hvor stor forskel mlm 10/30/100
# Non-fixed cpu-speed: 50% spread=1-2 ms
# Fixed cpu-speed: 50% spread=0.7-1.5 ms
# 4-cpu: 30% faster: 9 ms -> 6 ms
if ! /tmp/bin/parallel-20140722 --version; then
wget -c ftp://ftp.uni-kl.de/pub/gnu/parallel/p*
parallel 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve {}' ::: *.sig
parallel --plus 'tar xvf {.} && cd {...} && ./configure --prefix /tmp/{.}-bin && make && make install' ::: *sig
perl -i -pe 's/qw\(keys/(keys/' parallel*/src/parallel
mkdir /tmp/bin
parallel cp {} /tmp/bin/'{=s:/.*::=}' ::: parallel*/src/parallel
fi
measure() {
# 100: Much jumping
# 300: Same sort order every time
# 1000: Same sort order every time
OUTER=$1
INNER=$2
CORES=$3
VERSION=$4
# Force cpuspeed at 1.7GHz
forever 'parallel sudo cpufreq-set -f 1700MHz -c{} ::: {0..7}' &
PATH=/tmp/bin:$PATH
cd /tmp/bin
ls parallel-* | shuf | parallel -j$CORES --joblog /tmp/joblog.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) -
killall forever
Rscript - <<_
jl<-read.csv("/tmp/joblog.csv",sep="\t");
jl\$Command <- as.factor(substr(jl\$Command, 13, nchar(as.character(jl\$Command))-5))
pdf("/tmp/boxplot.pdf");
par(cex.axis=0.5);
boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F,
ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER");
_
cp /tmp/boxplot.pdf /tmp/boxplot-j$CORES-1.7ghz-$OUTER-${INNER}v$VERSION.pdf
evince /tmp/boxplot.pdf
}
measure 300 1000 8 1

View file

@ -34,11 +34,6 @@ use Getopt::Long;
use strict;
use File::Basename;
if(not $ENV{SHELL}) {
# $ENV{SHELL} is sometimes not set on Mac OS X and Windows
::warning("\$SHELL not set. Using /bin/sh.\n");
$ENV{SHELL} = "/bin/sh";
}
if(not $ENV{HOME}) {
# $ENV{HOME} is sometimes not set if called from PHP
::warning("\$HOME not set. Using /tmp\n");
@ -780,6 +775,7 @@ sub parse_options {
if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2
$Global::debug = $opt::D;
$Global::shell = parent_shell($$) || $ENV{'SHELL'} || "/bin/sh";
if(defined $opt::X) { $Global::ContextReplace = 1; }
if(defined $opt::silent) { $Global::verbose = 0; }
if(defined $opt::keeporder) { $Global::keeporder = 1; }
@ -1066,8 +1062,8 @@ sub parse_env_var {
my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
if(@bash_functions) {
# Functions are not supported for all shells
if($ENV{'SHELL'} !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
::warning("Shell functions may not be supported in $ENV{'SHELL'}\n");
if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
::warning("Shell functions may not be supported in $Global::shell\n");
}
}
push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_functions;
@ -1166,15 +1162,6 @@ sub find_compression_program {
return ("cat","cat");
}
sub which {
# Input:
# $program = program to find the path to
# Returns:
# $full_path = full path to $program. undef if not found
my $program = $_[0];
return (grep { -e $_."/".$program } split(":",$ENV{'PATH'}))[0];
}
sub read_options {
# Read options from command line, profile and $PARALLEL
@ -2833,6 +2820,116 @@ sub hostname {
return $Private::hostname;
}
sub which {
# Input:
# @programs = programs to find the path to
# Returns:
# @full_path = full paths to @programs. Nothing if not found
my @which;
for my $prg (@_) {
push @which, map { $_."/".$prg } grep { -x $_."/".$prg } split(":",$ENV{'PATH'});
}
return @which;
}
{
my ($regexp,%fakename);
sub parent_shell {
# Input:
# $pid = pid to see if (grand)*parent is a shell
# Returns:
# $shellpath = path to shell - undef if no shell found
my $pid = shift;
if(not $regexp) {
# All shells known to mankind
#
# ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh
# posh rbash rush rzsh sash sh static-sh tcsh yash zsh
my @shells = qw(ash bash csh dash fdsh fish fizsh ksh
ksh93 mksh pdksh posh rbash rush rzsh
sash sh static-sh tcsh yash zsh -sh -csh);
# Can be formatted as:
# [sh] -sh sh busybox sh
# /bin/sh /sbin/sh /opt/csw/sh
# NOT: foo.sh sshd crash flush pdflush scosh fsflush ssh
my $shell = "(?:".join("|",@shells).")";
$regexp = '^((\[)('. $shell. ')(\])|(|\S+/|busybox )('. $shell. '))($| )';
%fakename = (
# csh and tcsh disguise themselves as -sh/-csh
"-sh" => ["csh", "tcsh"],
"-csh" => ["tcsh", "csh"],
);
}
my ($children_of_ref, $parent_of_ref, $name_of_ref) = pid_table();
my $shellpath;
my $testpid = $pid;
while($testpid) {
::debug("init", "shell? ". $name_of_ref->{$testpid}."\n");
if($name_of_ref->{$testpid} =~ /$regexp/o) {
::debug("init", "which $3$6 => ");
$shellpath = (which($3.$6,@{$fakename{$3.$6}}))[0];
::debug("init", "shell path $shellpath\n");
$shellpath and last;
}
$testpid = $parent_of_ref->{$testpid};
}
return $shellpath;
}
}
{
my %pid_parentpid_cmd;
sub pid_table {
# return two tables:
# pid -> children of pid
# pid -> pid of parent
# pid -> commandname
if(not %pid_parentpid_cmd) {
# Filter for SysV-style `ps`
my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
q(s/^.{$s}//; print "@F[1,2] $_"' );
# BSD-style `ps`
my $bsd = q(ps -o pid,ppid,command -ax);
# TODO test these on Cygwin, darwin
%pid_parentpid_cmd =
(
'aix' => $sysv,
'cygwin' => $sysv,
'dec_osf' => $sysv,
'darwin' => $bsd,
'dragonfly' => $bsd,
'freebsd' => $bsd,
'gnu' => $sysv,
'hpux' => $sysv,
'linux' => $sysv,
'mirbsd' => $bsd,
'netbsd' => $bsd,
'nto' => $sysv,
'openbsd' => $bsd,
'solaris' => $sysv,
'svr5' => $sysv,
);
}
$pid_parentpid_cmd{$^O} or ::die_bug("pid_parentpid_cmd for $^O missing");
my (@pidtable,%parent_of,%children_of,%name_of);
# Table with pid -> children of pid
@pidtable = `$pid_parentpid_cmd{$^O}`;
my $p=$$;
for (@pidtable) {
# must match: 24436 21224 busybox ash
/(\S+)\s+(\S+)\s+(\S+.*)/ or ::die_bug("pidtable format: $_");
$parent_of{$1} = $2;
push @{$children_of{$2}}, $1;
$name_of{$1} = $3;
}
return(\%children_of, \%parent_of, \%name_of);
}
}
sub reap_usleep {
# Reap dead children.
# If no dead children: Sleep specified amount with exponential backoff
@ -4918,45 +5015,7 @@ sub kill {
}
}
{
my %pid_parentpid_cmd;
sub pid_table {
# return two tables:
# pid -> children of pid
# pid -> commandname
%pid_parentpid_cmd or %pid_parentpid_cmd =
(
'aix' => q( ps -ef | awk '{print $2" "$3}' ),
'cygwin' => q( ps -ef | awk '{print $2" "$3}' ),
'dec_osf' => q( ps -ef | awk '{print $2" "$3}' ),
'darwin' => q( ps -o pid,ppid -ax ),
'dragonfly' => q( ps -o pid,ppid -ax ),
'freebsd' => q( ps -o pid,ppid -ax ),
'gnu' => q( ps -ef | awk '{print $2" "$3}' ),
'hpux' => q( ps -ef | awk '{print $2" "$3}' ),
'linux' => q( ps -ef | grep tange | awk '{print $2" "$3" "$8}' ),
'mirbsd' => q( ps -o pid,ppid -ax ),
'netbsd' => q( ps -o pid,ppid -ax ),
'nto' => q( ps -ef | awk '{print $2" "$3}' ),
'openbsd' => q( ps -o pid,ppid -ax ),
'solaris' => q( ps -ef | awk '{print $2" "$3}' ),
'svr5' => q( ps -ef | awk '{print $2" "$3}' ),
);
$pid_parentpid_cmd{$^O} or ::die_bug("pid_parentpid_cmd for $^O missing");
my (@pidtable,%children_of,%name_of);
# Table with pid -> children of pid
@pidtable = `$pid_parentpid_cmd{$^O}`;
for (@pidtable) {
/(\S+)\s+(\S+)\s+(\S+)/ or ::die_bug("pidtable format: $_");
push @{$children_of{$2}}, $1;
$name_of{$2} = $3;
}
return(\%children_of, \%name_of);
}
}
sub family_pids {
# Find the pids with this->pid as (grand)*parent
@ -4964,7 +5023,7 @@ sub family_pids {
my $pid = $self->pid();
my @pids;
my ($children_of_ref,$name_of_ref) = pid_table();
my ($children_of_ref, $parent_of_ref, $name_of_ref) = ::pid_table();
my @more = ($pid);
# While more (grand)*children
@ -5415,7 +5474,7 @@ sub start {
# The eval is needed to catch exception from open3
eval {
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $Global::shell, "-c", $command) ||
::die_bug("open3-pipe");
1;
};
@ -5430,7 +5489,7 @@ sub start {
$command = $job->tmux_wrap($command);
}
eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) ||
::die_bug("open3-a");
1;
};
@ -5446,7 +5505,7 @@ sub start {
$command = $job->tmux_wrap($command);
}
eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) ||
::die_bug("open3-/dev/tty");
$Global::tty_taken = $pid;
close $devtty_fh;
@ -5457,7 +5516,7 @@ sub start {
$command = $job->tmux_wrap($command);
}
eval {
$pid = ::open3(::gensym, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
$pid = ::open3(::gensym, ">&OUT", ">&ERR", $Global::shell, "-c", $command) ||
::die_bug("open3-gensym");
1;
};
@ -6230,9 +6289,9 @@ sub replaced {
# Prepend \nice -n19 $SHELL -c
# and quote
# \ before nice is needed to avoid tcsh's built-in
$self->{'replaced'} = '\nice' ." -n" . $opt::nice . " "
. $ENV{SHELL}." -c "
. ::shell_quote_scalar($self->{'replaced'});
$self->{'replaced'} = '\nice'. " -n". $opt::nice. " ".
$Global::shell. " -c ".
::shell_quote_scalar($self->{'replaced'});
}
if($opt::shellquote) {
# Prepend echo

View file

@ -40,9 +40,6 @@ echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop';
echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop (II)';
echo a | parallel -qX echo "'{}'"
echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL';
seq 1 2 | SHELL=tcsh MANPATH=. stdout parallel -k --nice 8 setenv a b\;echo \$SHELL
echo '### bug #42041: Implement $PARALLEL_JOBSLOT'
parallel -k --slotreplace // -j2 sleep 1\;echo // ::: {1..4}
parallel -k -j2 sleep 1\;echo {%} ::: {1..4}
@ -69,6 +66,19 @@ echo '### bug #42902: profiles containing arguments with space'
echo '### bug #42892: parallel -a nonexiting --pipepart'
parallel --pipepart -a nonexisting wc
echo '### bug #42913: Dont use $SHELL but the shell currently running'
echo '## Unknown shell => $SHELL (bash)'
parallel -j1 "cp \`which {}\` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;"
::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
echo '## Known shells -c'
parallel -k "\`which {}\` -c 'parallel -Dinit echo ::: 1' | grep which;"
::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
echo '## Known shells |'
parallel -k "echo 'parallel -Dinit echo ::: 1' | \`which {}\` | grep which;"
::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
echo '## Started directly from perl'
perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which
EOF
# TODO This is too unstable

View file

@ -29,10 +29,6 @@ echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop';
echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop (II)'; echo a | parallel -qX echo "'{}'"
### Bug before 2009-08-26 causing regexp compile error or infinite loop (II)
'a'
echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL'; seq 1 2 | SHELL=tcsh MANPATH=. stdout parallel -k --nice 8 setenv a b\;echo \$SHELL
### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL
tcsh 1
tcsh 2
echo '### bug #42041: Implement $PARALLEL_JOBSLOT'
### bug #42041: Implement $PARALLEL_JOBSLOT
parallel -k --slotreplace // -j2 sleep 1\;echo // ::: {1..4}
@ -89,3 +85,121 @@ echo '### bug #42892: parallel -a nonexiting --pipepart'
### bug #42892: parallel -a nonexiting --pipepart
parallel --pipepart -a nonexisting wc
parallel: Error: Cannot open input file `nonexisting': No such file or directory.
echo '### bug #42913: Dont use $SHELL but the shell currently running'
### bug #42913: Dont use $SHELL but the shell currently running
echo '## Unknown shell => $SHELL (bash)'
## Unknown shell => $SHELL (bash)
parallel -j1 "cp \`which {}\` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
shell? /bin/bash -c cp `which ash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which bash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which csh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which dash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which fish` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
which zsh => shell path /usr/bin/zsh
shell? /bin/bash -c cp `which ksh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which ksh93` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which mksh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which pdksh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which posh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which rbash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which rzsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which sash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which sh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which tcsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which yash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c cp `which zsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
MANPATH: Undefined variable.
/tmp/SHELL: -c: bad option(s)
Local configuration error occurred.
Contact the systems administrator for further assistance.
SHELL: applet not found
MANPATH: Undefined variable.
echo '## Known shells -c'
## Known shells -c
parallel -k "\`which {}\` -c 'parallel -Dinit echo ::: 1' | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
which ash => shell path /bin/ash
shell? /bin/bash -c `which bash` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
which csh => shell path /bin/csh
which dash => shell path /bin/dash
which fdsh => shell path /usr/bin/fdsh
which fish => shell path /usr/bin/fish
which zsh => shell path /usr/bin/zsh
shell? /bin/bash -c `which ksh` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c `which ksh93` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c `which mksh` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c `which pdksh` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
which posh => shell path /bin/posh
shell? /bin/bash -c `which rbash` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c `which rzsh` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
which sash => shell path /bin/sash
which sh => shell path /bin/sh
which static-sh => shell path /bin/static-sh
which tcsh => shell path /usr/bin/tcsh
shell? /bin/bash -c `which yash` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
shell? /bin/bash -c `which zsh` -c 'parallel -Dinit echo ::: 1' | grep which;
which bash => shell path /bin/bash
MANPATH: Undefined variable.
MANPATH: Undefined variable.
Local configuration error occurred.
Contact the systems administrator for further assistance.
MANPATH: Undefined variable.
MANPATH: Undefined variable.
echo '## Known shells |'
## Known shells |
parallel -k "echo 'parallel -Dinit echo ::: 1' | \`which {}\` | grep which;" ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh
which ash => shell path /bin/ash
which bash => shell path /bin/bash
which csh => shell path /bin/csh
which dash => shell path /bin/dash
which fdsh => shell path /usr/bin/fdsh
which fish => shell path /usr/bin/fish
which zsh => shell path /usr/bin/zsh
which ksh => shell path /usr/bin/ksh
which ksh93 => shell path /bin/ksh93
which mksh => shell path /bin/mksh
which pdksh => shell path /bin/pdksh
which posh => shell path /bin/posh
which rbash => shell path /bin/rbash
which rzsh => shell path /bin/rzsh
which sash => shell path /bin/sash
which sh => shell path /bin/sh
which static-sh => shell path /bin/static-sh
which tcsh => shell path /usr/bin/tcsh
which yash => shell path /usr/bin/yash
which zsh => shell path /usr/bin/zsh
MANPATH: Undefined variable.
MANPATH: Undefined variable.
Local configuration error occurred.
Contact the systems administrator for further assistance.
MANPATH: Undefined variable.
MANPATH: Undefined variable.
echo '## Started directly from perl'
## Started directly from perl
perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which
shell? /bin/bash -c perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which
which bash => shell path /bin/bash