tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

env_parallel.{bash,ksh,pdksh,zsh}: Minimal --record-env.

Browse source
This commit is contained in:
Ole Tange 2016-11-06 12:29:48 +01:00
parent e458f35453
commit 3e440983e7
11 changed files with 112 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/env_parallel.bash
View file

@ -63,6 +63,13 @@ env_parallel() {
' -- "$@"
)"
# --record-env
if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
(compgen -a; compgen -A function; compgen -A variable) |
cat > $HOME/.parallel/ignored_vars
return 0
fi
# Grep alias names
local _alias_NAMES="$(compgen -a |
grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

7
src/env_parallel.ksh
View file

@ -63,6 +63,13 @@ env_parallel() {
' -- "$@"
)"
# --record-env
if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
(compgen -a; compgen -A function; compgen -A variable) |
cat > $HOME/.parallel/ignored_vars
return 0
fi
# Grep alias names
_alias_NAMES="$(alias | perl -pe 's/=.*//' |
grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

7
src/env_parallel.pdksh
View file

@ -63,6 +63,13 @@ env_parallel() {
' -- "$@"
)"
# --record-env
if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
(compgen -a; compgen -A function; compgen -A variable) |
cat > $HOME/.parallel/ignored_vars
return 0
fi
# Grep alias names
_alias_NAMES="$(alias | perl -pe 's/=.*//' |
grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

7
src/env_parallel.zsh
View file

@ -63,6 +63,13 @@ env_parallel() {
' -- "$@"
)"
# --record-env
if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
(compgen -a; compgen -A function; compgen -A variable) |
cat > $HOME/.parallel/ignored_vars
return 0
fi
# Grep alias names
_alias_NAMES="$(print -l ${(k)aliases} |
grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )"

82
src/parallel
View file

@ -74,40 +74,44 @@ if($opt::skip_first_line) {
my $fh = $input_source_fh[0];
<$fh>;
}
if($opt::header and not $opt::pipe) {
# split with colsep or \t
# $header force $colsep = \t if undef?
my $delimiter = $opt::colsep;
$delimiter ||= "\t";
# regexp for {=
my $left = "\Q$Global::parensleft\E";
my $l = $Global::parensleft;
# regexp for =}
my $right = "\Q$Global::parensright\E";
my $r = $Global::parensright;
my $id = 1;
for my $fh (@input_source_fh) {
my $line = <$fh>;
chomp($line);
::debug("init", "Delimiter: '$delimiter'");
for my $s (split /$delimiter/o, $line) {
::debug("init", "Colname: '$s'");
# Replace {colname} with {2}
for(@command,@Global::ret_files,@Global::transfer_files) {
s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
# {=header1 ... =} => {=1 ... =}
s:$left $s (.*?) $right:$l$id$1$r:gx;
set_input_source_header();
sub set_input_source_header {
if($opt::header and not $opt::pipe) {
# split with colsep or \t
# $header force $colsep = \t if undef?
my $delimiter = defined $opt::colsep ? $opt::colsep : "\t";
# regexp for {=
my $left = "\Q$Global::parensleft\E";
my $l = $Global::parensleft;
# regexp for =}
my $right = "\Q$Global::parensright\E";
my $r = $Global::parensright;
my $id = 1;
for my $fh (@input_source_fh) {
my $line = <$fh>;
chomp($line);
::debug("init", "Delimiter: '$delimiter'");
for my $s (split /$delimiter/o, $line) {
::debug("init", "Colname: '$s'");
# Replace {colname} with {2}
for(@command,@Global::ret_files,@Global::transfer_files) {
s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
# {=header1 ... =} => {=1 ... =}
s:$left $s (.*?) $right:$l$id$1$r:gx;
}
$Global::input_source_header{$id} = $s;
$id++;
}
$Global::input_source_header{$id} = $s;
}
} else {
my $id = 1;
for my $fh (@input_source_fh) {
$Global::input_source_header{$id} = $id;
$id++;
}
}
} else {
my $id = 1;
for my $fh (@input_source_fh) {
$Global::input_source_header{$id} = $id;
$id++;
}
}
if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
@ -187,10 +191,8 @@ if($Global::semaphore) {
}
$SIG{TERM} = \&start_no_new_jobs;
start_more_jobs();
if(not $opt::pipepart) {
if($opt::pipe) {
spreadstdin();
}
if($opt::pipe and not $opt::pipepart) {
spreadstdin();
}
::debug("init", "Start draining\n");
drain_job_queue();
@ -206,14 +208,10 @@ if($opt::pipe and @opt::a) {
}
}
::debug("init", "Cleaning\n");
cleanup();
if($Global::semaphore) {
$sem->release();
}
for(keys %Global::sshmaster) {
# If 'ssh -M's are running: kill them
kill "TERM", $_;
}
cleanup();
::debug("init", "Halt\n");
if($opt::halt and $Global::halt_when ne "never") {
if(not defined $Global::halt_exitstatus) {
@ -1820,6 +1818,10 @@ sub cleanup {
unlink keys %Global::unlink;
map { rmdir $_ } keys %Global::unlink;
if(@opt::basefile and $opt::cleanup) { cleanup_basefile(); }
for(keys %Global::sshmaster) {
# If 'ssh -M's are running: kill them
kill "TERM", $_;
}
}
sub __QUOTING_ARGUMENTS_FOR_SHELL__ {}
@ -9459,7 +9461,7 @@ sub new {
if($opt::sqlworker) {
# Open SQL table
$arg_sub_queue = SQLRecordQueue->new();
} elsif($colsep) {
} elsif(defined $colsep) {
# Open one file with colsep
$arg_sub_queue = RecordColQueue->new($fhs);
} else {

4
src/parallel.pod
View file

@ -807,7 +807,9 @@ soon,fail=1-99%
Use regexp as header. For normal usage the matched header (typically
the first line: B<--header '.*\n'>) will be split using B<--colsep>
(which will default to '\t') and column names can be used as
replacement variables: B<{column name}>.
replacement variables: B<{column name}>, B<{column name/}>, B<{column
name//}>, B<{column name/.}>, B<{column name.}>, B<{=column name perl
expression =}>, ..
For B<--pipe> the matched header will be prepended to each output.

4
testsuite/tests-to-run/parallel-freebsd.sh
View file

@ -27,8 +27,8 @@ echo 'bug #40135: FreeBSD: sem --fg does not finish under /bin/sh'
sem --fg 'sleep 1; echo The job finished'
echo 'bug #40133: FreeBSD: --round-robin gives no output'
jot 1000000 | parallel --round-robin --pipe -kj3 wc | sort
jot 1000000 | parallel --round-robin --pipe -kj4 wc | sort
jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc
jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc
echo 'bug #40134: FreeBSD: --shebang not working'
(echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang;

14
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -647,10 +647,22 @@ echo '### bug #48658: --linebuffer --files'
echo '**'
bug #49538: --header and {= =}
echo 'bug #49538: --header and {= =}'
parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
parallel --header : echo {var/.} ::: var sub/dir/file.ext
parallel --header : echo {var//} ::: var sub/dir/file.ext
parallel --header : echo {var/.} ::: var sub/dir/file.ext
parallel --header : echo {var/} ::: var sub/dir/file.ext
parallel --header : echo {var.} ::: var sub/dir/file.ext
echo '**'
echo 'bug --colsep 0'
parallel --colsep 0 echo {2} ::: a0OK0c
parallel --header : --colsep 0 echo {ok} ::: A0ok0B a0OK0b
echo '**'

2
testsuite/tests-to-run/parallel-local-sql.sh
View file

@ -26,7 +26,7 @@ p_wrapper() {
$DEBUG && sort -u $T1 $T2;
rm $T1 $T2
p_showsqlresult $SERVERURL $TABLE
$DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null
$DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
}
p_template() {

13
testsuite/wanted-results/parallel-freebsd
View file

@ -28,15 +28,10 @@ bug #40135: FreeBSD: sem --fg does not finish under /bin/sh
The job finished
echo 'bug #40133: FreeBSD: --round-robin gives no output'
bug #40133: FreeBSD: --round-robin gives no output
jot 1000000 | parallel --round-robin --pipe -kj3 wc | sort
299592 299592 2097144
299592 299592 2097144
400816 400816 2694608
jot 1000000 | parallel --round-robin --pipe -kj4 wc | sort
149796 149796 1048572
235148 235148 1646037
299592 299592 2097144
315464 315464 2097143
jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc
1000000 1000000 6888896
jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc
1000000 1000000 6888896
echo 'bug #40134: FreeBSD: --shebang not working'
bug #40134: FreeBSD: --shebang not working
(echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang; chmod 755 ./shebang; ./shebang

20
testsuite/wanted-results/parallel-local-0.3s
View file

@ -1765,11 +1765,29 @@ echo '### bug #48658: --linebuffer --files'
10
echo '**'
**
echo 'bug #49538: --header and {= =}'
bug #49538: --header and {= =}
/bin/bash: bug: command not found
parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
OK
parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
OK
parallel --header : echo {var/.} ::: var sub/dir/file.ext
file
parallel --header : echo {var//} ::: var sub/dir/file.ext
sub/dir
parallel --header : echo {var/.} ::: var sub/dir/file.ext
file
parallel --header : echo {var/} ::: var sub/dir/file.ext
file.ext
parallel --header : echo {var.} ::: var sub/dir/file.ext
sub/dir/file
echo '**'
**
echo 'bug --colsep 0'
bug --colsep 0
parallel --colsep 0 echo {2} ::: a0OK0c
OK
parallel --header : --colsep 0 echo {ok} ::: A0ok0B a0OK0b
OK
echo '**'
**