tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge ../parallel.stable

Browse source
This commit is contained in:
Ole Tange 2015-05-15 02:30:35 +02:00
parent f290cf9b05 09fccc68fc
commit ec8f851b6a
6 changed files with 77 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
src/parallel
View file

@ -6334,24 +6334,29 @@ sub wrapped {
}
}
if($opt::cat) {
# Prepend 'cat > {};'
# Append 'unlink {} without affecting $?'
$command =
$self->{'commandline'}->replace_placeholders(["cat > \257<\257>; "], 0, 0).
$command.";". postpone_exit_and_cleanup().
$self->{'commandline'}->replace_placeholders(["\257<\257>"], 0, 0);
'$PARALLEL_TMP';
} elsif($opt::fifo) {
# Prepend 'mkfifo {}; ('
# Append ') & _PID=$!; cat > {}; wait $_PID; '
# (This makes it fail in csh, but give the correct exit code in bash)
# Append 'unlink {} without affecting $?'
# Set $ENV{PARALLEL_TMP} when starting a job
# Set $ENV{PARALLEL_TMP} in the remote wrapper
# mkfifo $PARALLEL_TMP;
# {} = $PARALLEL_TMP;
# (...) &
# cat > $PARALLEL_TMP; wait \$_PID; cleanup $PARALLEL_TMP
# perl -e 'open($fifo,">",shift); while(read){print FIFO};unlink $fifo;waitpid($pid,0);exit $?' $! $PARALLEL_FIFO
$command =
$self->{'commandline'}->replace_placeholders(["mkfifo \257<\257>;\n ("], 0, 0).
"mkfifo \$PARALLEL_TMP\n (".
$command.";".
$self->{'commandline'}->replace_placeholders([") & _PID=\$!; cat > \257<\257>; ",
"wait \$_PID; "]).
') & _PID=$!; cat > $PARALLEL_TMP; wait $_PID; '.
postpone_exit_and_cleanup().
$self->{'commandline'}->replace_placeholders(["\257<\257>"], 0, 0);
'$PARALLEL_TMP';
}
# Wrap with ssh + tranferring of files
$command = $self->sshlogin_wrap($command);
@ -6472,7 +6477,13 @@ sub sshlogin_wrap {
$monitor_parent_sshd_script =
# This will be packed in ', so only use "
::spacefree(0,'$shell = "'.($ENV{'PARALLEL_SHELL'} || '$ENV{SHELL}').'";'.
'$tmpdir = "'.::perl_quote_scalar($ENV{'TMPDIR'}).'";'.
q{
# Set $PARALLEL_TMP to a non-existent file name in $TMPDIR
do {
$ENV{PARALLEL_TMP} = $tmpdir."/par".
join"", map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5);
} while(-e $ENV{PARALLEL_TMP});
$SIG{CHLD} = sub { $done = 1; };
$pid = fork;
unless($pid) {
@ -6899,6 +6910,7 @@ sub start {
}
$ENV{'PARALLEL_SEQ'} = $job->seq();
$ENV{'PARALLEL_PID'} = $$;
$ENV{'PARALLEL_TMP'} = ::tmpname("par");
::debug("run", $Global::total_running, " processes . Starting (",
$job->seq(), "): $command\n");
if($opt::pipe) {
@ -7007,12 +7019,8 @@ sub print_dryrun_and_verbose {
my $self = shift;
my $actual_command = shift;
# Temporary file name. Used for fifo to communicate exit val
my ($fh, $tmpfifo) = ::tmpfile(SUFFIX => ".tmx");
$Global::unlink{$tmpfifo} = 1;
close $fh;
unlink $tmpfifo;
# # FIFO for communicating exit val
# my $tmpfifo = ::tmpfifo();
my $tmpfifo=::tmpname("tmx");
if(length($tmpfifo) >=100) {
::error("tmux does not support sockets with path > 100.");
::wait_and_exit(255);
@ -7030,12 +7038,13 @@ sub print_dryrun_and_verbose {
my $l_act = length($actual_command);
my $l_tit = length($title);
my $l_fifo = length($tmpfifo);
# The line to run contains a 118 chars extra code + the title 2x
my $l_tot = 2 * $l_tit + $l_act + 30;
my $l_tot = 2 * $l_tit + $l_act + $l_fifo;
while($l_tit < 1000 and
(
(900 < $l_tot and $l_tot < 1300)
(890 < $l_tot and $l_tot < 1350)
or
(9250 < $l_tot and $l_tot < 9800)
)) {
@ -7048,7 +7057,7 @@ sub print_dryrun_and_verbose {
# 9331 < (title + whole command) < 9636
$title = $title.('\ 'x75);
$l_tit = length($title);
$l_tot = 2 * $l_tit + $l_act + 30;
$l_tot = 2 * $l_tit + $l_act + $l_fifo;
}
my $tmux;
@ -7065,7 +7074,7 @@ sub print_dryrun_and_verbose {
$Limits::Command::line_max_len, " tot ",
$l_tot, "\n");
return "mkfifo $tmpfifo; $tmux ".
return "mkfifo $tmpfifo && $tmux ".
# Run in tmux
::shell_quote_scalar
(
@ -7654,14 +7663,14 @@ sub populate {
my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
if($opt::cat) {
# Generate a tempfile name that will be used as {}
# $PARALLEL_TMP will point to a tempfile that will be used as {}
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
unget([Arg->new(::tmpname("cat"))]);
unget([Arg->new('$PARALLEL_TMP')]);
}
if($opt::fifo) {
# Generate a tempfile name that will be used as {}
# $PARALLEL_TMP will point to a tempfile that will be used as {}
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
unget([Arg->new(::tmpname("fif"))]);
unget([Arg->new('$PARALLEL_TMP')]);
}
while (not $self->{'arg_queue'}->empty()) {
$next_arg = $self->{'arg_queue'}->get();
@ -7955,6 +7964,8 @@ sub replaced {
if(not defined $self->{'replaced'}) {
# Don't quote arguments if the input is the full command line
my $quote_arg = $Global::noquote ? 0 : not $Global::quoting;
# or if ($opt::cat or $opt::pipe) as they use $PARALLEL_TMP
$quote_arg = ($opt::cat || $opt::fifo) ? 0 : $quote_arg;
$self->{'replaced'} = $self->
replace_placeholders($self->{'command'},$Global::quoting,
$quote_arg);

4
testsuite/tests-to-run/parallel-local-ssh2.sh
View file

@ -45,8 +45,8 @@ echo '2bug #43358: shellshock breaks exporting functions using --env'
echo '### bug #42999: --pipepart with remote does not work'
seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999;
parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc;
parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:(/tmp\S+fif)\S+:${1}XXXXX:' ;
parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:(/tmp\S+cat)\S+:${1}XXXXX:' ;
parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:(/tmp\S+par)\S+:${1}XXXXX:' ;
parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:(/tmp\S+par)\S+:${1}XXXXX:' ;
rm /tmp/bug42999
echo '### --cat gives incorrect exit value in csh'

4
testsuite/tests-to-run/parallel-local22.sh
View file

@ -46,8 +46,8 @@ echo '### bug #42041: Implement $PARALLEL_JOBSLOT'
echo '### bug #42363: --pipepart and --fifo/--cat does not work'
seq 100 > /tmp/bug42363;
parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:(/tmp\S+fif).....:${1}XXXXX:';
parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:(/tmp\S+cat).....:${1}XXXXX:';
parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:(/tmp\S+par).....:${1}XXXXX:';
parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:(/tmp\S+par).....:${1}XXXXX:';
rm /tmp/bug42363
echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile'

38
testsuite/wanted-results/parallel-local-ssh2
View file

@ -44,7 +44,7 @@ shellshock-hardened to non-shellshock-hardened
Function non-shellshock-hardened
echo '### bug #42999: --pipepart with remote does not work'
### bug #42999: --pipepart with remote does not work
seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:(/tmp\S+fif)\S+:${1}XXXXX:' ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:(/tmp\S+cat)\S+:${1}XXXXX:' ; rm /tmp/bug42999
seq 100 > /tmp/bug42999; chmod 600 /tmp/bug42999; parallel --sshdelay 0.3 --pipepart --block 31 -a /tmp/bug42999 -k -S parallel@lo wc; parallel --sshdelay 0.2 --pipepart --block 31 -a /tmp/bug42999 -k --fifo -S parallel@lo wc | perl -pe 's:(/tmp\S+par)\S+:${1}XXXXX:' ; parallel --sshdelay 0.1 --pipepart --block 31 -a /tmp/bug42999 -k --cat -S parallel@lo wc | perl -pe 's:(/tmp\S+par)\S+:${1}XXXXX:' ; rm /tmp/bug42999
14 14 33
11 11 33
11 11 33
@ -54,24 +54,24 @@ echo '### bug #42999: --pipepart with remote does not work'
11 11 33
11 11 33
9 9 28
14 14 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
11 11 33 /tmp/parallel-local-ssh2/fifXXXXX
9 9 28 /tmp/parallel-local-ssh2/fifXXXXX
14 14 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
11 11 33 /tmp/parallel-local-ssh2/catXXXXX
9 9 28 /tmp/parallel-local-ssh2/catXXXXX
14 14 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
9 9 28 /tmp/parallel-local-ssh2/parXXXXX
14 14 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
11 11 33 /tmp/parallel-local-ssh2/parXXXXX
9 9 28 /tmp/parallel-local-ssh2/parXXXXX
echo '### --cat gives incorrect exit value in csh'
### --cat gives incorrect exit value in csh
echo false | parallel --pipe --cat -Scsh@lo 'cat {}; false' ; echo $?; echo false | parallel --pipe --cat -Stcsh@lo 'cat {}; false' ; echo $?; echo true | parallel --pipe --cat -Scsh@lo 'cat {}; true' ; echo $?; echo true | parallel --pipe --cat -Stcsh@lo 'cat {}; true' ; echo $?; echo '### --cat and --fifo exit value in bash'

38
testsuite/wanted-results/parallel-local22
View file

@ -43,25 +43,25 @@ echo '### bug #42041: Implement $PARALLEL_JOBSLOT'
2
echo '### bug #42363: --pipepart and --fifo/--cat does not work'
### bug #42363: --pipepart and --fifo/--cat does not work
seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:(/tmp\S+fif).....:${1}XXXXX:'; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:(/tmp\S+cat).....:${1}XXXXX:'; rm /tmp/bug42363
14 14 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
11 11 33 /tmp/parallel-local22/fifXXXXX
9 9 28 /tmp/parallel-local22/fifXXXXX
14 14 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
11 11 33 /tmp/parallel-local22/catXXXXX
9 9 28 /tmp/parallel-local22/catXXXXX
seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe 's:(/tmp\S+par).....:${1}XXXXX:'; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe 's:(/tmp\S+par).....:${1}XXXXX:'; rm /tmp/bug42363
14 14 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
9 9 28 /tmp/parallel-local22/parXXXXX
14 14 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
11 11 33 /tmp/parallel-local22/parXXXXX
9 9 28 /tmp/parallel-local22/parXXXXX
echo '### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile'
### bug #42055: --pipepart -a bigfile should not require sequential reading of bigfile
parallel --pipepart -a /etc/passwd -L 1 should not be run

3
testsuite/wanted-results/parallel-tutorial
View file

@ -639,6 +639,7 @@ ORACLE_SID
PARALLEL
PARALLEL_PID
PARALLEL_SEQ
PARALLEL_TMP
PATH
PERL_MB_OPT
PERL_MM_OPT
@ -693,7 +694,7 @@ _
/bin/bash: my_func2: command not found
parallel -vv -S $SERVER1 echo ::: bar
ssh parallel@lo exec perl\ -e\ \\\$ENV\\\{\\\"PARALLEL_PID\\\"\\\}=\\\"000000\\\"\\\;\\\$ENV\\\{\\\"PARALLEL_SEQ\\\"\\\}=\\\"1\\\"\\\;\\\$bashfunc\\\ =\\\ \\\"\\\"\\\;@ARGV=\\\"echo\\\ bar\\\"\\\;\\\$shell=\\\"\\\$ENV\\\{SHELL\\\}\\\"\\\;\\\$SIG\\\{CHLD\\\}=sub\\\{\\\$done=1\\\;\\\}\\\;\\\$pid=fork\\\;unless\\\(\\\$pid\\\)\\\{setpgrp\\\;exec\\\$shell,\\\"-c\\\",\\\(\\\$bashfunc.\\\"@ARGV\\\"\\\)\\\;die\\\"exec:\\\$\\\!\\\\n\\\"\\\;\\\}do\\\{\\\$s=\\\$s\\\<1\\\?0.001+\\\$s\\\*1.03:\\\$s\\\;select\\\(undef,undef,undef,\\\$s\\\)\\\;\\\}until\\\(\\\$done\\\|\\\|getppid==1\\\)\\\;kill\\\(SIGHUP,-\\\$\\\{pid\\\}\\\)unless\\\$done\\\;wait\\\;exit\\\(\\\$\\\?\\\&127\\\?128+\\\(\\\$\\\?\\\&127\\\):1+\\\$\\\?\\\>\\\>8\\\);
ssh parallel@lo exec perl\ -e\ \\\$ENV\\\{\\\"PARALLEL_PID\\\"\\\}=\\\"000000\\\"\\\;\\\$ENV\\\{\\\"PARALLEL_SEQ\\\"\\\}=\\\"1\\\"\\\;\\\$bashfunc\\\ =\\\ \\\"\\\"\\\;@ARGV=\\\"echo\\\ bar\\\"\\\;\\\$shell=\\\"\\\$ENV\\\{SHELL\\\}\\\"\\\;\\\$tmpdir=\\\"/tmp/parallel-tutorial\\\"\\\;do\\\{\\\$ENV\\\{PARALLEL_TMP\\\}=\\\$tmpdir.\\\"/par\\\".join\\\"\\\",map\\\{\\\(0..9,\\\"a\\\"..\\\"z\\\",\\\"A\\\"..\\\"Z\\\"\\\)\\\[rand\\\(62\\\)\\\]\\\}\\\(1..5\\\)\\\;\\\}while\\\(-e\\\$ENV\\\{PARALLEL_TMP\\\}\\\)\\\;\\\$SIG\\\{CHLD\\\}=sub\\\{\\\$done=1\\\;\\\}\\\;\\\$pid=fork\\\;unless\\\(\\\$pid\\\)\\\{setpgrp\\\;exec\\\$shell,\\\"-c\\\",\\\(\\\$bashfunc.\\\"@ARGV\\\"\\\)\\\;die\\\"exec:\\\$\\\!\\\\n\\\"\\\;\\\}do\\\{\\\$s=\\\$s\\\<1\\\?0.001+\\\$s\\\*1.03:\\\$s\\\;select\\\(undef,undef,undef,\\\$s\\\)\\\;\\\}until\\\(\\\$done\\\|\\\|getppid==1\\\)\\\;kill\\\(SIGHUP,-\\\$\\\{pid\\\}\\\)unless\\\$done\\\;wait\\\;exit\\\(\\\$\\\?\\\&127\\\?128+\\\(\\\$\\\?\\\&127\\\):1+\\\$\\\?\\\>\\\>8\\\);
bar
my_func3() {
echo in my_func $1 > $1.out