mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
parallel: Implemented setpgrp. Passes testsuite except for some csh tests.
This commit is contained in:
Ole Tange
parent
a0adb0a1b1
commit
cb20cd2c8e
|
|
@ -40,20 +40,19 @@ measure() {
|
|||
|
||||
PATH=/tmp/bin:$PATH
|
||||
cd /tmp/bin
|
||||
# ls parallel-* | shuf | parallel -j$CORES --joblog /tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) -
|
||||
ls parallel-* | parallel --shuf -j$CORES --joblog /tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) -
|
||||
ls parallel-* | parallel --shuf -j$CORES --joblog ~/tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) -
|
||||
|
||||
killall forever
|
||||
|
||||
Rscript - <<_
|
||||
jl<-read.csv("/tmp/joblog$CORES-$INNER-$OUTER.csv",sep="\t");
|
||||
jl<-read.csv("$HOME/tmp/joblog$CORES-$INNER-$OUTER.csv",sep="\t");
|
||||
jl\$Command <- as.factor(substr(jl\$Command, 12, nchar(as.character(jl\$Command))-5))
|
||||
pdf("/tmp/boxplot.pdf");
|
||||
par(cex.axis=0.5);
|
||||
boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F,
|
||||
ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER");
|
||||
_
|
||||
cp /tmp/boxplot.pdf /tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf
|
||||
cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf
|
||||
evince /tmp/boxplot.pdf
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ New in this release:
|
|||
|
||||
* --retry-failed will retry all failed jobs in a joblog. It will ignore any command given.
|
||||
|
||||
* --ssh and $PARALLEL_SSH can be used to set the command used for ssh. The command is assume to behave the same as ssh.
|
||||
* --ssh and $PARALLEL_SSH can be used to set the command used for ssh. The command is assumed to behave the same way as ssh.
|
||||
|
||||
* --fifo now works in csh, too.
|
||||
|
||||
|
|
@ -250,7 +250,7 @@ New in this release:
|
|||
|
||||
* GNU Parallel was cited in: A composite genome approach to identify phylogenetically informative data from next-generation sequencing http://www.biomedcentral.com/content/pdf/s12859-015-0632-y.pdf
|
||||
|
||||
Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://arxiv.org/pdf/1505.07691.pdf
|
||||
* GNU Parallel was cited in: Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://arxiv.org/pdf/1505.07691.pdf
|
||||
|
||||
* <<Har angiveligt submittet ny version - afventer opdatering>> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
|
||||
|
||||
|
|
@ -264,6 +264,8 @@ Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://
|
|||
|
||||
* GNU Parallel is used in: https://github.com/d2207197/local-mapreduce
|
||||
|
||||
* GNU Parallel is used in: wiki-edit-news https://github.com/albert-decatur/wiki-edit-news
|
||||
|
||||
* GNU Parallel is used in: A Fingerprint Identification System https://curve.carleton.ca/system/files/theses/28733.pdf
|
||||
|
||||
* Job ad asking for GNU Parallel experience: http://searchjobs.intel.com/gdansk-pol/software-validation-engineer/63A06826DAF24797AB414DC146201C2E/job/
|
||||
|
|
|
|||
125
src/parallel
125
src/parallel
|
|
@ -890,6 +890,7 @@ sub parse_options {
|
|||
if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2
|
||||
$Global::debug = $opt::D;
|
||||
$Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh";
|
||||
$Global::cshell = $Global::shell =~ m:/csh:;
|
||||
if(defined $opt::X) { $Global::ContextReplace = 1; }
|
||||
if(defined $opt::silent) { $Global::verbose = 0; }
|
||||
if(defined $opt::0) { $/ = "\0"; }
|
||||
|
|
@ -908,6 +909,7 @@ sub parse_options {
|
|||
if(defined $opt::colsep) { $Global::trim = 'lr'; }
|
||||
if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
|
||||
if(defined $opt::trim) { $Global::trim = $opt::trim; }
|
||||
if(defined $opt::roundrobin) { $opt::pipe = 1; }
|
||||
if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
|
||||
if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
|
||||
if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
|
||||
|
|
@ -1259,7 +1261,7 @@ sub parse_semaphore {
|
|||
if(defined $opt::semaphorename) { $Global::semaphore = 1; }
|
||||
if(defined $opt::fg) { $Global::semaphore = 1; }
|
||||
if(defined $opt::bg) { $Global::semaphore = 1; }
|
||||
if(defined $opt::wait) { $Global::semaphore = 1; }
|
||||
if(defined $opt::wait) { $Global::semaphore = 1; @ARGV = "true"; }
|
||||
if($Global::semaphore) {
|
||||
if(@opt::a) {
|
||||
# A semaphore does not take input from neither stdin nor file
|
||||
|
|
@ -1660,7 +1662,7 @@ sub shell_quote_scalar {
|
|||
if(defined $a) {
|
||||
if(($a =~ s/'/''/g)
|
||||
+
|
||||
($a =~ s/[\n\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]+/'$&'/go)) {
|
||||
($a =~ s/[\n\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\'\202-\377]+/'$&'/go)) {
|
||||
# A string was replaced
|
||||
# No need to test for "" or \0
|
||||
} elsif($a eq "") {
|
||||
|
|
@ -6321,7 +6323,7 @@ sub total_failed {
|
|||
# * cat > fifo
|
||||
# * waitpid to get the exit code from $command
|
||||
# * be less than 1000 chars long
|
||||
$script = "perl -e ". ::shell_quote_scalar
|
||||
$script = "perl -e '".
|
||||
(::spacefree
|
||||
(0, q{
|
||||
($s,$c,$f) = @ARGV;
|
||||
|
|
@ -6340,7 +6342,7 @@ sub total_failed {
|
|||
# Cleanup
|
||||
unlink $f;
|
||||
exit $?/256;
|
||||
}));
|
||||
}))."'";
|
||||
}
|
||||
return $script;
|
||||
}
|
||||
|
|
@ -6446,6 +6448,13 @@ sub wrapped {
|
|||
# Wrap command with 'tmux'
|
||||
$command = $self->tmux_wrap($command);
|
||||
}
|
||||
if($Global::cshell
|
||||
and
|
||||
length $command > 499) {
|
||||
# csh does not like words longer than 1000 (499 quoted)
|
||||
$command = "perl -e '".base64_zip_eval()."' ".
|
||||
join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"');
|
||||
}
|
||||
$self->{'wrapped'} = $command;
|
||||
}
|
||||
return $self->{'wrapped'};
|
||||
|
|
@ -6954,18 +6963,62 @@ sub parentdirs_of {
|
|||
return @parents;
|
||||
}
|
||||
|
||||
sub open3_setpgrp_external {
|
||||
my ($in,$command) = @_;
|
||||
my $pid;
|
||||
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
$pid = ::open3($in, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|
||||
|| ::die_bug("open3-$in");
|
||||
1;
|
||||
};
|
||||
return $pid;
|
||||
}
|
||||
|
||||
sub open3_setpgrp_internal {
|
||||
my ($in,$command) = @_;
|
||||
my $pid;
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
if(not $pid = ::open3($in, ">&OUT", ">&ERR", "-")) {
|
||||
# Each child gets its own process group to make it safe to killall
|
||||
setpgrp(0,0);
|
||||
exec("exec $Global::shell -c ".::shell_quote_scalar($command))
|
||||
|| ::die_bug("open3-$in $command");
|
||||
}
|
||||
};
|
||||
return $pid;
|
||||
}
|
||||
|
||||
sub open3_setpgrp {
|
||||
# If the OS supports open3(x,x,x,"-") use that
|
||||
eval{ ::open3(Symbol::gensym(), ">&STDOUT", ">&STDERR", "-")};
|
||||
# if($@) {
|
||||
if(1) {
|
||||
# Does not support open3(x,x,x,"-")
|
||||
*open3_setpgrp = \&open3_setpgrp_external;
|
||||
} else {
|
||||
# Supports open3(x,x,x,"-")
|
||||
*open3_setpgrp = \&open3_setpgrp_internal;
|
||||
}
|
||||
# The sub is now redefined. Call it
|
||||
return open3_setpgrp(@_);
|
||||
}
|
||||
|
||||
sub start {
|
||||
# Setup STDOUT and STDERR for a job and start it.
|
||||
# Returns:
|
||||
# job-object or undef if job not to run
|
||||
|
||||
my $job = shift;
|
||||
# Get the shell command to be executed (possibly with ssh infront).
|
||||
my $command = $job->wrapped();
|
||||
my $pid;
|
||||
|
||||
if($Global::interactive or $Global::stderr_verbose) {
|
||||
$command = interactive_start($command);
|
||||
}
|
||||
my $pid;
|
||||
$job->openoutputfiles();
|
||||
my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w"));
|
||||
local (*IN,*OUT,*ERR);
|
||||
|
|
@ -6978,21 +7031,33 @@ sub start {
|
|||
if($opt::dryrun) {
|
||||
$command = "true";
|
||||
}
|
||||
# Each child gets its own process group to make it safe to killall
|
||||
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
|
||||
$ENV{'PARALLEL_SEQ'} = $job->seq();
|
||||
$ENV{'PARALLEL_PID'} = $$;
|
||||
$ENV{'PARALLEL_TMP'} = ::tmpname("par");
|
||||
::debug("run", $Global::total_running, " processes . Starting (",
|
||||
$job->seq(), "): $command\n");
|
||||
$command = "exec $Global::shell -c ".::shell_quote_scalar($command);
|
||||
|
||||
if($opt::pipe) {
|
||||
my ($stdin_fh);
|
||||
# $pid = open3_setpgrp($stdin_fh,$command);
|
||||
# $pid = open3_setpgrp($stdin_fh,$command);
|
||||
# The eval is needed to catch exception from open3
|
||||
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) ||
|
||||
::die_bug("open3-pipe");
|
||||
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|
||||
|| ::die_bug("open3-stdin");
|
||||
1;
|
||||
};
|
||||
# eval {
|
||||
# if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
|
||||
# # Each child gets its own process group to make it safe to killall
|
||||
# setpgrp(0,0);
|
||||
# exec($command) || ::die_bug("open3-pipe");
|
||||
# }
|
||||
# 1;
|
||||
# };
|
||||
$job->set_fh(0,"w",$stdin_fh);
|
||||
} elsif ($opt::tty and not $Global::tty_taken and -c "/dev/tty" and
|
||||
open(my $devtty_fh, "<", "/dev/tty")) {
|
||||
|
|
@ -7000,7 +7065,7 @@ sub start {
|
|||
*IN = $devtty_fh;
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) ||
|
||||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $command) ||
|
||||
::die_bug("open3-/dev/tty");
|
||||
$Global::tty_taken = $pid;
|
||||
close $devtty_fh;
|
||||
|
|
@ -7011,22 +7076,46 @@ sub start {
|
|||
# Give STDIN to the first job if using -a (but only if running
|
||||
# locally - otherwise CTRL-C does not work for other jobs Bug#36585)
|
||||
*IN = *STDIN;
|
||||
# $pid = open3_setpgrp_internal("<&IN",$command);
|
||||
# $pid = open3_setpgrp("<&IN",$command);
|
||||
# The eval is needed to catch exception from open3
|
||||
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) ||
|
||||
::die_bug("open3-a");
|
||||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|
||||
|| ::die_bug("open3-IN");
|
||||
1;
|
||||
};
|
||||
# eval {
|
||||
# if(not $pid = ::open3("<&IN", ">&OUT", ">&ERR", "-")) {
|
||||
# # Each child gets its own process group to make it safe to killall
|
||||
# setpgrp(0,0);
|
||||
# exec($command) || ::die_bug("open3-a");
|
||||
# }
|
||||
# 1;
|
||||
# };
|
||||
# Re-open to avoid complaining
|
||||
open(STDIN, "<&", $Global::original_stdin)
|
||||
or ::die_bug("dup-\$Global::original_stdin: $!");
|
||||
} else {
|
||||
# $pid = open3_setpgrp_internal(::gensym,$command);
|
||||
# $pid = open3_setpgrp(::gensym,$command);
|
||||
# The eval is needed to catch exception from open3
|
||||
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
$pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command) ||
|
||||
::die_bug("open3-gensym");
|
||||
$pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|
||||
|| ::die_bug("open3-gens");
|
||||
1;
|
||||
};
|
||||
# eval {
|
||||
# if(not $pid = ::open3(::gensym, ">&OUT", ">&ERR", "-")) {
|
||||
# # Each child gets its own process group to make it safe to killall
|
||||
# setpgrp(0,0);
|
||||
# exec($command) || ::die_bug("open3-gensym $command");
|
||||
# }
|
||||
# 1;
|
||||
# };
|
||||
}
|
||||
if($pid) {
|
||||
# A job was started
|
||||
|
|
@ -8022,7 +8111,6 @@ sub len {
|
|||
}
|
||||
# If we are using --env, add the prefix for that, too.
|
||||
$len += $Global::envvarlen;
|
||||
|
||||
return $len;
|
||||
}
|
||||
|
||||
|
|
@ -8500,14 +8588,15 @@ sub max_length {
|
|||
}
|
||||
$Limits::Command::line_max_len = tmux_length($cached_limit);
|
||||
if($opt::max_chars) {
|
||||
if($opt::max_chars <= $cached_limit) {
|
||||
$Limits::Command::line_max_len = $opt::max_chars;
|
||||
if($opt::max_chars * 2 <= $cached_limit) {
|
||||
# $opt::max_chars quoting causes the length to double
|
||||
$Limits::Command::line_max_len = $opt::max_chars * 2;
|
||||
} else {
|
||||
::warning("Value for -s option should be < $cached_limit.");
|
||||
}
|
||||
}
|
||||
}
|
||||
return $Limits::Command::line_max_len;
|
||||
return int($Limits::Command::line_max_len/2);
|
||||
}
|
||||
|
||||
sub real_max_length {
|
||||
|
|
|
|||
|
|
@ -1557,6 +1557,8 @@ time to initialize.
|
|||
B<--keep-order> will not work with B<--round-robin> as it is
|
||||
impossible to track which input block corresponds to which output.
|
||||
|
||||
B<--round-robin> implies B<--pipe>.
|
||||
|
||||
|
||||
=item B<--rpl> 'I<tag> I<perl expression>'
|
||||
|
||||
|
|
@ -2369,6 +2371,18 @@ If B<my_program> fails a red FAIL will be printed followed by the failing
|
|||
command; otherwise a green OK will be printed followed by the command.
|
||||
|
||||
|
||||
=head1 EXAMPLE: Log rotate
|
||||
|
||||
Log rotation renames a logfile to an extension with a higher number:
|
||||
log.1 becomes log.2, log.2 becomes log.3, and so on. The oldest log is
|
||||
removed. To avoid overwriting files the process starts backwards from
|
||||
the high number to the low number. This will keep 10 old versions of
|
||||
the log:
|
||||
|
||||
seq 9 -1 1 | parallel -j1 mv log.{} log.'{= $_++ =}'
|
||||
mv log log.1
|
||||
|
||||
|
||||
=head1 EXAMPLE: Removing file extension when processing files
|
||||
|
||||
When processing files removing the file extension using B<{.}> is
|
||||
|
|
|
|||
|
|
@ -39,6 +39,9 @@ export -f run_test
|
|||
# Create a monitor script
|
||||
echo forever pstree -lp $$ >/tmp/monitor
|
||||
chmod 755 /tmp/monitor
|
||||
# log rotate
|
||||
parallel -j1 mv testsuite.log.{} testsuite.log.'{= $_++ =}' ::: 10 9 8 7 6 5 4 3 2 1
|
||||
mv testsuite.log testsuite.log.1
|
||||
date
|
||||
mkdir -p actual-results
|
||||
ls -t tests-to-run/*${1}*.sh | egrep -v "${2}" |
|
||||
|
|
@ -51,5 +54,4 @@ else
|
|||
rm -rf src-passing-testsuite
|
||||
cp -a ../src src-passing-testsuite
|
||||
fi
|
||||
rm testsuite.log
|
||||
date
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
|
|||
echo '** bug #41805 done'
|
||||
|
||||
echo '### Deal with long command lines on remote servers'
|
||||
perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
|
||||
perl -e 'print((("\""x5000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
|
||||
|
||||
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
|
||||
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
|
||||
|
|
|
|||
|
|
@ -14,13 +14,13 @@ echo '### zsh'
|
|||
|
||||
echo '### csh'
|
||||
echo "3 big vars run remotely - length(base64) > 1000"
|
||||
ssh csh@lo 'setenv A `seq 200|xargs`;
|
||||
stdout ssh csh@lo 'setenv A `seq 200|xargs`;
|
||||
setenv B `seq 200 -1 1|xargs`;
|
||||
setenv C `seq 300 -2 1|xargs`;
|
||||
parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
|
||||
echo "3 big vars run locally"
|
||||
ssh csh@lo 'setenv A `seq 200|xargs`;
|
||||
stdout ssh csh@lo 'setenv A `seq 200|xargs`;
|
||||
setenv B `seq 200 -1 1|xargs`;
|
||||
setenv C `seq 300 -2 1|xargs`;
|
||||
parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),
|
|||
|
||||
echo '### Test output ascii'
|
||||
rm -f /tmp/paralocal7*;
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter;
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-10..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter;
|
||||
sort /tmp/paralocal7* | md5sum
|
||||
|
||||
echo '### Test critical lengths. Must not block'
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ cd testsuite 2>/dev/null
|
|||
rm -rf tmp
|
||||
mkdir tmp
|
||||
cd tmp
|
||||
touch ~/.parallel/will-cite
|
||||
echo '### test parallel_tutorial'
|
||||
rm -f /tmp/runs
|
||||
export SERVER1=parallel@lo
|
||||
|
|
|
|||
|
|
@ -359,12 +359,8 @@ pdksh "#&/
|
|||
pdksh ()*=?'
|
||||
posh "#&/
|
||||
posh ()*=?'
|
||||
rbash "#&/
|
||||
rbash ()*=?'
|
||||
rc "#&/
|
||||
rc ()*=?'
|
||||
rzsh "#&/
|
||||
rzsh ()*=?'
|
||||
sash "#&/
|
||||
sash ()*=?'
|
||||
sh "#&/
|
||||
|
|
@ -377,6 +373,8 @@ yash "#&/
|
|||
yash ()*=?'
|
||||
zsh "#&/
|
||||
zsh ()*=?'
|
||||
rbash rbash: line 0: exec: restricted
|
||||
rzsh zsh:exec:1: rzsh: restricted
|
||||
echo '**'
|
||||
**
|
||||
### 1 .par file from --files expected
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ HOME=/home/tange
|
|||
** bug #41805 done
|
||||
echo '### Deal with long command lines on remote servers'
|
||||
### Deal with long command lines on remote servers
|
||||
perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
|
||||
4 10 100010
|
||||
perl -e 'print((("\""x5000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
|
||||
4 10 50010
|
||||
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
|
||||
### Test bug #34241: --pipe should not spawn unneeded processes
|
||||
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ echo '### csh'
|
|||
### csh
|
||||
echo "3 big vars run remotely - length(base64) > 1000"
|
||||
3 big vars run remotely - length(base64) > 1000
|
||||
ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
1 200 692
|
||||
1 200 692
|
||||
1 150 547
|
||||
echo "3 big vars run locally"
|
||||
3 big vars run locally
|
||||
ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
1 200 692
|
||||
1 200 692
|
||||
1 150 547
|
||||
|
|
@ -48,7 +48,7 @@ echo '### works'
|
|||
### works
|
||||
stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
|
||||
stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output'
|
||||
parallel: Error: Command line too long (402 >= 236) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...
|
||||
parallel: Error: Command line too long (402 >= 118) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...
|
||||
echo '### These blocked due to length'
|
||||
### These blocked due to length
|
||||
stdout parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -456,6 +456,7 @@ echo '### true < 32767-ys.xi'
|
|||
### true < 32767-ys.xi
|
||||
stdout xargs true < 32767-ys.xi
|
||||
stdout parallel -k true < 32767-ys.xi
|
||||
parallel: Error: Command line too long (98306 >= 65535) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ...
|
||||
echo '### true < 16383-ys.xi'
|
||||
### true < 16383-ys.xi
|
||||
stdout xargs true < 16383-ys.xi
|
||||
|
|
|
|||
|
|
@ -107,24 +107,24 @@ seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -kX echo a{}b{.}c{.}
|
|||
a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6
|
||||
echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -km echo a{}b{.}c{.} | tee >(wc) >(md5sum) >/dev/null; wait
|
||||
### Test -m with 60000 args
|
||||
565b18712ac24f5c46f6e64cf4548733 -
|
||||
10 179980 1286692
|
||||
0891d955e7e2659ee6a88a96f3309b5c -
|
||||
20 179960 1286702
|
||||
echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc) >(md5sum) >/dev/null; wait
|
||||
### Test -X with 60000 args
|
||||
f09f466888e1c737eb581f41cb0087ba -
|
||||
11 60000 1346682
|
||||
af6c8c69409b5cbc55aeb95174e23c37 -
|
||||
21 60000 1346682
|
||||
echo '### Test -X with 60000 args and 5 expansions'
|
||||
### Test -X with 60000 args and 5 expansions
|
||||
seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
|
||||
17
|
||||
34
|
||||
seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l
|
||||
13
|
||||
25
|
||||
seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | wc -l
|
||||
11
|
||||
21
|
||||
seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c | wc -l
|
||||
9
|
||||
17
|
||||
seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b | wc -l
|
||||
6
|
||||
11
|
||||
echo '### Test {.} does not repeat more than {}'
|
||||
### Test {.} does not repeat more than {}
|
||||
seq 1 15 | perl -pe 's/$/.gif/' | parallel -j1 -s 80 -kX echo a{}b{.}c{.}
|
||||
|
|
@ -146,13 +146,13 @@ a14.gif 15.gifb14 15c14 15
|
|||
echo '### Test -I with shell meta chars'
|
||||
### Test -I with shell meta chars
|
||||
seq 1 60000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l
|
||||
9
|
||||
17
|
||||
seq 1 60000 | parallel -j1 -I '<>' -X echo 'a<>b<>c<>' | wc -l
|
||||
9
|
||||
17
|
||||
seq 1 60000 | parallel -j1 -I '<' -X echo 'a<b<c<' | wc -l
|
||||
9
|
||||
17
|
||||
seq 1 60000 | parallel -j1 -I '>' -X echo 'a>b>c>' | wc -l
|
||||
9
|
||||
17
|
||||
echo '### Test {.}'
|
||||
### Test {.}
|
||||
echo a | parallel -qX echo "'"{.}"' "
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; CHAR=$(cat /tmp/a$$ | wc -c); LINES=$(cat /tmp/a$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/awc$$; rm /tmp/a$$ /tmp/awc$$
|
||||
### Test of xargs -m command lines > 130k
|
||||
31d9274be5fdc2de59487cb05ba57776 -
|
||||
Chars per line: 116300
|
||||
6 119994 697800
|
||||
cb2184f70b8e9058cae1413426f8a9ae -
|
||||
Chars per line: 63437
|
||||
11 119989 697810
|
||||
echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$
|
||||
### Test of xargs -X command lines > 130k
|
||||
22074f9acada52462defb18ba912d744 -
|
||||
Chars per line: 116826
|
||||
7 60000 817788
|
||||
514f33181f346d416a1ada982b43a542 -
|
||||
Chars per line: 62906
|
||||
13 60000 817788
|
||||
echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum
|
||||
### Test of xargs -m command lines > 130k
|
||||
b35d8e49be8d94899b719c40d3f1f4bb -
|
||||
1e763b036fe666d229520dad45610e6c -
|
||||
echo '### This causes problems if we kill child processes'; seq 2 40 | parallel -j 0 seq 1 10 | sort | md5sum
|
||||
### This causes problems if we kill child processes
|
||||
437c0d47a99b9a7c5bcb1d132f94c2e6 -
|
||||
|
|
@ -191,8 +191,10 @@ which static-sh => shell path /bin/static-sh
|
|||
which tcsh => shell path /usr/bin/tcsh
|
||||
which yash => shell path /usr/bin/yash
|
||||
which zsh => shell path /usr/bin/zsh
|
||||
/bin/rbash: line 0: exec: restricted
|
||||
Local configuration error occurred.
|
||||
Contact the systems administrator for further assistance.
|
||||
zsh:exec:1: /bin/rzsh: restricted
|
||||
echo '## Started directly from perl'
|
||||
## Started directly from perl
|
||||
perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which; rm -f /tmp/par*.par
|
||||
|
|
|
|||
|
|
@ -86,8 +86,9 @@ please cite as described in 'parallel --bibtex'.
|
|||
echo '### bug #39787: --xargs broken'
|
||||
### bug #39787: --xargs broken
|
||||
nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"'
|
||||
131064
|
||||
37830
|
||||
65526
|
||||
65526
|
||||
37842
|
||||
echo '### --delay should grow by 3 sec per arg'
|
||||
### --delay should grow by 3 sec per arg
|
||||
stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"'
|
||||
|
|
|
|||
|
|
@ -31,51 +31,51 @@ See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
|||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
seq 1000 1100 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007
|
||||
255
|
||||
seq 1100 1200 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (2209 >= 2023) at input 0: 1100
|
||||
255
|
||||
seq 1200 1300 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (2409 >= 2023) at input 0: 1200
|
||||
255
|
||||
seq 1300 1400 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (2609 >= 2023) at input 0: 1300
|
||||
255
|
||||
seq 1400 1500 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (2809 >= 2023) at input 0: 1400
|
||||
255
|
||||
seq 1500 1600 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (3009 >= 2023) at input 0: 1500
|
||||
255
|
||||
seq 1600 1700 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (3209 >= 2023) at input 0: 1600
|
||||
255
|
||||
seq 1700 1800 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (3409 >= 2023) at input 0: 1700
|
||||
255
|
||||
seq 1800 1900 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (3609 >= 2023) at input 0: 1800
|
||||
255
|
||||
seq 1900 2000 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (3809 >= 2023) at input 0: 1900
|
||||
255
|
||||
seq 2000 2018 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (4009 >= 2023) at input 0: 2000
|
||||
255
|
||||
echo '### tmux1.9 fails'
|
||||
### tmux1.9 fails
|
||||
echo 2019 | TMUX=tmux1.9 par_tmux
|
||||
parallel: Error: Command line too long (4047 >= 4047) at input 0: 2019
|
||||
parallel: Error: Command line too long (4047 >= 2023) at input 0: 2019
|
||||
255
|
||||
echo 2020 | TMUX=tmux1.9 par_tmux
|
||||
parallel: Error: Command line too long (4049 >= 4047) at input 0: 2020
|
||||
parallel: Error: Command line too long (4049 >= 2023) at input 0: 2020
|
||||
255
|
||||
echo 2021 | TMUX=tmux1.9 par_tmux
|
||||
parallel: Error: Command line too long (4051 >= 4047) at input 0: 2021
|
||||
parallel: Error: Command line too long (4051 >= 2023) at input 0: 2021
|
||||
255
|
||||
echo 2022 | TMUX=tmux1.9 par_tmux
|
||||
parallel: Error: Command line too long (4053 >= 4047) at input 0: 2022
|
||||
parallel: Error: Command line too long (4053 >= 2023) at input 0: 2022
|
||||
255
|
||||
echo '### tmux1.8'
|
||||
### tmux1.8
|
||||
|
|
@ -87,34 +87,36 @@ See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
|||
0
|
||||
seq 101 150 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (236 >= 236) at input 0: 114
|
||||
255
|
||||
seq 151 200 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (310 >= 236) at input 0: 151
|
||||
255
|
||||
seq 201 231 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
0
|
||||
parallel: Error: Command line too long (410 >= 236) at input 0: 201
|
||||
255
|
||||
echo '### tmux1.8 fails'
|
||||
### tmux1.8 fails
|
||||
echo 232 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (472 >= 472) at input 0: 232
|
||||
parallel: Error: Command line too long (472 >= 236) at input 0: 232
|
||||
255
|
||||
echo 233 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (474 >= 472) at input 0: 233
|
||||
parallel: Error: Command line too long (474 >= 236) at input 0: 233
|
||||
255
|
||||
echo 234 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (476 >= 472) at input 0: 234
|
||||
parallel: Error: Command line too long (476 >= 236) at input 0: 234
|
||||
255
|
||||
echo 235 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (478 >= 472) at input 0: 235
|
||||
parallel: Error: Command line too long (478 >= 236) at input 0: 235
|
||||
255
|
||||
echo 236 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (480 >= 472) at input 0: 236
|
||||
parallel: Error: Command line too long (480 >= 236) at input 0: 236
|
||||
255
|
||||
echo '### tmux1.8 0..255 ascii'
|
||||
### tmux1.8 0..255 ascii
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
|
||||
See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
parallel: Error: Command line too long (236 >= 236) at input 0: 155
!"#$%&'()*+,-./0...
|
||||
0
|
||||
echo '### tmux1.9 0..255 ascii'
|
||||
### tmux1.9 0..255 ascii
|
||||
|
|
@ -123,16 +125,17 @@ See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
|||
0
|
||||
echo '### Test output ascii'
|
||||
### Test output ascii
|
||||
rm -f /tmp/paralocal7*; perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; sort /tmp/paralocal7* | md5sum
|
||||
rm -f /tmp/paralocal7*; perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-10..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; sort /tmp/paralocal7* | md5sum
|
||||
See output with: tmux -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
a7ee232967c8aab2edf227169e8cdce0 -
|
||||
053c7e8e945ef7641fc63bc309ac069d -
|
||||
echo '### Test critical lengths. Must not block'
|
||||
### Test critical lengths. Must not block
|
||||
seq 70 130 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
parallel: Error: Command line too long (236 >= 236) at input 0: 114
|
||||
seq 70 130 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
seq 280 425 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
parallel: Error: Command line too long (288 >= 236) at input 0: 280
|
||||
seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
|
||||
|
|
|
|||
|
|
@ -403,12 +403,12 @@ a1b1^c1[.}c
|
|||
echo "### BUG: The length for -X is not close to max (131072)"
|
||||
### BUG: The length for -X is not close to max (131072)
|
||||
seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc
|
||||
1 12821 131060
|
||||
1 6861 65500
|
||||
echo "### BUG: empty lines with --show-limit"
|
||||
### BUG: empty lines with --show-limit
|
||||
echo | $NICEPAR --show-limits
|
||||
Maximal size of command: 131071
|
||||
Maximal used size of command: 131071
|
||||
Maximal used size of command: 65535
|
||||
|
||||
Execution of will continue now, and it will try to read its input
|
||||
and run commands; if this is not what you wanted to happen, please
|
||||
|
|
|
|||
|
|
@ -241,11 +241,11 @@ f1=B f2=D
|
|||
f1=A f2=B
|
||||
f1=C f2=D
|
||||
cat num30000 | parallel --xargs echo | wc -l
|
||||
2
|
||||
3
|
||||
cat num30000 | parallel --xargs -s 10000 echo | wc -l
|
||||
17
|
||||
cat num30000 | parallel --jobs 4 -m echo | wc -l
|
||||
5
|
||||
6
|
||||
parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10
|
||||
1 2 3
|
||||
4 5 6
|
||||
|
|
|
|||
Loading…
Reference in a new issue