mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 14:07:55 +00:00
Fixed bug #45769: --round-robin --pipepart gives wrong results.
This commit is contained in:
Ole Tange
parent
f47bdf0a0d
commit
18ab03c98d
|
|
@ -231,6 +231,10 @@ New in this release:
|
|||
|
||||
* <<kontaktet 2015-06-22 Afventer svar fra journal>> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
|
||||
|
||||
* Joys of gnu parallel http://scottolesen.com/index.php/2015/08/26/joys-of-gnu-parallel/
|
||||
|
||||
* Crop and resize images with bash and ImageMagick https://www.simonholywell.com/post/2015/08/image-resize-crop-bash-imagemagick/
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
|
|
|||
21
src/parallel
21
src/parallel
|
|
@ -51,6 +51,7 @@ my @command = @ARGV;
|
|||
|
||||
my @input_source_fh;
|
||||
if($opt::pipepart) {
|
||||
# -a is used for data - not for command line args
|
||||
@input_source_fh = map { open_or_exit($_) } "/dev/null";
|
||||
} else {
|
||||
@input_source_fh = map { open_or_exit($_) } @opt::a;
|
||||
|
|
@ -115,16 +116,25 @@ if($opt::eta or $opt::bar or $opt::shuf or $Global::halt_pct) {
|
|||
# before starting any
|
||||
$Global::JobQueue->total_jobs();
|
||||
}
|
||||
# Compute $Global::max_jobs_running
|
||||
for my $sshlogin (values %Global::host) {
|
||||
$sshlogin->max_jobs_running();
|
||||
}
|
||||
if($opt::pipepart) {
|
||||
if($opt::roundrobin) {
|
||||
# Compute size of -a
|
||||
my $size = 0;
|
||||
$size += -s $_ for @opt::a;
|
||||
$Global::max_jobs_running or ::die_bug("Global::max_jobs_running not set");
|
||||
# Set --blocksize = size / no of proc
|
||||
$opt::blocksize = 1 + $size / $Global::max_jobs_running;
|
||||
}
|
||||
@Global::cat_partials = map { pipe_part_files($_) } @opt::a;
|
||||
# Unget the command as many times as there are parts
|
||||
$Global::JobQueue->{'commandlinequeue'}->unget(
|
||||
map { $Global::JobQueue->{'commandlinequeue'}->get() } @Global::cat_partials
|
||||
);
|
||||
}
|
||||
for my $sshlogin (values %Global::host) {
|
||||
$sshlogin->max_jobs_running();
|
||||
}
|
||||
|
||||
init_run_jobs();
|
||||
my $sem;
|
||||
|
|
@ -916,7 +926,6 @@ sub parse_options {
|
|||
if(defined $opt::colsep) { $Global::trim = 'lr'; }
|
||||
if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
|
||||
if(defined $opt::trim) { $Global::trim = $opt::trim; }
|
||||
if(defined $opt::roundrobin) { $opt::pipe = 1; }
|
||||
if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
|
||||
if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
|
||||
if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
|
||||
|
|
@ -4834,7 +4843,9 @@ sub compute_number_of_processes {
|
|||
$max_system_proc_reached and last;
|
||||
|
||||
my $before_getting_arg = time;
|
||||
get_args_or_jobs() or last;
|
||||
if(!$opt::roundrobin) {
|
||||
get_args_or_jobs() or last;
|
||||
}
|
||||
$wait_time_for_getting_args += time - $before_getting_arg;
|
||||
$system_limit++;
|
||||
|
||||
|
|
|
|||
|
|
@ -185,6 +185,15 @@ testquote() { printf '"#&/\n()*=?'"'" | PARALLEL_SHELL=$1 parallel -0 echo; };
|
|||
|
||||
echo '**'
|
||||
|
||||
echo '### bug #45769: --round-robin --pipepart gives wrong results'
|
||||
|
||||
seq 10000 >/tmp/seq10000;
|
||||
parallel -j2 --pipepart -a /tmp/seq10000 --block 14 --round-robin wc | wc -l;
|
||||
rm /tmp/seq10000
|
||||
|
||||
echo '**'
|
||||
|
||||
|
||||
EOF
|
||||
echo '### 1 .par file from --files expected'
|
||||
find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l
|
||||
|
|
|
|||
|
|
@ -210,8 +210,46 @@ env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
|
|||
env_parallel -S ksh@lo func_echo ::: function_works_over_ssh
|
||||
EOS
|
||||
|
||||
echo
|
||||
echo "### Fish environment"
|
||||
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
|
||||
set myvar "myvar works"
|
||||
setenv myenvvar "myenvvar works"
|
||||
|
||||
set funky (perl -e 'print pack "c*", 1..255')
|
||||
setenv funkyenv (perl -e 'print pack "c*", 1..255')
|
||||
|
||||
set myarray '' array_val2 3 '' 5
|
||||
#typeset -A assocarr
|
||||
#assocarr[a]=assoc_val_a
|
||||
#assocarr[b]=assoc_val_b
|
||||
alias alias_echo="echo 3 arg";
|
||||
|
||||
function func_echo
|
||||
echo $argv;
|
||||
echo "$myvar"
|
||||
echo "$myenvvar"
|
||||
echo $myarray[2]
|
||||
# echo ${assocarr[a]}
|
||||
echo Funky-"$funky"-funky
|
||||
echo Funky-"$funkyenv"-funky
|
||||
end
|
||||
|
||||
function env_parallel
|
||||
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/')
|
||||
parallel $argv;
|
||||
set -e PARALLEL_ENV
|
||||
end
|
||||
|
||||
env_parallel alias_echo ::: alias_does_not_work
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel -S fish@lo alias_echo ::: alias_does_not_work_over_ssh
|
||||
env_parallel -S fish@lo func_echo ::: function_works_over_ssh
|
||||
EOS
|
||||
|
||||
echo
|
||||
echo "### csh environment"
|
||||
# http://hyperpolyglot.org/unix-shells
|
||||
stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
|
||||
set myvar = "myvar works"
|
||||
set funky = "`perl -e 'print pack q(c*), 1..255'`"
|
||||
|
|
|
|||
|
|
@ -384,5 +384,11 @@ zsh "#&/
|
|||
zsh ()*=?'
|
||||
echo '**'
|
||||
**
|
||||
echo '### bug #45769: --round-robin --pipepart gives wrong results'
|
||||
### bug #45769: --round-robin --pipepart gives wrong results
|
||||
seq 10000 >/tmp/seq10000; parallel -j2 --pipepart -a /tmp/seq10000 --block 14 --round-robin wc | wc -l; rm /tmp/seq10000
|
||||
2
|
||||
echo '**'
|
||||
**
|
||||
### 1 .par file from --files expected
|
||||
1
|
||||
|
|
|
|||
|
|
@ -177,6 +177,39 @@ assoc_val_a
|
|||
Funky-
|
||||
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
|
||||
### Fish environment
|
||||
|
||||
* Documentation: http://www.linuxmint.com
|
||||
|
||||
0 updates are security updates.
|
||||
|
||||
3 arg alias_does_not_work
|
||||
function_works
|
||||
|
||||
myenvvar works
|
||||
Funky--funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€<7F>‚ƒ„…†‡ˆ‰Š‹Œ<E280B9>Ž<EFBFBD><C5BD>‘’“”•–—˜™š›œ<E280BA>žŸ ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky
|
||||
Array index out of bounds
|
||||
fish: echo $myarray[2]
|
||||
^
|
||||
in function “func_echoâ€<C3A2>
|
||||
called on standard input
|
||||
with parameter list “function_worksâ€<C3A2>
|
||||
|
||||
3 arg alias_does_not_work_over_ssh
|
||||
function_works_over_ssh
|
||||
|
||||
|
||||
Funky--funky
|
||||
Funky--funky
|
||||
Array index out of bounds
|
||||
fish: echo $myarray[2]
|
||||
^
|
||||
in function “func_echoâ€<C3A2>
|
||||
called on standard input
|
||||
with parameter list “function_works_over_sshâ€<C3A2>
|
||||
|
||||
|
||||
### csh environment
|
||||
|
||||
* Documentation: http://www.linuxmint.com
|
||||
|
|
|
|||
Loading…
Reference in a new issue