Fixed bug #45769: --round-robin --pipepart gives wrong results.

This commit is contained in:
Ole Tange 2015-08-27 22:28:32 +02:00
parent f47bdf0a0d
commit 18ab03c98d
6 changed files with 106 additions and 5 deletions

View file

@ -231,6 +231,10 @@ New in this release:
* <<kontaktet 2015-06-22 Afventer svar fra journal>> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
* Joys of gnu parallel http://scottolesen.com/index.php/2015/08/26/joys-of-gnu-parallel/
* Crop and resize images with bash and ImageMagick https://www.simonholywell.com/post/2015/08/image-resize-crop-bash-imagemagick/
* Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane.

View file

@ -51,6 +51,7 @@ my @command = @ARGV;
my @input_source_fh;
if($opt::pipepart) {
# -a is used for data - not for command line args
@input_source_fh = map { open_or_exit($_) } "/dev/null";
} else {
@input_source_fh = map { open_or_exit($_) } @opt::a;
@ -115,16 +116,25 @@ if($opt::eta or $opt::bar or $opt::shuf or $Global::halt_pct) {
# before starting any
$Global::JobQueue->total_jobs();
}
# Compute $Global::max_jobs_running
for my $sshlogin (values %Global::host) {
$sshlogin->max_jobs_running();
}
if($opt::pipepart) {
if($opt::roundrobin) {
# Compute size of -a
my $size = 0;
$size += -s $_ for @opt::a;
$Global::max_jobs_running or ::die_bug("Global::max_jobs_running not set");
# Set --blocksize = size / no of proc
$opt::blocksize = 1 + $size / $Global::max_jobs_running;
}
@Global::cat_partials = map { pipe_part_files($_) } @opt::a;
# Unget the command as many times as there are parts
$Global::JobQueue->{'commandlinequeue'}->unget(
map { $Global::JobQueue->{'commandlinequeue'}->get() } @Global::cat_partials
);
}
for my $sshlogin (values %Global::host) {
$sshlogin->max_jobs_running();
}
init_run_jobs();
my $sem;
@ -916,7 +926,6 @@ sub parse_options {
if(defined $opt::colsep) { $Global::trim = 'lr'; }
if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
if(defined $opt::trim) { $Global::trim = $opt::trim; }
if(defined $opt::roundrobin) { $opt::pipe = 1; }
if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
@ -4834,7 +4843,9 @@ sub compute_number_of_processes {
$max_system_proc_reached and last;
my $before_getting_arg = time;
get_args_or_jobs() or last;
if(!$opt::roundrobin) {
get_args_or_jobs() or last;
}
$wait_time_for_getting_args += time - $before_getting_arg;
$system_limit++;

View file

@ -185,6 +185,15 @@ testquote() { printf '"#&/\n()*=?'"'" | PARALLEL_SHELL=$1 parallel -0 echo; };
echo '**'
echo '### bug #45769: --round-robin --pipepart gives wrong results'
seq 10000 >/tmp/seq10000;
parallel -j2 --pipepart -a /tmp/seq10000 --block 14 --round-robin wc | wc -l;
rm /tmp/seq10000
echo '**'
EOF
echo '### 1 .par file from --files expected'
find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l

View file

@ -210,8 +210,46 @@ env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
env_parallel -S ksh@lo func_echo ::: function_works_over_ssh
EOS
echo
echo "### Fish environment"
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
set myvar "myvar works"
setenv myenvvar "myenvvar works"
set funky (perl -e 'print pack "c*", 1..255')
setenv funkyenv (perl -e 'print pack "c*", 1..255')
set myarray '' array_val2 3 '' 5
#typeset -A assocarr
#assocarr[a]=assoc_val_a
#assocarr[b]=assoc_val_b
alias alias_echo="echo 3 arg";
function func_echo
echo $argv;
echo "$myvar"
echo "$myenvvar"
echo $myarray[2]
# echo ${assocarr[a]}
echo Funky-"$funky"-funky
echo Funky-"$funkyenv"-funky
end
function env_parallel
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/')
parallel $argv;
set -e PARALLEL_ENV
end
env_parallel alias_echo ::: alias_does_not_work
env_parallel func_echo ::: function_works
env_parallel -S fish@lo alias_echo ::: alias_does_not_work_over_ssh
env_parallel -S fish@lo func_echo ::: function_works_over_ssh
EOS
echo
echo "### csh environment"
# http://hyperpolyglot.org/unix-shells
stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
set myvar = "myvar works"
set funky = "`perl -e 'print pack q(c*), 1..255'`"

View file

@ -384,5 +384,11 @@ zsh "#&/
zsh ()*=?'
echo '**'
**
echo '### bug #45769: --round-robin --pipepart gives wrong results'
### bug #45769: --round-robin --pipepart gives wrong results
seq 10000 >/tmp/seq10000; parallel -j2 --pipepart -a /tmp/seq10000 --block 14 --round-robin wc | wc -l; rm /tmp/seq10000
2
echo '**'
**
### 1 .par file from --files expected
1

View file

@ -177,6 +177,39 @@ assoc_val_a
Funky-
 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、¥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
### Fish environment
* Documentation: http://www.linuxmint.com
0 updates are security updates.
3 arg alias_does_not_work
function_works
myenvvar works
Funky--funky
Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~<7F>ƒ„…†‡ˆ‰ŠŒ<E280B9>Ž<EFBFBD><C5BD>“”•˜™šœ<E280BA>žŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky
Array index out of bounds
fish: echo $myarray[2]
^
in function “func_echoâ€<C3A2>
called on standard input
with parameter list “function_worksâ€<C3A2>
3 arg alias_does_not_work_over_ssh
function_works_over_ssh
Funky--funky
Funky--funky
Array index out of bounds
fish: echo $myarray[2]
^
in function “func_echoâ€<C3A2>
called on standard input
with parameter list “function_works_over_sshâ€<C3A2>
### csh environment
* Documentation: http://www.linuxmint.com