tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #48074: --pipe --block 90k blocks on windows.

Browse source 
Fixed bug with open3(x,x,x,"-") detection.
Cleaned up some tests.
This commit is contained in:
Ole Tange 2016-06-04 15:49:01 +02:00
parent 7542609ef1
commit bf0571e4a7
7 changed files with 75 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/release_new_version
View file

@ -258,6 +258,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
* <<link No citation: Next-generation TCP for ns-3 simulator http://www.sciencedirect.com/science/article/pii/S1569190X15300939>>
* Unix parallel: Populating all the USB sticks http://www.markhneedham.com/blog/2016/06/01/unix-parallel-populating-all-the-usb-sticks/
* GNU Parallel is a fantastic utility https://www.dray.be/parallel/
* Tools of the trade http://onox.com.br/2015/05/tools-of-the-trade/#more-198

67
src/parallel
View file

@ -466,19 +466,15 @@ sub spreadstdin {
}
$Global::start_no_new_jobs ||= 1;
if($opt::roundrobin) {
for my $job (values %Global::running) {
close $job->fh(0,"w");
}
my %incomplete_jobs = %Global::running;
# Flush blocks to roundrobin procs
my $sleep = 1;
while(keys %incomplete_jobs) {
while(%Global::running) {
my $something_written = 0;
for my $pid (keys %incomplete_jobs) {
my $job = $incomplete_jobs{$pid};
for my $job (values %Global::running) {
if($job->block_length()) {
$something_written += $job->non_blocking_write();
} else {
delete $incomplete_jobs{$pid}
close $job->fh(0,"w");
}
}
if($something_written) {
@ -6457,11 +6453,16 @@ sub write {
my $len = length $$remaining_ref;
# syswrite may not write all in one go,
# so make sure everything is written.
while($len) {
my $written = syswrite($stdin_fh,$$remaining_ref);
my $written;
while($written = syswrite($stdin_fh,$$remaining_ref)){
substr($$remaining_ref,0,$written) = "";
$len -= $written;
# Crazy cygwin - Without this, some data will silently be dropped
# seq 100000 | parallel --block 83k --pipe -kj3 cat | wc
$^O eq "cygwin" and ::usleep(200);
}
# Crazy cygwin - Without this, some data will silently be dropped
syswrite($stdin_fh,"");
print $stdin_fh "";
}
sub set_block {
@ -6511,25 +6512,28 @@ sub non_blocking_write {
my $self = shift;
my $something_written = 0;
use POSIX qw(:errno_h);
# for loop used to avoid copying substr: $buf will be an alias for the substr
for my $buf (substr($self->{'block'},$self->{'block_pos'})) {
my $in = $self->fh(0,"w");
my $rv = syswrite($in, $buf);
if (!defined($rv) && $! == EAGAIN) {
# would block
$something_written = 0;
} elsif ($self->{'block_pos'}+$rv != $self->{'block_length'}) {
# incomplete write
# Remove the written part
$self->{'block_pos'} += $rv;
$something_written = $rv;
} else {
# successfully wrote everything
my $a = "";
$self->set_block(\$a,\$a,0,"","");
$something_written = $rv;
}
my $in = $self->fh(0,"w");
my $rv = syswrite($in,
substr($self->{'block'},$self->{'block_pos'}));
if (!defined($rv) && $! == EAGAIN) {
# would block
$something_written = 0;
} elsif ($self->{'block_pos'}+$rv != $self->{'block_length'}) {
# incomplete write
# Remove the written part
$self->{'block_pos'} += $rv;
$something_written = $rv;
} else {
# successfully wrote everything
# Empty block to free memory
my $a = "";
$self->set_block(\$a,\$a,0,"","");
$something_written = $rv;
}
# Crazy cygwin - Without this, some data will silently be dropped
# seq 100000 | parallel --block 83k --pipe -kj3 --round-robin cat | wc
$^O eq "cygwin" and ::usleep(200);
::debug("pipe", "Non-block: ", $something_written);
return $something_written;
@ -7529,11 +7533,12 @@ sub start {
# perl -e 'sleep(1);if(fork()){sleep(3)}while(not fork and time-$^T<8){ }'
no warnings 'redefine';
my ($outfh,$name) = ::tmpfile(SUFFIX => ".tst");
# Test to see if open3(,,,"-") is supported
# Test to see if open3(x,x,x,"-") is supported
my $script = 'if(not $pid=::open3($i,$o,$e,"-")) { unlink shift }';
qx{ perl -MIPC::Open3 -e '$script' $name };
qx{ perl -MIPC::Open3 -e '$script' $name 2>/dev/null };
if(-e $name) {
# Does not support open3(x,x,x,"-")
# Use (slow) external version
unlink($name);
*open3_setpgrp = \&open3_setpgrp_external;
} else {

2
testsuite/Start.sh
View file

@ -12,6 +12,8 @@ run_test() {
base=`basename "$script" .sh`
export TMPDIR=/tmp/"$base"-tmpdir
mkdir -p "$TMPDIR"
# Clean before. May be owned by other users
sudo rm -f /tmp/*.{tmx,pac,arg,all,log,swp,loa,ssh,df,pip,tmb,chr,tms,par}
if [ "$TRIES" = "3" ] ; then
# Try 3 times
bash "$script" > actual-results/"$base"

33
testsuite/tests-to-run/parallel-local7.sh
View file

@ -22,38 +22,23 @@ echo '### tmux1.9'
seq 700 800 | TMUX=tmux1.9 par_tmux
seq 800 900 | TMUX=tmux1.9 par_tmux
seq 900 1000 | TMUX=tmux1.9 par_tmux
seq 1000 1100 | TMUX=tmux1.9 par_tmux
seq 1100 1200 | TMUX=tmux1.9 par_tmux
seq 1200 1300 | TMUX=tmux1.9 par_tmux
seq 1300 1400 | TMUX=tmux1.9 par_tmux
seq 1400 1500 | TMUX=tmux1.9 par_tmux
seq 1500 1600 | TMUX=tmux1.9 par_tmux
seq 1600 1700 | TMUX=tmux1.9 par_tmux
seq 1700 1800 | TMUX=tmux1.9 par_tmux
seq 1800 1900 | TMUX=tmux1.9 par_tmux
seq 1900 2000 | TMUX=tmux1.9 par_tmux
seq 2000 2018 | TMUX=tmux1.9 par_tmux
seq 1000 1006 | TMUX=tmux1.9 par_tmux
echo '### tmux1.9 fails'
echo 2019 | TMUX=tmux1.9 par_tmux
echo 2020 | TMUX=tmux1.9 par_tmux
echo 2021 | TMUX=tmux1.9 par_tmux
echo 2022 | TMUX=tmux1.9 par_tmux
echo 1007 | TMUX=tmux1.9 par_tmux
echo 1008 | TMUX=tmux1.9 par_tmux
echo 1009 | TMUX=tmux1.9 par_tmux
echo '### tmux1.8'
seq 1 50 | TMUX=tmux1.8 par_tmux
seq 51 100 | TMUX=tmux1.8 par_tmux
seq 101 150 | TMUX=tmux1.8 par_tmux
seq 151 200 | TMUX=tmux1.8 par_tmux
seq 201 231 | TMUX=tmux1.8 par_tmux
seq 101 113 | TMUX=tmux1.8 par_tmux
echo '### tmux1.8 fails'
echo 232 | TMUX=tmux1.8 par_tmux
echo 233 | TMUX=tmux1.8 par_tmux
echo 234 | TMUX=tmux1.8 par_tmux
echo 235 | TMUX=tmux1.8 par_tmux
echo 236 | TMUX=tmux1.8 par_tmux
echo 114 | TMUX=tmux1.8 par_tmux
echo 115 | TMUX=tmux1.8 par_tmux
echo 116 | TMUX=tmux1.8 par_tmux
echo '### tmux1.8 0..255 ascii'
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' |
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-110..$_),"\n" } 0..255' |
TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
echo '### tmux1.9 0..255 ascii'

4
testsuite/tests-to-run/parallel-tutorial.sh
View file

@ -72,6 +72,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
# Timings are often off
s/^(\d)$/9/;
s/^(\d\d)$/99/;
# Fails often due to race
s/cat: input_file: No such file or directory\n//;
s{rsync: link_stat "/home/parallel/input_file.out" .*\n}{};
s{rsync error: some files/attrs were not transferred .*\n}{};
'
# 3+3 .par files (from --files), 1 .tms-file from tmux attach
find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l

8
testsuite/wanted-results/parallel-local-mem
View file

@ -8,7 +8,7 @@ echo '### bug #44358: 2 GB records cause problems for -N'
### bug #44358: 2 GB records cause problems for -N
echo '5 GB version: Eats 12.5 GB'
5 GB version: Eats 12.5 GB
(yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c
PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c
8000000007
1000000001
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001.
@ -16,7 +16,7 @@ parallel: Warning: A record was longer than 1300000001. Increasing to --blocksiz
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647.
echo '2 GB version: eats 10 GB'
2 GB version: eats 10 GB
(yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c
PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c
4823449607
1048576001
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001.
@ -42,12 +42,12 @@ perl -e '$buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print ((length $buf)
0
echo 'Eats 4.7 GB'
Eats 4.7 GB
(yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum
PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum
8d87694eab8991a10815067709c3bd51 -
parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1.
echo 'Eats 4.7 GB'
Eats 4.7 GB
(yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c
PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c
2411724804
parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1.
echo '**'

83
testsuite/wanted-results/parallel-local7
View file

@ -30,52 +30,19 @@ See output with: tmux1.9 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
seq 900 1000 | TMUX=tmux1.9 par_tmux
See output with: tmux1.9 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
0
seq 1000 1100 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007
255
seq 1100 1200 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2209 >= 2023) at input 0: 1100
255
seq 1200 1300 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2409 >= 2023) at input 0: 1200
255
seq 1300 1400 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2609 >= 2023) at input 0: 1300
255
seq 1400 1500 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2809 >= 2023) at input 0: 1400
255
seq 1500 1600 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (3009 >= 2023) at input 0: 1500
255
seq 1600 1700 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (3209 >= 2023) at input 0: 1600
255
seq 1700 1800 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (3409 >= 2023) at input 0: 1700
255
seq 1800 1900 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (3609 >= 2023) at input 0: 1800
255
seq 1900 2000 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (3809 >= 2023) at input 0: 1900
255
seq 2000 2018 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (4009 >= 2023) at input 0: 2000
255
seq 1000 1006 | TMUX=tmux1.9 par_tmux
See output with: tmux1.9 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
0
echo '### tmux1.9 fails'
### tmux1.9 fails
echo 2019 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (4047 >= 2023) at input 0: 2019
echo 1007 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007
255
echo 2020 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (4049 >= 2023) at input 0: 2020
echo 1008 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2025 >= 2023) at input 0: 1008
255
echo 2021 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (4051 >= 2023) at input 0: 2021
255
echo 2022 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (4053 >= 2023) at input 0: 2022
echo 1009 | TMUX=tmux1.9 par_tmux
parallel: Error: Command line too long (2027 >= 2023) at input 0: 1009
255
echo '### tmux1.8'
### tmux1.8
@ -85,38 +52,24 @@ See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
seq 51 100 | TMUX=tmux1.8 par_tmux
See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
0
seq 101 150 | TMUX=tmux1.8 par_tmux
seq 101 113 | TMUX=tmux1.8 par_tmux
See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
parallel: Error: Command line too long (236 >= 236) at input 0: 114
255
seq 151 200 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (310 >= 236) at input 0: 151
255
seq 201 231 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (410 >= 236) at input 0: 201
255
0
echo '### tmux1.8 fails'
### tmux1.8 fails
echo 232 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (472 >= 236) at input 0: 232
echo 114 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (236 >= 236) at input 0: 114
255
echo 233 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (474 >= 236) at input 0: 233
echo 115 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (238 >= 236) at input 0: 115
255
echo 234 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (476 >= 236) at input 0: 234
255
echo 235 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (478 >= 236) at input 0: 235
255
echo 236 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (480 >= 236) at input 0: 236
echo 116 | TMUX=tmux1.8 par_tmux
parallel: Error: Command line too long (240 >= 236) at input 0: 116
255
echo '### tmux1.8 0..255 ascii'
### tmux1.8 0..255 ascii
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-110..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach
parallel: Error: Command line too long (236 >= 236) at input 0: 155  !"#$%&'()*+,-./0...
0
echo '### tmux1.9 0..255 ascii'
### tmux1.9 0..255 ascii