diff --git a/doc/release_new_version b/doc/release_new_version index ce1ea1cf..9f184cf5 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -258,6 +258,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> +* Unix parallel: Populating all the USB sticks http://www.markhneedham.com/blog/2016/06/01/unix-parallel-populating-all-the-usb-sticks/ + * GNU Parallel is a fantastic utility https://www.dray.be/parallel/ * Tools of the trade http://onox.com.br/2015/05/tools-of-the-trade/#more-198 diff --git a/src/parallel b/src/parallel index b9765cf0..869117f8 100755 --- a/src/parallel +++ b/src/parallel @@ -466,19 +466,15 @@ sub spreadstdin { } $Global::start_no_new_jobs ||= 1; if($opt::roundrobin) { - for my $job (values %Global::running) { - close $job->fh(0,"w"); - } - my %incomplete_jobs = %Global::running; + # Flush blocks to roundrobin procs my $sleep = 1; - while(keys %incomplete_jobs) { + while(%Global::running) { my $something_written = 0; - for my $pid (keys %incomplete_jobs) { - my $job = $incomplete_jobs{$pid}; + for my $job (values %Global::running) { if($job->block_length()) { $something_written += $job->non_blocking_write(); } else { - delete $incomplete_jobs{$pid} + close $job->fh(0,"w"); } } if($something_written) { @@ -6457,11 +6453,16 @@ sub write { my $len = length $$remaining_ref; # syswrite may not write all in one go, # so make sure everything is written. - while($len) { - my $written = syswrite($stdin_fh,$$remaining_ref); + my $written; + while($written = syswrite($stdin_fh,$$remaining_ref)){ substr($$remaining_ref,0,$written) = ""; - $len -= $written; + # Crazy cygwin - Without this, some data will silently be dropped + # seq 100000 | parallel --block 83k --pipe -kj3 cat | wc + $^O eq "cygwin" and ::usleep(200); } + # Crazy cygwin - Without this, some data will silently be dropped + syswrite($stdin_fh,""); + print $stdin_fh ""; } sub set_block { @@ -6511,25 +6512,28 @@ sub non_blocking_write { my $self = shift; my $something_written = 0; use POSIX qw(:errno_h); - # for loop used to avoid copying substr: $buf will be an alias for the substr - for my $buf (substr($self->{'block'},$self->{'block_pos'})) { - my $in = $self->fh(0,"w"); - my $rv = syswrite($in, $buf); - if (!defined($rv) && $! == EAGAIN) { - # would block - $something_written = 0; - } elsif ($self->{'block_pos'}+$rv != $self->{'block_length'}) { - # incomplete write - # Remove the written part - $self->{'block_pos'} += $rv; - $something_written = $rv; - } else { - # successfully wrote everything - my $a = ""; - $self->set_block(\$a,\$a,0,"",""); - $something_written = $rv; - } + + my $in = $self->fh(0,"w"); + my $rv = syswrite($in, + substr($self->{'block'},$self->{'block_pos'})); + if (!defined($rv) && $! == EAGAIN) { + # would block + $something_written = 0; + } elsif ($self->{'block_pos'}+$rv != $self->{'block_length'}) { + # incomplete write + # Remove the written part + $self->{'block_pos'} += $rv; + $something_written = $rv; + } else { + # successfully wrote everything + # Empty block to free memory + my $a = ""; + $self->set_block(\$a,\$a,0,"",""); + $something_written = $rv; } + # Crazy cygwin - Without this, some data will silently be dropped + # seq 100000 | parallel --block 83k --pipe -kj3 --round-robin cat | wc + $^O eq "cygwin" and ::usleep(200); ::debug("pipe", "Non-block: ", $something_written); return $something_written; @@ -7529,11 +7533,12 @@ sub start { # perl -e 'sleep(1);if(fork()){sleep(3)}while(not fork and time-$^T<8){ }' no warnings 'redefine'; my ($outfh,$name) = ::tmpfile(SUFFIX => ".tst"); - # Test to see if open3(,,,"-") is supported + # Test to see if open3(x,x,x,"-") is supported my $script = 'if(not $pid=::open3($i,$o,$e,"-")) { unlink shift }'; - qx{ perl -MIPC::Open3 -e '$script' $name }; + qx{ perl -MIPC::Open3 -e '$script' $name 2>/dev/null }; if(-e $name) { # Does not support open3(x,x,x,"-") + # Use (slow) external version unlink($name); *open3_setpgrp = \&open3_setpgrp_external; } else { diff --git a/testsuite/Start.sh b/testsuite/Start.sh index c686047f..64d81c11 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -12,6 +12,8 @@ run_test() { base=`basename "$script" .sh` export TMPDIR=/tmp/"$base"-tmpdir mkdir -p "$TMPDIR" + # Clean before. May be owned by other users + sudo rm -f /tmp/*.{tmx,pac,arg,all,log,swp,loa,ssh,df,pip,tmb,chr,tms,par} if [ "$TRIES" = "3" ] ; then # Try 3 times bash "$script" > actual-results/"$base" diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index b7c9d6f5..6203f7e7 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -22,38 +22,23 @@ echo '### tmux1.9' seq 700 800 | TMUX=tmux1.9 par_tmux seq 800 900 | TMUX=tmux1.9 par_tmux seq 900 1000 | TMUX=tmux1.9 par_tmux - seq 1000 1100 | TMUX=tmux1.9 par_tmux - seq 1100 1200 | TMUX=tmux1.9 par_tmux - seq 1200 1300 | TMUX=tmux1.9 par_tmux - seq 1300 1400 | TMUX=tmux1.9 par_tmux - seq 1400 1500 | TMUX=tmux1.9 par_tmux - seq 1500 1600 | TMUX=tmux1.9 par_tmux - seq 1600 1700 | TMUX=tmux1.9 par_tmux - seq 1700 1800 | TMUX=tmux1.9 par_tmux - seq 1800 1900 | TMUX=tmux1.9 par_tmux - seq 1900 2000 | TMUX=tmux1.9 par_tmux - seq 2000 2018 | TMUX=tmux1.9 par_tmux + seq 1000 1006 | TMUX=tmux1.9 par_tmux echo '### tmux1.9 fails' - echo 2019 | TMUX=tmux1.9 par_tmux - echo 2020 | TMUX=tmux1.9 par_tmux - echo 2021 | TMUX=tmux1.9 par_tmux - echo 2022 | TMUX=tmux1.9 par_tmux + echo 1007 | TMUX=tmux1.9 par_tmux + echo 1008 | TMUX=tmux1.9 par_tmux + echo 1009 | TMUX=tmux1.9 par_tmux echo '### tmux1.8' seq 1 50 | TMUX=tmux1.8 par_tmux seq 51 100 | TMUX=tmux1.8 par_tmux - seq 101 150 | TMUX=tmux1.8 par_tmux - seq 151 200 | TMUX=tmux1.8 par_tmux - seq 201 231 | TMUX=tmux1.8 par_tmux + seq 101 113 | TMUX=tmux1.8 par_tmux echo '### tmux1.8 fails' - echo 232 | TMUX=tmux1.8 par_tmux - echo 233 | TMUX=tmux1.8 par_tmux - echo 234 | TMUX=tmux1.8 par_tmux - echo 235 | TMUX=tmux1.8 par_tmux - echo 236 | TMUX=tmux1.8 par_tmux + echo 114 | TMUX=tmux1.8 par_tmux + echo 115 | TMUX=tmux1.8 par_tmux + echo 116 | TMUX=tmux1.8 par_tmux echo '### tmux1.8 0..255 ascii' -perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | +perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-110..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $? echo '### tmux1.9 0..255 ascii' diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index f5ca99fe..3e5e4aaa 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -72,6 +72,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ # Timings are often off s/^(\d)$/9/; s/^(\d\d)$/99/; + # Fails often due to race + s/cat: input_file: No such file or directory\n//; + s{rsync: link_stat "/home/parallel/input_file.out" .*\n}{}; + s{rsync error: some files/attrs were not transferred .*\n}{}; ' # 3+3 .par files (from --files), 1 .tms-file from tmux attach find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l diff --git a/testsuite/wanted-results/parallel-local-mem b/testsuite/wanted-results/parallel-local-mem index d0bb272c..3a654b81 100644 --- a/testsuite/wanted-results/parallel-local-mem +++ b/testsuite/wanted-results/parallel-local-mem @@ -8,7 +8,7 @@ echo '### bug #44358: 2 GB records cause problems for -N' ### bug #44358: 2 GB records cause problems for -N echo '5 GB version: Eats 12.5 GB' 5 GB version: Eats 12.5 GB - (yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c 8000000007 1000000001 parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. @@ -16,7 +16,7 @@ parallel: Warning: A record was longer than 1300000001. Increasing to --blocksiz parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. echo '2 GB version: eats 10 GB' 2 GB version: eats 10 GB - (yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c 4823449607 1048576001 parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. @@ -42,12 +42,12 @@ perl -e '$buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print ((length $buf) 0 echo 'Eats 4.7 GB' Eats 4.7 GB - (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum 8d87694eab8991a10815067709c3bd51 - parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. echo 'Eats 4.7 GB' Eats 4.7 GB - (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c 2411724804 parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. echo '**' diff --git a/testsuite/wanted-results/parallel-local7 b/testsuite/wanted-results/parallel-local7 index 89628452..05aaba35 100644 --- a/testsuite/wanted-results/parallel-local7 +++ b/testsuite/wanted-results/parallel-local7 @@ -30,52 +30,19 @@ See output with: tmux1.9 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach seq 900 1000 | TMUX=tmux1.9 par_tmux See output with: tmux1.9 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach 0 - seq 1000 1100 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007 -255 - seq 1100 1200 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (2209 >= 2023) at input 0: 1100 -255 - seq 1200 1300 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (2409 >= 2023) at input 0: 1200 -255 - seq 1300 1400 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (2609 >= 2023) at input 0: 1300 -255 - seq 1400 1500 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (2809 >= 2023) at input 0: 1400 -255 - seq 1500 1600 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (3009 >= 2023) at input 0: 1500 -255 - seq 1600 1700 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (3209 >= 2023) at input 0: 1600 -255 - seq 1700 1800 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (3409 >= 2023) at input 0: 1700 -255 - seq 1800 1900 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (3609 >= 2023) at input 0: 1800 -255 - seq 1900 2000 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (3809 >= 2023) at input 0: 1900 -255 - seq 2000 2018 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4009 >= 2023) at input 0: 2000 -255 + seq 1000 1006 | TMUX=tmux1.9 par_tmux +See output with: tmux1.9 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach +0 echo '### tmux1.9 fails' ### tmux1.9 fails - echo 2019 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4047 >= 2023) at input 0: 2019 + echo 1007 | TMUX=tmux1.9 par_tmux +parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007 255 - echo 2020 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4049 >= 2023) at input 0: 2020 + echo 1008 | TMUX=tmux1.9 par_tmux +parallel: Error: Command line too long (2025 >= 2023) at input 0: 1008 255 - echo 2021 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4051 >= 2023) at input 0: 2021 -255 - echo 2022 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4053 >= 2023) at input 0: 2022 + echo 1009 | TMUX=tmux1.9 par_tmux +parallel: Error: Command line too long (2027 >= 2023) at input 0: 1009 255 echo '### tmux1.8' ### tmux1.8 @@ -85,38 +52,24 @@ See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach seq 51 100 | TMUX=tmux1.8 par_tmux See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach 0 - seq 101 150 | TMUX=tmux1.8 par_tmux + seq 101 113 | TMUX=tmux1.8 par_tmux See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach -parallel: Error: Command line too long (236 >= 236) at input 0: 114 -255 - seq 151 200 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (310 >= 236) at input 0: 151 -255 - seq 201 231 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (410 >= 236) at input 0: 201 -255 +0 echo '### tmux1.8 fails' ### tmux1.8 fails - echo 232 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (472 >= 236) at input 0: 232 + echo 114 | TMUX=tmux1.8 par_tmux +parallel: Error: Command line too long (236 >= 236) at input 0: 114 255 - echo 233 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (474 >= 236) at input 0: 233 + echo 115 | TMUX=tmux1.8 par_tmux +parallel: Error: Command line too long (238 >= 236) at input 0: 115 255 - echo 234 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (476 >= 236) at input 0: 234 -255 - echo 235 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (478 >= 236) at input 0: 235 -255 - echo 236 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (480 >= 236) at input 0: 236 + echo 116 | TMUX=tmux1.8 par_tmux +parallel: Error: Command line too long (240 >= 236) at input 0: 116 255 echo '### tmux1.8 0..255 ascii' ### tmux1.8 0..255 ascii -perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $? +perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-110..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $? See output with: tmux1.8 -S /tmp/parallel-local7-tmpdir/tmsXXXXX attach -parallel: Error: Command line too long (236 >= 236) at input 0: 155  !"#$%&'()*+,-./0... 0 echo '### tmux1.9 0..255 ascii' ### tmux1.9 0..255 ascii