parallel: Warning if --pipe with --block >64k on CygWin.

This commit is contained in:
Ole Tange 2016-06-20 12:33:05 +02:00
parent 9d3c6997cd
commit da82bea501
6 changed files with 41 additions and 24 deletions

12
README
View file

@ -40,9 +40,9 @@ document.
Full installation of GNU Parallel is as simple as: Full installation of GNU Parallel is as simple as:
wget http://ftpmirror.gnu.org/parallel/parallel-20160522.tar.bz2 wget http://ftpmirror.gnu.org/parallel/parallel-20160622.tar.bz2
bzip2 -dc parallel-20160522.tar.bz2 | tar xvf - bzip2 -dc parallel-20160622.tar.bz2 | tar xvf -
cd parallel-20160522 cd parallel-20160622
./configure && make && sudo make install ./configure && make && sudo make install
@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as:
If you are not root you can add ~/bin to your path and install in If you are not root you can add ~/bin to your path and install in
~/bin and ~/share: ~/bin and ~/share:
wget http://ftpmirror.gnu.org/parallel/parallel-20160522.tar.bz2 wget http://ftpmirror.gnu.org/parallel/parallel-20160622.tar.bz2
bzip2 -dc parallel-20160522.tar.bz2 | tar xvf - bzip2 -dc parallel-20160622.tar.bz2 | tar xvf -
cd parallel-20160522 cd parallel-20160622
./configure --prefix=$HOME && make && make install ./configure --prefix=$HOME && make && make install
Or if your system lacks 'make' you can simply copy src/parallel Or if your system lacks 'make' you can simply copy src/parallel

View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20160522], [bug-parallel@gnu.org]) AC_INIT([parallel], [20160622], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h]) AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([ AC_CONFIG_FILES([

View file

@ -219,20 +219,29 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>, Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com> Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20160622 ('brexit/ali') released <<[stable]>> Subject: GNU Parallel 20160622 ('Orlando') released <<[stable]>>
GNU Parallel 20160622 ('brexit') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ GNU Parallel 20160622 ('Orlando') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
Haiku of the month: Haiku of the month:
<<>> Does path on remote
not include GNU Parallel?
Try --env PATH.
-- Ole Tange -- Ole Tange
New in this release: New in this release:
* The cookie from 2016-01-04 was won by Morgan Rodgers 5 months. * $PATH can now be exported using --env PATH. Useful if GNU Parallel is not in your path on remote machines.
* If --block is left out, --pipepart will use a block size that will result in 10 jobs per jobslot.
* The cookie from 2016-01-04 was won by Morgan Rodgers on the 2016-06-06 after 5 months.
* Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046
* <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>> * <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>
@ -260,12 +269,22 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
* <<link No citation: Next-generation TCP for ns-3 simulator http://www.sciencedirect.com/science/article/pii/S1569190X15300939>> * <<link No citation: Next-generation TCP for ns-3 simulator http://www.sciencedirect.com/science/article/pii/S1569190X15300939>>
* GNU Parallel was cited in: Genotypic Resistance Tests Sequences Reveal the Role of Marginalized Populations in HIV-1 Transmission in Switzerland http://www.nature.com/articles/srep27580
* GNU Parallel was cited in: A Deep-Learning Approach for Operation of an Automated Realtime Flare Forecast https://arxiv.org/pdf/1606.01587.pdf
* Unix parallel: Populating all the USB sticks http://www.markhneedham.com/blog/2016/06/01/unix-parallel-populating-all-the-usb-sticks/ * Unix parallel: Populating all the USB sticks http://www.markhneedham.com/blog/2016/06/01/unix-parallel-populating-all-the-usb-sticks/
* GNU Parallel is a fantastic utility https://www.dray.be/parallel/ * GNU Parallel is a fantastic utility https://www.dray.be/parallel/
* Combining a large number of VCF files https://gtamazian.com/2016/06/16/combining-a-large-number-of-vcf-files/
* Useful Scraping Techniques http://blog.apps.npr.org/2016/06/17/scraping-tips.html
* Tools of the trade http://onox.com.br/2015/05/tools-of-the-trade/#more-198 * Tools of the trade http://onox.com.br/2015/05/tools-of-the-trade/#more-198
* Armslist scraper https://github.com/nprapps/armslist-scraper/
* Bug fixes and man page updates. * Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane. GNU Parallel - For people who live life in the parallel lane.

View file

@ -24,7 +24,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20160523; $Global::version = 20160622;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

View file

@ -1012,6 +1012,11 @@ sub parse_options {
warning("--blocksize >= 2G causes problems. Using 2G-1."); warning("--blocksize >= 2G causes problems. Using 2G-1.");
$Global::blocksize = 2**31-1; $Global::blocksize = 2**31-1;
} }
if($^O eq "cygwin" and
($opt::pipe or $opt::pipepart or $opt::roundrobin)
and $Global::blocksize > 65535) {
warning("--blocksize >= 64K causes problems on Cygwin.");
}
$opt::memfree = multiply_binary_prefix($opt::memfree); $opt::memfree = multiply_binary_prefix($opt::memfree);
check_invalid_option_combinations(); check_invalid_option_combinations();
if((defined $opt::fifo or defined $opt::cat) if((defined $opt::fifo or defined $opt::cat)
@ -1190,7 +1195,7 @@ sub check_invalid_option_combinations {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20160529; $Global::version = 20160622;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -3053,6 +3058,9 @@ sub parse_host_filtering {
# ncores: server 8 # ncores: server 8
# ncpus: server 2 # ncpus: server 2
# maxlen: server 131071 # maxlen: server 131071
if(/parallel: Warning: Cannot figure out number of/) {
next;
}
if(not $ncores{$col[0]}) { if(not $ncores{$col[0]}) {
$ncores{$col[0]} = $col[1]; $ncores{$col[0]} = $col[1];
} elsif(not $ncpus{$col[0]}) { } elsif(not $ncpus{$col[0]}) {
@ -6456,13 +6464,7 @@ sub write {
my $written; my $written;
while($written = syswrite($stdin_fh,$$remaining_ref)){ while($written = syswrite($stdin_fh,$$remaining_ref)){
substr($$remaining_ref,0,$written) = ""; substr($$remaining_ref,0,$written) = "";
# Crazy cygwin - Without this, some data will silently be dropped
# seq 100000 | parallel --block 83k --pipe -kj3 cat | wc
$^O eq "cygwin" and ::usleep(200);
} }
# Crazy cygwin - Without this, some data will silently be dropped
syswrite($stdin_fh,"");
print $stdin_fh "";
} }
sub set_block { sub set_block {
@ -6531,10 +6533,6 @@ sub non_blocking_write {
$self->set_block(\$a,\$a,0,"",""); $self->set_block(\$a,\$a,0,"","");
$something_written = $rv; $something_written = $rv;
} }
# Crazy cygwin - Without this, some data will silently be dropped
# seq 100000 | parallel --block 83k --pipe -kj3 --round-robin cat | wc
$^O eq "cygwin" and ::usleep(200);
::debug("pipe", "Non-block: ", $something_written); ::debug("pipe", "Non-block: ", $something_written);
return $something_written; return $something_written;
} }

View file

@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20160523; $Global::version = 20160622;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself