parallel: Use open3(x,x,x,"-") for setpgrp if available.

doc/boxplot-runtime

@@ -32,11 +32,10 @@ measure() {
     INNER=$2
     CORES=$3
     VERSION=$4
-    GHZ=3.0
+    MHZ=1700
 
     # Force cpuspeed at 1.7GHz - seems to give tighter results
-#    forever 'sleep 10;parallel sudo cpufreq-set -f ${GHZ}GHz -c{} ::: {0..7}' &
+    forever 'parallel sudo cpufreq-set -g performance -u '$MHZ'MHz -d '$MHZ'MHz -c{} ::: {0..3};sleep 10' &
-#    forever 'sleep 10;parallel sudo cpufreq-set -f 1700MHz -c{} ::: {0..7}' &
 
     PATH=/tmp/bin:$PATH
     cd /tmp/bin
@@ -52,8 +51,11 @@ measure() {
       boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F,
               ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER");
 _
-    cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf
+    cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${MHZ}MHz-$OUTER-${INNER}v$VERSION.pdf
     evince /tmp/boxplot.pdf
 }
 
+#measure 3000 1000 2 1
+measure 30 10 2 1
+measure 300 100 2 1
 measure 3000 1000 2 1

doc/release_new_version

@@ -227,17 +227,19 @@ GNU Parallel 20160522 ('ttipleaks') <<[stable]>> has been released. It is availa
 
 Haiku of the month:
 
-<<>>
+Programs using net
+only spare capacity
+niceload dash dash net
     -- Ole Tange
 
 New in this release:
 
-* niceload --net
+* niceload --net pauses the program if the internet connection is overloaded.
-
-* 
 
 * Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046
 
+* Updated speed comparison between versions https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf
+
 * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>>
 
 * <<Har angiveligt submittet ny version - afventer opdatering>> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
@@ -254,7 +256,41 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
 
 * <<Citation needed: Introspecting for RSA Key Material to Assist Intrusion Detection http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=7331177&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7331177>>
 
-* GNU Parallel was cited in: Structure and evolutionary history of a large family of NLR proteins in the zebrafish http://rsob.royalsocietypublishing.org/content/royopenbio/6/4/160009.full.pdf
+* <<Citation needed: 3D-GNOME: an integrated web service for structural modeling of the 3D genome http://nar.oxfordjournals.org/content/early/2016/05/16/nar.gkw437.full.pdf+html>>
+
+* <<link No citation: Next-generation TCP for ns-3 simulator http://www.sciencedirect.com/science/article/pii/S1569190X15300939>>
+
+* GNU Parallel was cited in: Improving computation efficiency by parallel programming http://www.irbis-nbuv.gov.ua/cgi-bin/irbis_nbuv/cgiirbis_64.exe?C21COM=2&I21DBN=UJRN&P21DBN=UJRN&IMAGE_FILE_DOWNLOAD=1&Image_file_name=PDF/ape_2013_3_44.pdf
+
+* GNU Parallel was cited in: A supernova feedback implementation for the astrophysical simulation software Arepo https://arxiv.org/abs/1604.06071
+
+* GNU Parallel was cited in: Lorenz-Mie theory for 2D scattering and resonance calculations https://arxiv.org/pdf/1505.07691v2.pdf
+
+* GNU Parallel was cited in: Host-pathogen co-evolution and the emergence of broadly neutralizing antibodies in chronic infections https://arxiv.org/abs/1512.06296
+
+* GNU Parallel was cited in: Pacific People, Metabolic Disease and Evolutionary Processes: a mitochondrial DNA study https://otago.ourarchive.ac.nz/handle/10523/6340
+
+* GNU Parallel was cited in: The effect of domain modeling on efficiency of planning: Lessons from the Nomystery domain http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7407131
+
+* GNU Parallel was cited in: Oops, my tests broke the build: An analysis of Travis CI buildswith GitHub https://peerj.com/preprints/1984/
+
+* GNU Parallel was cited in: Drosophila Muller F Elements Maintain a Distinct Set of Genomic Properties Over 40 Million Years of Evolution http://www.g3journal.org/content/5/5/719.full.pdf+html
+
+* GNU Parallel was cited in: An Empirical Comparison of Neural Architectures for Reinforcement Learning in Partially Observable Environments http://biorxiv.org/content/biorxiv/early/2016/03/24/022707.full.pdf
+
+* GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/early/2016/04/11/048066.abstract
+
+* GNU Parallel was cited in: PleaseTM: Enabling Transaction Conflict Management in Requester-wins Hardware Transactional Memory http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7446072
+
+* GNU Parallel was cited in: EASE-MM: Sequence-Based Prediction of Mutation-Induced Stability Changes with Feature-Based Multiple Models http://www.sciencedirect.com/science/article/pii/S0022283616000310
+
+* GNU Parallel was cited in: A new orthology assessment method for phylogenomic data: Unrooted Phylogenetic Orthology http://mbe.oxfordjournals.org/content/early/2016/04/06/molbev.msw069.short https://github.com/ballesterus/UPhO
+
+* GNU Parallel was cited in: Distinctive Interest Point Selection for Efficient Near-duplicate Image Retrieval http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7459172
+
+* GNU Parallel was cited in: The Evolution of C Programming Practices: A Study of the Unix Operating System 1973–2015 https://dl.acm.org/citation.cfm?id=2884799 (It has the cutest thumbnail graphs I have ever seen scattered all over the text)
+
+* GNU Parallel was cited in: StrAuto: Automation and Parallelization of STRUCTURE Analysis http://vchhatre.w3.uvm.edu/download/strauto/strauto_doc.pdf
 
 * GNU Parallel was cited in: Data processing pipeline for serial femtosecond crystallography at SACLA http://journals.iucr.org/j/issues/2016/03/00/zw5001/index.html
 
@@ -274,16 +310,24 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
 
 * GNU Parallel was cited in: How attention influences perceptual decision making: Single-trial EEG correlates of drift-diffusion model parameters http://www.cidlab.com/prints/nunez2016attention.pdf
 
+* GNU Parallel was cited in: Telomere And Proximal Sequence Analysis Using High-Throughput Sequencing Reads http://repository.upenn.edu/edissertations/1460/
+
 * GNU Parallel was cited in: AT-GIS: Highly Parallel Spatial Query Processing with Associative Transducers http://lsds.doc.ic.ac.uk/sites/default/files/ATGIS-SIGMOD16.pdf
 
 * GNU Parallel was cited in: PVAIR: Partial Variable Assignment InterpolatoR http://verify.inf.usi.ch/sites/default/files/main-2.pdf
 
+* GNU Parallel was cited in: Comparative Cladistics: Fossils, Morphological Data Partitions and Lost Branches in the Fossil Tree of Life http://opus.bath.ac.uk/43955/ 
+
 * GNU Parallel was cited in: Shannon: An Information-Optimal de NovoRNA-Seq Assembler http://biorxiv.org/content/biorxiv/early/2016/02/09/039230.full.pdf
 
+* GNU Parallel was cited in: Adaptive Measure-Theoretic Parameter Estimation for Coastal Ocean Modeling https://repositories.lib.utexas.edu/handle/2152/32435
+
 * GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16
 
 * GNU Parallel was cited in: Reference genotype and exome data from an Australian Aboriginal population for health-based research http://www.nature.com/articles/sdata201623
 
+* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16
+
 * GNU Parallel was cited in: Do aye-ayes echolocate? http://biorxiv.org/content/biorxiv/early/2016/04/11/048165.full.pdf
 
 * GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/biorxiv/early/2016/04/11/048066.full.pdf

src/niceload

@@ -321,7 +321,7 @@ sub get_options_from_array {
 	$opt::load = -1;
     }
     if($opt::net) {
-	$opt::nethops ||= 2;
+	$opt::nethops ||= 3;
     }
     if($opt::nethops) {
 	# niceload -l 0.01 --sensor 'netsensor_script'

src/niceload.pod

@@ -104,7 +104,7 @@ B<--noswap> will set both B<--start-noswap> and B<run-noswap>.
 
 =item B<--net> (alpha testing)
 
-Shorthand for B<--nethops 2>.
+Shorthand for B<--nethops 3>.
 
 
 =item B<--nethops> I<h> (alpha testing)

src/parallel

@@ -7482,7 +7482,7 @@ sub start {
 	eval {
 	    if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
 		# Each child gets its own process group to make it safe to killall
-		setpgrp(0,0);
+		setpgrp(0,0) || ::die_bug("setpgrp failed");
 		exec("exec $Global::shell -c ".::shell_quote_scalar_default($command))
 		    || ::die_bug("open3-$stdin_fh $command");
 	    }
@@ -7520,16 +7520,23 @@ sub start {
     }
 
     sub open3_setpgrp {
-	# If the OS supports open3(x,x,x,"-") use that
+	# Select and run open3_setpgrp_internal/open3_setpgrp_external
-	# eval { if(not $pid=::open3($i,$o,$e,"-")) { exit } }
+	# parallel -j0 --timeout 5 '( (sleep {}) & )& sleep 7' ::: {1..55} &
-	# if $!: external
+	# ps -opid,ppid,pgrp,cmd
-	# TODO build a selector that works with out side effects
+	# parallel --timeout 6 -q perl -e 'sleep(1);if(fork()){sleep(30)}while(not fork and $t++<30000){ }' ::: 1
+	# perl -e 'sleep(1);if(fork()){sleep(3)}while(not fork and time-$^T<8){ }'
 	no warnings 'redefine';
-	if(1) {
+	my ($outfh,$name) = ::tmpfile(SUFFIX => ".tst");
+	# Test to see if open3(,,,"-") is supported
+	my $script = 'if(not $pid=::open3($i,$o,$e,"-")) { unlink shift }';
+	qx{ perl -MIPC::Open3 -e '$script' $name };
+	if(-e $name) {
 	    # Does not support open3(x,x,x,"-")
+	    unlink($name);
 	    *open3_setpgrp = \&open3_setpgrp_external;
 	} else {
 	    # Supports open3(x,x,x,"-")
+	    # This is 0.5 ms faster to run
 	    *open3_setpgrp = \&open3_setpgrp_internal;
 	}
 	# The sub is now redefined. Call it

src/parallel.pod

@@ -1272,10 +1272,10 @@ with 'y' or 'Y'.  Implies B<-t>.
 
 =item B<--parens> I<parensstring>
 
-Use to define start and end parenthesis for B<{= perl expression =}>. The
+Define start and end parenthesis for B<{= perl expression =}>. The
 left and the right parenthesis can be multiple characters and are
-assumed to be the same length. The default is B<{==}> giving
+assumed to be the same length. The default is B<{==}> giving B<{=> as
-B<{=> as the start parenthesis and B<=}> as the end parenthesis.
+the start parenthesis and B<=}> as the end parenthesis.
 
 Another useful setting is B<,,,,> which would make both parenthesis
 B<,,>:
@@ -1546,7 +1546,7 @@ it possible to define your own replacement strings. GNU B<parallel>'s
   --rpl '{/.} s:.*/::; s:\.[^/.]+$::;'
   --rpl '{.} s:\.[^/.]+$::'
 
-The <--plus> replacement strings are implemented as:
+The B<--plus> replacement strings are implemented as:
 
   --rpl '{+/} s:/[^/]*$::'
   --rpl '{+.} s:.*\.::'
@@ -1567,12 +1567,12 @@ of GNU B<parallel>'s internal functions and data structures.
 
 Here are a few examples:
 
-  Remove 2 extensions (e.g. .tar.gz)
-  --rpl '{..} s:\.[^/.]+$::;s:\.[^/.]+$::;'
-  Keep only the extension
-  --rpl '{ext} s:.*\.::'
   Is the job sequence even or odd?
-  --rpl '{odd} $_=$job->seq()%2?"odd":"even"'
+  --rpl '{odd} $_ = $job->seq() % 2 ? "odd" : "even"'
+  Pad job sequence with leading zeros to get equal width
+  --rpl '{0#} $f = "%0".int(1+log(total_jobs())/log(10))."d"; $_=sprintf($f,$job->seq())'
+  Job sequence counting from 0
+  --rpl '{#0} $_ = $job->seq() - 1'
 
 See also: B<{= perl expression =}> B<--parens>
 
@@ -1981,13 +1981,13 @@ Use B<tmux> for output. Start a B<tmux> session and run each job in a
 window in that session. No other output will be produced.
 
 
-=item B<--timeout> I<val>
+=item B<--timeout> I<secs>
 
-Time out for command. If the command runs for longer than I<val>
+Time out for command. If the command runs for longer than I<secs>
 seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms
 later, followed by SIGKILL 200 ms later.
 
-If I<val> is followed by a % then the timeout will dynamically be
+If I<secs> is followed by a % then the timeout will dynamically be
 computed as a percentage of the median average runtime. Only values
 > 100% will make sense.
 

src/parallel_tutorial.html

@@ -2561,7 +2561,7 @@
   Web site: http://www.gnu.org/software/parallel
   
   When using programs that use GNU Parallel to process data for publication
-  please cite as described in &#39;parallel --bibtex&#39;.</code></pre>
+  please cite as described in &#39;parallel --citation&#39;.</code></pre>
 
 <p>In scripts <b>--minversion</b> can be used to ensure the user has at least this version:</p>
 
@@ -2572,9 +2572,9 @@
 <pre><code>  20160322
   Your version is at least 20130722.</code></pre>
 
-<p>If using GNU <b>parallel</b> for research the BibTeX citation can be generated using <b>--bibtex</b>:</p>
+<p>If you are using GNU <b>parallel</b> for research the BibTeX citation can be generated using <b>--citation</b>:</p>
 
-<pre><code>  parallel --bibtex</code></pre>
+<pre><code>  parallel --citation</code></pre>
 
 <p>Output:</p>
 
@@ -2661,10 +2661,10 @@
 <li><p>(Re-)walk through the tutorial if you have not done so in the past year (http://www.gnu.org/software/parallel/parallel_tutorial.html)</p>
 
 </li>
-<li><p>Give a demo at your local user group/team/colleagues</p>
+<li><p>Give a demo at your local user group/your team/your colleagues</p>
 
 </li>
-<li><p>Post the intro videos and the tutorial on Reddit, Diaspora*, forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, mailing lists</p>
+<li><p>Post the intro videos and the tutorial on Reddit, Diaspora*, forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, and mailing lists</p>
 
 </li>
 <li><p>Request or write a review for your favourite blog or magazine (especially if you do something cool with GNU <b>parallel</b>)</p>
@@ -2679,7 +2679,7 @@
 
 <ul>
 
-<li><p>Please cite GNU <b>parallel</b> in you publications (use <b>--bibtex</b>)</p>
+<li><p>Please cite GNU <b>parallel</b> in you publications (use <b>--citation</b>)</p>
 
 </li>
 </ul>

src/parallel_tutorial.pod

@@ -2603,7 +2603,7 @@ Output:
   Web site: http://www.gnu.org/software/parallel
   
   When using programs that use GNU Parallel to process data for publication
-  please cite as described in 'parallel --bibtex'.
+  please cite as described in 'parallel --citation'.
 
 In scripts B<--minversion> can be used to ensure the user has at least
 this version:
@@ -2615,10 +2615,10 @@ Output:
   20160322
   Your version is at least 20130722.
 
-If using GNU B<parallel> for research the BibTeX citation can be
+If you are using GNU B<parallel> for research the BibTeX citation can be
-generated using B<--bibtex>:
+generated using B<--citation>:
 
-  parallel --bibtex
+  parallel --citation
 
 Output:
 
@@ -2714,13 +2714,13 @@ If you like GNU B<parallel>:
 
 =item *
 
-Give a demo at your local user group/team/colleagues
+Give a demo at your local user group/your team/your colleagues
 
 =item *
 
 Post the intro videos and the tutorial on Reddit, Diaspora*,
 forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin,
-mailing lists
+and mailing lists
 
 =item *
 
@@ -2739,7 +2739,7 @@ If you use GNU B<parallel> for research:
 
 =item *
 
-Please cite GNU B<parallel> in you publications (use B<--bibtex>)
+Please cite GNU B<parallel> in you publications (use B<--citation>)
 
 =back
 

testsuite/tests-to-run/parallel-polarhome.sh

@@ -16,7 +16,7 @@ S_POLAR=`parallel -k echo -S 1/{}.polarhome.com ::: $P`
 # 20150414 --timeout 80 -> 40
 # 20151219 --retries 5 -> 2
 TIMEOUT=10
-RETRIES=2
+RETRIES=4
 
 echo '### Tests on polarhome machines'
 echo 'Setup on polarhome machines'

testsuite/wanted-results/parallel-tutorial

@@ -1269,7 +1269,7 @@ please cite as described in 'parallel --citation'.
   parallel --minversion VERSION && echo Your version is at least VERSION.
 VERSION
 Your version is at least VERSION.
-  parallel --bibtex
+  parallel --citation
 Academic tradition requires you to cite works you base your article on.
 When using programs that use GNU Parallel to process data for publication
 please cite: