From 4e909725846f7662927ea9cb163aedfd3c3e3966 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 22 May 2016 14:08:47 +0200 Subject: [PATCH] parallel: Use open3(x,x,x,"-") for setpgrp if available. --- doc/boxplot-runtime | 10 ++-- doc/release_new_version | 54 ++++++++++++++++++-- src/niceload | 2 +- src/niceload.pod | 2 +- src/parallel | 19 ++++--- src/parallel.pod | 24 ++++----- src/parallel_tutorial.html | 12 ++--- src/parallel_tutorial.pod | 14 ++--- testsuite/tests-to-run/parallel-polarhome.sh | 2 +- testsuite/wanted-results/parallel-tutorial | 2 +- 10 files changed, 97 insertions(+), 44 deletions(-) diff --git a/doc/boxplot-runtime b/doc/boxplot-runtime index 0261faab..31c209ad 100644 --- a/doc/boxplot-runtime +++ b/doc/boxplot-runtime @@ -32,11 +32,10 @@ measure() { INNER=$2 CORES=$3 VERSION=$4 - GHZ=3.0 + MHZ=1700 # Force cpuspeed at 1.7GHz - seems to give tighter results -# forever 'sleep 10;parallel sudo cpufreq-set -f ${GHZ}GHz -c{} ::: {0..7}' & -# forever 'sleep 10;parallel sudo cpufreq-set -f 1700MHz -c{} ::: {0..7}' & + forever 'parallel sudo cpufreq-set -g performance -u '$MHZ'MHz -d '$MHZ'MHz -c{} ::: {0..3};sleep 10' & PATH=/tmp/bin:$PATH cd /tmp/bin @@ -52,8 +51,11 @@ measure() { boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F, ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER"); _ - cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf + cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${MHZ}MHz-$OUTER-${INNER}v$VERSION.pdf evince /tmp/boxplot.pdf } +#measure 3000 1000 2 1 +measure 30 10 2 1 +measure 300 100 2 1 measure 3000 1000 2 1 diff --git a/doc/release_new_version b/doc/release_new_version index 35ce8f98..3ab4ece0 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -227,17 +227,19 @@ GNU Parallel 20160522 ('ttipleaks') <<[stable]>> has been released. It is availa Haiku of the month: -<<>> +Programs using net +only spare capacity +niceload dash dash net -- Ole Tange New in this release: -* niceload --net - -* +* niceload --net pauses the program if the internet connection is overloaded. * Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046 +* Updated speed comparison between versions https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf + * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>> * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 @@ -254,7 +256,41 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> -* GNU Parallel was cited in: Structure and evolutionary history of a large family of NLR proteins in the zebrafish http://rsob.royalsocietypublishing.org/content/royopenbio/6/4/160009.full.pdf +* <> + +* <> + +* GNU Parallel was cited in: Improving computation efficiency by parallel programming http://www.irbis-nbuv.gov.ua/cgi-bin/irbis_nbuv/cgiirbis_64.exe?C21COM=2&I21DBN=UJRN&P21DBN=UJRN&IMAGE_FILE_DOWNLOAD=1&Image_file_name=PDF/ape_2013_3_44.pdf + +* GNU Parallel was cited in: A supernova feedback implementation for the astrophysical simulation software Arepo https://arxiv.org/abs/1604.06071 + +* GNU Parallel was cited in: Lorenz-Mie theory for 2D scattering and resonance calculations https://arxiv.org/pdf/1505.07691v2.pdf + +* GNU Parallel was cited in: Host-pathogen co-evolution and the emergence of broadly neutralizing antibodies in chronic infections https://arxiv.org/abs/1512.06296 + +* GNU Parallel was cited in: Pacific People, Metabolic Disease and Evolutionary Processes: a mitochondrial DNA study https://otago.ourarchive.ac.nz/handle/10523/6340 + +* GNU Parallel was cited in: The effect of domain modeling on efficiency of planning: Lessons from the Nomystery domain http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7407131 + +* GNU Parallel was cited in: Oops, my tests broke the build: An analysis of Travis CI buildswith GitHub https://peerj.com/preprints/1984/ + +* GNU Parallel was cited in: Drosophila Muller F Elements Maintain a Distinct Set of Genomic Properties Over 40 Million Years of Evolution http://www.g3journal.org/content/5/5/719.full.pdf+html + +* GNU Parallel was cited in: An Empirical Comparison of Neural Architectures for Reinforcement Learning in Partially Observable Environments http://biorxiv.org/content/biorxiv/early/2016/03/24/022707.full.pdf + +* GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/early/2016/04/11/048066.abstract + +* GNU Parallel was cited in: PleaseTM: Enabling Transaction Conflict Management in Requester-wins Hardware Transactional Memory http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7446072 + +* GNU Parallel was cited in: EASE-MM: Sequence-Based Prediction of Mutation-Induced Stability Changes with Feature-Based Multiple Models http://www.sciencedirect.com/science/article/pii/S0022283616000310 + +* GNU Parallel was cited in: A new orthology assessment method for phylogenomic data: Unrooted Phylogenetic Orthology http://mbe.oxfordjournals.org/content/early/2016/04/06/molbev.msw069.short https://github.com/ballesterus/UPhO + +* GNU Parallel was cited in: Distinctive Interest Point Selection for Efficient Near-duplicate Image Retrieval http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7459172 + +* GNU Parallel was cited in: The Evolution of C Programming Practices: A Study of the Unix Operating System 1973–2015 https://dl.acm.org/citation.cfm?id=2884799 (It has the cutest thumbnail graphs I have ever seen scattered all over the text) + +* GNU Parallel was cited in: StrAuto: Automation and Parallelization of STRUCTURE Analysis http://vchhatre.w3.uvm.edu/download/strauto/strauto_doc.pdf * GNU Parallel was cited in: Data processing pipeline for serial femtosecond crystallography at SACLA http://journals.iucr.org/j/issues/2016/03/00/zw5001/index.html @@ -274,16 +310,24 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * GNU Parallel was cited in: How attention influences perceptual decision making: Single-trial EEG correlates of drift-diffusion model parameters http://www.cidlab.com/prints/nunez2016attention.pdf +* GNU Parallel was cited in: Telomere And Proximal Sequence Analysis Using High-Throughput Sequencing Reads http://repository.upenn.edu/edissertations/1460/ + * GNU Parallel was cited in: AT-GIS: Highly Parallel Spatial Query Processing with Associative Transducers http://lsds.doc.ic.ac.uk/sites/default/files/ATGIS-SIGMOD16.pdf * GNU Parallel was cited in: PVAIR: Partial Variable Assignment InterpolatoR http://verify.inf.usi.ch/sites/default/files/main-2.pdf +* GNU Parallel was cited in: Comparative Cladistics: Fossils, Morphological Data Partitions and Lost Branches in the Fossil Tree of Life http://opus.bath.ac.uk/43955/ + * GNU Parallel was cited in: Shannon: An Information-Optimal de NovoRNA-Seq Assembler http://biorxiv.org/content/biorxiv/early/2016/02/09/039230.full.pdf +* GNU Parallel was cited in: Adaptive Measure-Theoretic Parameter Estimation for Coastal Ocean Modeling https://repositories.lib.utexas.edu/handle/2152/32435 + * GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 * GNU Parallel was cited in: Reference genotype and exome data from an Australian Aboriginal population for health-based research http://www.nature.com/articles/sdata201623 +* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 + * GNU Parallel was cited in: Do aye-ayes echolocate? http://biorxiv.org/content/biorxiv/early/2016/04/11/048165.full.pdf * GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/biorxiv/early/2016/04/11/048066.full.pdf diff --git a/src/niceload b/src/niceload index 212aeebf..aee1878e 100755 --- a/src/niceload +++ b/src/niceload @@ -321,7 +321,7 @@ sub get_options_from_array { $opt::load = -1; } if($opt::net) { - $opt::nethops ||= 2; + $opt::nethops ||= 3; } if($opt::nethops) { # niceload -l 0.01 --sensor 'netsensor_script' diff --git a/src/niceload.pod b/src/niceload.pod index 0887e7d9..87c975f8 100644 --- a/src/niceload.pod +++ b/src/niceload.pod @@ -104,7 +104,7 @@ B<--noswap> will set both B<--start-noswap> and B. =item B<--net> (alpha testing) -Shorthand for B<--nethops 2>. +Shorthand for B<--nethops 3>. =item B<--nethops> I (alpha testing) diff --git a/src/parallel b/src/parallel index 9d5dc7a8..4ced9efe 100755 --- a/src/parallel +++ b/src/parallel @@ -7482,7 +7482,7 @@ sub start { eval { if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) { # Each child gets its own process group to make it safe to killall - setpgrp(0,0); + setpgrp(0,0) || ::die_bug("setpgrp failed"); exec("exec $Global::shell -c ".::shell_quote_scalar_default($command)) || ::die_bug("open3-$stdin_fh $command"); } @@ -7520,16 +7520,23 @@ sub start { } sub open3_setpgrp { - # If the OS supports open3(x,x,x,"-") use that - # eval { if(not $pid=::open3($i,$o,$e,"-")) { exit } } - # if $!: external - # TODO build a selector that works with out side effects + # Select and run open3_setpgrp_internal/open3_setpgrp_external + # parallel -j0 --timeout 5 '( (sleep {}) & )& sleep 7' ::: {1..55} & + # ps -opid,ppid,pgrp,cmd + # parallel --timeout 6 -q perl -e 'sleep(1);if(fork()){sleep(30)}while(not fork and $t++<30000){ }' ::: 1 + # perl -e 'sleep(1);if(fork()){sleep(3)}while(not fork and time-$^T<8){ }' no warnings 'redefine'; - if(1) { + my ($outfh,$name) = ::tmpfile(SUFFIX => ".tst"); + # Test to see if open3(,,,"-") is supported + my $script = 'if(not $pid=::open3($i,$o,$e,"-")) { unlink shift }'; + qx{ perl -MIPC::Open3 -e '$script' $name }; + if(-e $name) { # Does not support open3(x,x,x,"-") + unlink($name); *open3_setpgrp = \&open3_setpgrp_external; } else { # Supports open3(x,x,x,"-") + # This is 0.5 ms faster to run *open3_setpgrp = \&open3_setpgrp_internal; } # The sub is now redefined. Call it diff --git a/src/parallel.pod b/src/parallel.pod index c22265a9..3983188a 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1272,10 +1272,10 @@ with 'y' or 'Y'. Implies B<-t>. =item B<--parens> I -Use to define start and end parenthesis for B<{= perl expression =}>. The +Define start and end parenthesis for B<{= perl expression =}>. The left and the right parenthesis can be multiple characters and are -assumed to be the same length. The default is B<{==}> giving -B<{=> as the start parenthesis and B<=}> as the end parenthesis. +assumed to be the same length. The default is B<{==}> giving B<{=> as +the start parenthesis and B<=}> as the end parenthesis. Another useful setting is B<,,,,> which would make both parenthesis B<,,>: @@ -1546,7 +1546,7 @@ it possible to define your own replacement strings. GNU B's --rpl '{/.} s:.*/::; s:\.[^/.]+$::;' --rpl '{.} s:\.[^/.]+$::' -The <--plus> replacement strings are implemented as: +The B<--plus> replacement strings are implemented as: --rpl '{+/} s:/[^/]*$::' --rpl '{+.} s:.*\.::' @@ -1567,12 +1567,12 @@ of GNU B's internal functions and data structures. Here are a few examples: - Remove 2 extensions (e.g. .tar.gz) - --rpl '{..} s:\.[^/.]+$::;s:\.[^/.]+$::;' - Keep only the extension - --rpl '{ext} s:.*\.::' Is the job sequence even or odd? - --rpl '{odd} $_=$job->seq()%2?"odd":"even"' + --rpl '{odd} $_ = $job->seq() % 2 ? "odd" : "even"' + Pad job sequence with leading zeros to get equal width + --rpl '{0#} $f = "%0".int(1+log(total_jobs())/log(10))."d"; $_=sprintf($f,$job->seq())' + Job sequence counting from 0 + --rpl '{#0} $_ = $job->seq() - 1' See also: B<{= perl expression =}> B<--parens> @@ -1981,13 +1981,13 @@ Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. -=item B<--timeout> I +=item B<--timeout> I -Time out for command. If the command runs for longer than I +Time out for command. If the command runs for longer than I seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms later, followed by SIGKILL 200 ms later. -If I is followed by a % then the timeout will dynamically be +If I is followed by a % then the timeout will dynamically be computed as a percentage of the median average runtime. Only values > 100% will make sense. diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index 1b0d9d07..9bf4313f 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -2561,7 +2561,7 @@ Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication - please cite as described in 'parallel --bibtex'. + please cite as described in 'parallel --citation'.

In scripts --minversion can be used to ensure the user has at least this version:

@@ -2572,9 +2572,9 @@ 
  20160322
   Your version is at least 20130722.
-

If using GNU parallel for research the BibTeX citation can be generated using --bibtex:

+

If you are using GNU parallel for research the BibTeX citation can be generated using --citation:

-
  parallel --bibtex
+
  parallel --citation

Output:

@@ -2661,10 +2661,10 @@

  • (Re-)walk through the tutorial if you have not done so in the past year (http://www.gnu.org/software/parallel/parallel_tutorial.html)

    • -

  • Give a demo at your local user group/team/colleagues

    +

  • Give a demo at your local user group/your team/your colleagues

    • -

  • Post the intro videos and the tutorial on Reddit, Diaspora*, forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, mailing lists

    +

  • Post the intro videos and the tutorial on Reddit, Diaspora*, forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, and mailing lists

  • Request or write a review for your favourite blog or magazine (especially if you do something cool with GNU parallel)

    @@ -2679,7 +2679,7 @@
      -

    • Please cite GNU parallel in you publications (use --bibtex)

      +

    • Please cite GNU parallel in you publications (use --citation)

    diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 803ac3cc..04753ea4 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -2603,7 +2603,7 @@ Output: Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication - please cite as described in 'parallel --bibtex'. + please cite as described in 'parallel --citation'. In scripts B<--minversion> can be used to ensure the user has at least this version: @@ -2615,10 +2615,10 @@ Output: 20160322 Your version is at least 20130722. -If using GNU B for research the BibTeX citation can be -generated using B<--bibtex>: +If you are using GNU B for research the BibTeX citation can be +generated using B<--citation>: - parallel --bibtex + parallel --citation Output: @@ -2714,13 +2714,13 @@ If you like GNU B: =item * -Give a demo at your local user group/team/colleagues +Give a demo at your local user group/your team/your colleagues =item * Post the intro videos and the tutorial on Reddit, Diaspora*, forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, -mailing lists +and mailing lists =item * @@ -2739,7 +2739,7 @@ If you use GNU B for research: =item * -Please cite GNU B in you publications (use B<--bibtex>) +Please cite GNU B in you publications (use B<--citation>) =back diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index b3443ede..434a0fa0 100755 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -16,7 +16,7 @@ S_POLAR=`parallel -k echo -S 1/{}.polarhome.com ::: $P` # 20150414 --timeout 80 -> 40 # 20151219 --retries 5 -> 2 TIMEOUT=10 -RETRIES=2 +RETRIES=4 echo '### Tests on polarhome machines' echo 'Setup on polarhome machines' diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 7fb952e6..0ac9881b 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -1269,7 +1269,7 @@ please cite as described in 'parallel --citation'. parallel --minversion VERSION && echo Your version is at least VERSION. VERSION Your version is at least VERSION. - parallel --bibtex + parallel --citation Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: