From 618220910ade68f91e6b4deddf78c43a15befc09 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 22 May 2016 22:43:53 +0200 Subject: [PATCH] parallel: limit buffering to 1..2 (stdout..stderr). testsuite: Compatibility fix with perl-5.14. --- NEWS | 171 ++++++++++++++----- doc/boxplot-runtime | 2 +- doc/release_new_version | 100 +---------- src/niceload | 2 +- src/niceload.pod | 12 +- src/parallel | 25 +-- src/parallel.pod | 10 +- src/sql | 2 +- testsuite/tests-to-run/parallel-local-mem.sh | 4 + 9 files changed, 167 insertions(+), 161 deletions(-) diff --git a/NEWS b/NEWS index e2a32803..3a83e1d0 100644 --- a/NEWS +++ b/NEWS @@ -1,90 +1,173 @@ 20160522 -* niceload --net pauses the program if the internet connection is overloaded. +* niceload --net pauses the program if the internet connection is + overloaded. -* Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046 +* Vote for GNU Parallel's community ad on + https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046 -* Updated speed comparison between versions https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf +* Updated speed comparison between versions + https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf -* GNU Parallel was cited in: Improving computation efficiency by parallel programming http://www.irbis-nbuv.gov.ua/cgi-bin/irbis_nbuv/cgiirbis_64.exe?C21COM=2&I21DBN=UJRN&P21DBN=UJRN&IMAGE_FILE_DOWNLOAD=1&Image_file_name=PDF/ape_2013_3_44.pdf +* GNU Parallel was cited in: Improving computation efficiency by + parallel programming + http://www.irbis-nbuv.gov.ua/cgi-bin/irbis_nbuv/cgiirbis_64.exe?C21COM=2&I21DBN=UJRN&P21DBN=UJRN&IMAGE_FILE_DOWNLOAD=1&Image_file_name=PDF/ape_2013_3_44.pdf -* GNU Parallel was cited in: A supernova feedback implementation for the astrophysical simulation software Arepo https://arxiv.org/abs/1604.06071 +* GNU Parallel was cited in: A supernova feedback implementation for + the astrophysical simulation software Arepo + https://arxiv.org/abs/1604.06071 -* GNU Parallel was cited in: Lorenz-Mie theory for 2D scattering and resonance calculations https://arxiv.org/pdf/1505.07691v2.pdf +* GNU Parallel was cited in: Lorenz-Mie theory for 2D scattering and + resonance calculations https://arxiv.org/pdf/1505.07691v2.pdf -* GNU Parallel was cited in: Host-pathogen co-evolution and the emergence of broadly neutralizing antibodies in chronic infections https://arxiv.org/abs/1512.06296 +* GNU Parallel was cited in: Host-pathogen co-evolution and the + emergence of broadly neutralizing antibodies in chronic infections + https://arxiv.org/abs/1512.06296 -* GNU Parallel was cited in: Pacific People, Metabolic Disease and Evolutionary Processes: a mitochondrial DNA study https://otago.ourarchive.ac.nz/handle/10523/6340 +* GNU Parallel was cited in: Pacific People, Metabolic Disease and + Evolutionary Processes: a mitochondrial DNA study + https://otago.ourarchive.ac.nz/handle/10523/6340 -* GNU Parallel was cited in: The effect of domain modeling on efficiency of planning: Lessons from the Nomystery domain http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7407131 +* GNU Parallel was cited in: The effect of domain modeling on + efficiency of planning: Lessons from the Nomystery domain + http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7407131 -* GNU Parallel was cited in: Oops, my tests broke the build: An analysis of Travis CI buildswith GitHub https://peerj.com/preprints/1984/ +* GNU Parallel was cited in: Oops, my tests broke the build: An + analysis of Travis CI buildswith GitHub + https://peerj.com/preprints/1984/ -* GNU Parallel was cited in: Drosophila Muller F Elements Maintain a Distinct Set of Genomic Properties Over 40 Million Years of Evolution http://www.g3journal.org/content/5/5/719.full.pdf+html +* GNU Parallel was cited in: Drosophila Muller F Elements Maintain a + Distinct Set of Genomic Properties Over 40 Million Years of + Evolution http://www.g3journal.org/content/5/5/719.full.pdf+html -* GNU Parallel was cited in: An Empirical Comparison of Neural Architectures for Reinforcement Learning in Partially Observable Environments http://biorxiv.org/content/biorxiv/early/2016/03/24/022707.full.pdf +* GNU Parallel was cited in: An Empirical Comparison of Neural + Architectures for Reinforcement Learning in Partially Observable + Environments + http://biorxiv.org/content/biorxiv/early/2016/03/24/022707.full.pdf -* GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/early/2016/04/11/048066.abstract +* GNU Parallel was cited in: Functional enrichments of disease + variants across thousands of independent loci in eight diseases + http://biorxiv.org/content/early/2016/04/11/048066.abstract -* GNU Parallel was cited in: PleaseTM: Enabling Transaction Conflict Management in Requester-wins Hardware Transactional Memory http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7446072 +* GNU Parallel was cited in: PleaseTM: Enabling Transaction Conflict + Management in Requester-wins Hardware Transactional Memory + http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7446072 -* GNU Parallel was cited in: EASE-MM: Sequence-Based Prediction of Mutation-Induced Stability Changes with Feature-Based Multiple Models http://www.sciencedirect.com/science/article/pii/S0022283616000310 +* GNU Parallel was cited in: EASE-MM: Sequence-Based Prediction of + Mutation-Induced Stability Changes with Feature-Based Multiple + Models + http://www.sciencedirect.com/science/article/pii/S0022283616000310 -* GNU Parallel was cited in: A new orthology assessment method for phylogenomic data: Unrooted Phylogenetic Orthology http://mbe.oxfordjournals.org/content/early/2016/04/06/molbev.msw069.short https://github.com/ballesterus/UPhO +* GNU Parallel was cited in: A new orthology assessment method for + phylogenomic data: Unrooted Phylogenetic Orthology + http://mbe.oxfordjournals.org/content/early/2016/04/06/molbev.msw069.short + https://github.com/ballesterus/UPhO -* GNU Parallel was cited in: Distinctive Interest Point Selection for Efficient Near-duplicate Image Retrieval http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7459172 +* GNU Parallel was cited in: Distinctive Interest Point Selection for + Efficient Near-duplicate Image Retrieval + http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7459172 -* GNU Parallel was cited in: The Evolution of C Programming Practices: A Study of the Unix Operating System 1973–2015 https://dl.acm.org/citation.cfm?id=2884799 (It has the cutest thumbnail graphs I have ever seen scattered all over the text) +* GNU Parallel was cited in: The Evolution of C Programming Practices: + A Study of the Unix Operating System 1973–2015 + https://dl.acm.org/citation.cfm?id=2884799 (It has the cutest + thumbnail graphs I have ever seen scattered all over the text) -* GNU Parallel was cited in: StrAuto: Automation and Parallelization of STRUCTURE Analysis http://vchhatre.w3.uvm.edu/download/strauto/strauto_doc.pdf +* GNU Parallel was cited in: StrAuto: Automation and Parallelization + of STRUCTURE Analysis + http://vchhatre.w3.uvm.edu/download/strauto/strauto_doc.pdf -* GNU Parallel was cited in: Data processing pipeline for serial femtosecond crystallography at SACLA http://journals.iucr.org/j/issues/2016/03/00/zw5001/index.html +* GNU Parallel was cited in: Data processing pipeline for serial + femtosecond crystallography at SACLA + http://journals.iucr.org/j/issues/2016/03/00/zw5001/index.html -* GNU Parallel was cited in: Reconstruction of Fine-Scale Auroral Dynamics http://arxiv.org/pdf/1512.01460.pdf +* GNU Parallel was cited in: Reconstruction of Fine-Scale Auroral + Dynamics http://arxiv.org/pdf/1512.01460.pdf -* GNU Parallel was cited in: The Outer Solar System Origins Survey: I. Design and First-Quarter Discoveries http://arxiv.org/pdf/1511.02895.pdf +* GNU Parallel was cited in: The Outer Solar System Origins Survey: + I. Design and First-Quarter Discoveries + http://arxiv.org/pdf/1511.02895.pdf -* GNU Parallel was cited in: Multiscale Estimation of Binding Kinetics Using Brownian Dynamics, Molecular Dynamics and Milestoning http://journals.plos.org/ploscompbiol/article/asset?id=10.1371%2Fjournal.pcbi.1004381.PDF +* GNU Parallel was cited in: Multiscale Estimation of Binding Kinetics + Using Brownian Dynamics, Molecular Dynamics and Milestoning + http://journals.plos.org/ploscompbiol/article/asset?id=10.1371%2Fjournal.pcbi.1004381.PDF -* GNU Parallel was cited in: Genomic legacy of the African cheetah, Acinonyx jubatus https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4676127/pdf/13059_2015_Article_837.pdf +* GNU Parallel was cited in: Genomic legacy of the African cheetah, + Acinonyx jubatus + https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4676127/pdf/13059_2015_Article_837.pdf -* GNU Parallel was cited in: Don’t Forget to Lock the Back Door! A Characterization of IPv6 Network Security Policy http://benign-research-probe2.eecs.umich.edu/ndss16_ipv6_final.pdf +* GNU Parallel was cited in: Don’t Forget to Lock the Back Door! A + Characterization of IPv6 Network Security Policy + http://benign-research-probe2.eecs.umich.edu/ndss16_ipv6_final.pdf -* GNU Parallel was cited in: Comprehensive Annotation of the Parastagonospora nodorum Reference Genome Using Next-Generation Genomics, Transcriptomics and Proteogenomics http://journals.plos.org/plosone/article/asset?id=10.1371%2Fjournal.pone.0147221.PDF +* GNU Parallel was cited in: Comprehensive Annotation of the + Parastagonospora nodorum Reference Genome Using Next-Generation + Genomics, Transcriptomics and Proteogenomics + http://journals.plos.org/plosone/article/asset?id=10.1371%2Fjournal.pone.0147221.PDF -* GNU Parallel was cited in: Stride Search: a general algorithm for storm detection in high-resolution climate data http://www.geosci-model-dev-discuss.net/8/7727/2015/gmdd-8-7727-2015.pdf +* GNU Parallel was cited in: Stride Search: a general algorithm for + storm detection in high-resolution climate data + http://www.geosci-model-dev-discuss.net/8/7727/2015/gmdd-8-7727-2015.pdf -* GNU Parallel was cited in: How attention influences perceptual decision making: Single-trial EEG correlates of drift-diffusion model parameters http://www.cidlab.com/prints/nunez2016attention.pdf +* GNU Parallel was cited in: How attention influences perceptual + decision making: Single-trial EEG correlates of drift-diffusion + model parameters http://www.cidlab.com/prints/nunez2016attention.pdf -* GNU Parallel was cited in: Telomere And Proximal Sequence Analysis Using High-Throughput Sequencing Reads http://repository.upenn.edu/edissertations/1460/ +* GNU Parallel was cited in: Telomere And Proximal Sequence Analysis + Using High-Throughput Sequencing Reads + http://repository.upenn.edu/edissertations/1460/ -* GNU Parallel was cited in: AT-GIS: Highly Parallel Spatial Query Processing with Associative Transducers http://lsds.doc.ic.ac.uk/sites/default/files/ATGIS-SIGMOD16.pdf +* GNU Parallel was cited in: AT-GIS: Highly Parallel Spatial Query + Processing with Associative Transducers + http://lsds.doc.ic.ac.uk/sites/default/files/ATGIS-SIGMOD16.pdf -* GNU Parallel was cited in: PVAIR: Partial Variable Assignment InterpolatoR http://verify.inf.usi.ch/sites/default/files/main-2.pdf +* GNU Parallel was cited in: PVAIR: Partial Variable Assignment + InterpolatoR http://verify.inf.usi.ch/sites/default/files/main-2.pdf -* GNU Parallel was cited in: Comparative Cladistics: Fossils, Morphological Data Partitions and Lost Branches in the Fossil Tree of Life http://opus.bath.ac.uk/43955/ +* GNU Parallel was cited in: Comparative Cladistics: Fossils, + Morphological Data Partitions and Lost Branches in the Fossil Tree + of Life http://opus.bath.ac.uk/43955/ -* GNU Parallel was cited in: Shannon: An Information-Optimal de NovoRNA-Seq Assembler http://biorxiv.org/content/biorxiv/early/2016/02/09/039230.full.pdf +* GNU Parallel was cited in: Shannon: An Information-Optimal de + NovoRNA-Seq Assembler + http://biorxiv.org/content/biorxiv/early/2016/02/09/039230.full.pdf -* GNU Parallel was cited in: Adaptive Measure-Theoretic Parameter Estimation for Coastal Ocean Modeling https://repositories.lib.utexas.edu/handle/2152/32435 +* GNU Parallel was cited in: Adaptive Measure-Theoretic Parameter + Estimation for Coastal Ocean Modeling + https://repositories.lib.utexas.edu/handle/2152/32435 -* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 +* GNU Parallel was cited in: Computational Design of DNA-Binding + Proteins + http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 -* GNU Parallel was cited in: Reference genotype and exome data from an Australian Aboriginal population for health-based research http://www.nature.com/articles/sdata201623 +* GNU Parallel was cited in: Reference genotype and exome data from an + Australian Aboriginal population for health-based research + http://www.nature.com/articles/sdata201623 -* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 +* GNU Parallel was cited in: Computational Design of DNA-Binding + Proteins + http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 -* GNU Parallel was cited in: Do aye-ayes echolocate? http://biorxiv.org/content/biorxiv/early/2016/04/11/048165.full.pdf +* GNU Parallel was cited in: Do aye-ayes echolocate? + http://biorxiv.org/content/biorxiv/early/2016/04/11/048165.full.pdf -* GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/biorxiv/early/2016/04/11/048066.full.pdf +* GNU Parallel was cited in: Functional enrichments of disease + variants across thousands of independent loci in eight diseases + http://biorxiv.org/content/biorxiv/early/2016/04/11/048066.full.pdf -* GNU Parallel was cited in: From genomes to phenotypes: Traitar, the microbial trait analyzer http://biorxiv.org/content/biorxiv/early/2016/03/12/043315.full.pdf +* GNU Parallel was cited in: From genomes to phenotypes: Traitar, the + microbial trait analyzer + http://biorxiv.org/content/biorxiv/early/2016/03/12/043315.full.pdf -* GNU Parallel was mentioned in: Fast Playback Framework for Analysis of Ground-Based Doppler Radar Observations Using MapReduce Technology http://journals.ametsoc.org/doi/pdf/10.1175/JTECH-D-15-0118.1 +* GNU Parallel was mentioned in: Fast Playback Framework for Analysis + of Ground-Based Doppler Radar Observations Using MapReduce + Technology + http://journals.ametsoc.org/doi/pdf/10.1175/JTECH-D-15-0118.1 -* NCBI blast tutorial https://github.com/enormandeau/ncbi_blast_tutorial +* NCBI blast tutorial + https://github.com/enormandeau/ncbi_blast_tutorial -* Distributed Preservation Made Simple https://blog.archive.org/2016/02/26/distributed-preservation-made-simple/ +* Distributed Preservation Made Simple + https://blog.archive.org/2016/02/26/distributed-preservation-made-simple/ * Parallel Jobs in Luigi http://rjbaxley.com/posts/2016/03/13/parallel_jobs_in_luigi.html @@ -232,7 +315,7 @@ * --workdir now accepts replacementstrings. * GNU Parallel was cited in: Possum - A Framework for - Three-Dimensional Reconstruction of Brain Images rfom Serial + Three-Dimensional Reconstruction of Brain Images from Serial Sections http://link.springer.com/article/10.1007/s12021-015-9286-1 * GNU Parallel was cited in: A Genetic Algorithm for the selection of diff --git a/doc/boxplot-runtime b/doc/boxplot-runtime index 31c209ad..ce7972f1 100644 --- a/doc/boxplot-runtime +++ b/doc/boxplot-runtime @@ -49,7 +49,7 @@ measure() { pdf("/tmp/boxplot.pdf"); par(cex.axis=0.5); boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F, - ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER"); + ylab="milliseconds/job",main="GNU Parallel performance of different versions\n$OUTER trials each running $INNER jobs"); _ cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${MHZ}MHz-$OUTER-${INNER}v$VERSION.pdf evince /tmp/boxplot.pdf diff --git a/doc/release_new_version b/doc/release_new_version index 3ab4ece0..be9e91ce 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -219,26 +219,24 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20160522 ('ttipleaks') released <<[stable]>> +Subject: GNU Parallel 20160622 ('brexit') released <<[stable]>> -GNU Parallel 20160522 ('ttipleaks') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20160622 ('brexit') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> Haiku of the month: -Programs using net -only spare capacity -niceload dash dash net + <<>> -- Ole Tange New in this release: -* niceload --net pauses the program if the internet connection is overloaded. +* <> -* Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046 +* <> -* Updated speed comparison between versions https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf +* <> * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>> @@ -260,88 +258,6 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> -* GNU Parallel was cited in: Improving computation efficiency by parallel programming http://www.irbis-nbuv.gov.ua/cgi-bin/irbis_nbuv/cgiirbis_64.exe?C21COM=2&I21DBN=UJRN&P21DBN=UJRN&IMAGE_FILE_DOWNLOAD=1&Image_file_name=PDF/ape_2013_3_44.pdf - -* GNU Parallel was cited in: A supernova feedback implementation for the astrophysical simulation software Arepo https://arxiv.org/abs/1604.06071 - -* GNU Parallel was cited in: Lorenz-Mie theory for 2D scattering and resonance calculations https://arxiv.org/pdf/1505.07691v2.pdf - -* GNU Parallel was cited in: Host-pathogen co-evolution and the emergence of broadly neutralizing antibodies in chronic infections https://arxiv.org/abs/1512.06296 - -* GNU Parallel was cited in: Pacific People, Metabolic Disease and Evolutionary Processes: a mitochondrial DNA study https://otago.ourarchive.ac.nz/handle/10523/6340 - -* GNU Parallel was cited in: The effect of domain modeling on efficiency of planning: Lessons from the Nomystery domain http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7407131 - -* GNU Parallel was cited in: Oops, my tests broke the build: An analysis of Travis CI buildswith GitHub https://peerj.com/preprints/1984/ - -* GNU Parallel was cited in: Drosophila Muller F Elements Maintain a Distinct Set of Genomic Properties Over 40 Million Years of Evolution http://www.g3journal.org/content/5/5/719.full.pdf+html - -* GNU Parallel was cited in: An Empirical Comparison of Neural Architectures for Reinforcement Learning in Partially Observable Environments http://biorxiv.org/content/biorxiv/early/2016/03/24/022707.full.pdf - -* GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/early/2016/04/11/048066.abstract - -* GNU Parallel was cited in: PleaseTM: Enabling Transaction Conflict Management in Requester-wins Hardware Transactional Memory http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7446072 - -* GNU Parallel was cited in: EASE-MM: Sequence-Based Prediction of Mutation-Induced Stability Changes with Feature-Based Multiple Models http://www.sciencedirect.com/science/article/pii/S0022283616000310 - -* GNU Parallel was cited in: A new orthology assessment method for phylogenomic data: Unrooted Phylogenetic Orthology http://mbe.oxfordjournals.org/content/early/2016/04/06/molbev.msw069.short https://github.com/ballesterus/UPhO - -* GNU Parallel was cited in: Distinctive Interest Point Selection for Efficient Near-duplicate Image Retrieval http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7459172 - -* GNU Parallel was cited in: The Evolution of C Programming Practices: A Study of the Unix Operating System 1973–2015 https://dl.acm.org/citation.cfm?id=2884799 (It has the cutest thumbnail graphs I have ever seen scattered all over the text) - -* GNU Parallel was cited in: StrAuto: Automation and Parallelization of STRUCTURE Analysis http://vchhatre.w3.uvm.edu/download/strauto/strauto_doc.pdf - -* GNU Parallel was cited in: Data processing pipeline for serial femtosecond crystallography at SACLA http://journals.iucr.org/j/issues/2016/03/00/zw5001/index.html - -* GNU Parallel was cited in: Reconstruction of Fine-Scale Auroral Dynamics http://arxiv.org/pdf/1512.01460.pdf - -* GNU Parallel was cited in: The Outer Solar System Origins Survey: I. Design and First-Quarter Discoveries http://arxiv.org/pdf/1511.02895.pdf - -* GNU Parallel was cited in: Multiscale Estimation of Binding Kinetics Using Brownian Dynamics, Molecular Dynamics and Milestoning http://journals.plos.org/ploscompbiol/article/asset?id=10.1371%2Fjournal.pcbi.1004381.PDF - -* GNU Parallel was cited in: Genomic legacy of the African cheetah, Acinonyx jubatus https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4676127/pdf/13059_2015_Article_837.pdf - -* GNU Parallel was cited in: Don’t Forget to Lock the Back Door! A Characterization of IPv6 Network Security Policy http://benign-research-probe2.eecs.umich.edu/ndss16_ipv6_final.pdf - -* GNU Parallel was cited in: Comprehensive Annotation of the Parastagonospora nodorum Reference Genome Using Next-Generation Genomics, Transcriptomics and Proteogenomics http://journals.plos.org/plosone/article/asset?id=10.1371%2Fjournal.pone.0147221.PDF - -* GNU Parallel was cited in: Stride Search: a general algorithm for storm detection in high-resolution climate data http://www.geosci-model-dev-discuss.net/8/7727/2015/gmdd-8-7727-2015.pdf - -* GNU Parallel was cited in: How attention influences perceptual decision making: Single-trial EEG correlates of drift-diffusion model parameters http://www.cidlab.com/prints/nunez2016attention.pdf - -* GNU Parallel was cited in: Telomere And Proximal Sequence Analysis Using High-Throughput Sequencing Reads http://repository.upenn.edu/edissertations/1460/ - -* GNU Parallel was cited in: AT-GIS: Highly Parallel Spatial Query Processing with Associative Transducers http://lsds.doc.ic.ac.uk/sites/default/files/ATGIS-SIGMOD16.pdf - -* GNU Parallel was cited in: PVAIR: Partial Variable Assignment InterpolatoR http://verify.inf.usi.ch/sites/default/files/main-2.pdf - -* GNU Parallel was cited in: Comparative Cladistics: Fossils, Morphological Data Partitions and Lost Branches in the Fossil Tree of Life http://opus.bath.ac.uk/43955/ - -* GNU Parallel was cited in: Shannon: An Information-Optimal de NovoRNA-Seq Assembler http://biorxiv.org/content/biorxiv/early/2016/02/09/039230.full.pdf - -* GNU Parallel was cited in: Adaptive Measure-Theoretic Parameter Estimation for Coastal Ocean Modeling https://repositories.lib.utexas.edu/handle/2152/32435 - -* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 - -* GNU Parallel was cited in: Reference genotype and exome data from an Australian Aboriginal population for health-based research http://www.nature.com/articles/sdata201623 - -* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 - -* GNU Parallel was cited in: Do aye-ayes echolocate? http://biorxiv.org/content/biorxiv/early/2016/04/11/048165.full.pdf - -* GNU Parallel was cited in: Functional enrichments of disease variants across thousands of independent loci in eight diseases http://biorxiv.org/content/biorxiv/early/2016/04/11/048066.full.pdf - -* GNU Parallel was cited in: From genomes to phenotypes: Traitar, the microbial trait analyzer http://biorxiv.org/content/biorxiv/early/2016/03/12/043315.full.pdf - -* GNU Parallel was mentioned in: Fast Playback Framework for Analysis of Ground-Based Doppler Radar Observations Using MapReduce Technology http://journals.ametsoc.org/doi/pdf/10.1175/JTECH-D-15-0118.1 - -* NCBI blast tutorial https://github.com/enormandeau/ncbi_blast_tutorial - -* Distributed Preservation Made Simple https://blog.archive.org/2016/02/26/distributed-preservation-made-simple/ - -* Parallel Jobs in Luigi http://rjbaxley.com/posts/2016/03/13/parallel_jobs_in_luigi.html - * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. @@ -376,9 +292,9 @@ If you like GNU Parallel: * Request or build a package for your favourite distribution (if it is not already there) * Invite me for your next conference -If you use GNU Parallel for research: +If you use programs that use GNU Parallel for research: -* Please cite GNU Parallel in you publications (use --bibtex) +* Please cite GNU Parallel in you publications (use --citation) If GNU Parallel saves you money: diff --git a/src/niceload b/src/niceload index 233fa2bc..8e0efa4e 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20160522; +$Global::version = 20160523; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/niceload.pod b/src/niceload.pod index 87c975f8..4039d1c2 100644 --- a/src/niceload.pod +++ b/src/niceload.pod @@ -32,9 +32,9 @@ run 1 second, suspend (3.00-1.00) seconds, run 1 second, suspend =over 9 -=item B<-B> (alpha testing) +=item B<-B> (beta testing) -=item B<--battery> (alpha testing) +=item B<--battery> (beta testing) Suspend if the system is running on battery. Shorthand for: -l -1 --sensor 'cat /sys/class/power_supply/BAT0/status /proc/acpi/battery/BAT0/state 2>/dev/null |grep -i -q discharging; echo $?' @@ -102,12 +102,12 @@ B<--noswap> is over limit if the system is swapping both in and out. B<--noswap> will set both B<--start-noswap> and B. -=item B<--net> (alpha testing) +=item B<--net> (beta testing) Shorthand for B<--nethops 3>. -=item B<--nethops> I (alpha testing) +=item B<--nethops> I (beta testing) Network nice. Pause if the internet connection is overloaded. @@ -140,9 +140,9 @@ Process ID of process to suspend. You can specify multiple process IDs with multiple B<-p> I. -=item B<--prg> I (alpha testing) +=item B<--prg> I (beta testing) -=item B<--program> I (alpha testing) +=item B<--program> I (beta testing) Name of running program to suspend. You can specify multiple programs with multiple B<--prg> I. If no processes with the name diff --git a/src/parallel b/src/parallel index 37d31548..b4ebc900 100755 --- a/src/parallel +++ b/src/parallel @@ -1183,7 +1183,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20160522; + $Global::version = 20160523; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1952,16 +1952,19 @@ sub save_stdin_stdout_stderr { # $Global::original_stderr # $Global::original_stdin # Returns: N/A - - # Find file descriptors that are already opened (by the shell) - for my $fdno (1..61) { - # /dev/fd/62 and above are used by bash for <(cmd) - my $fh; - # 2-argument-open is used to be compatible with old perl 5.8.0 - # bug #43570: Perl 5.8.0 creates 61 files - if(open($fh,">&=$fdno")) { - $Global::fd{$fdno}=$fh; - } + + # TODO Disabled until we have an open3 that will take n filehandles + # for my $fdno (1..61) { + # # /dev/fd/62 and above are used by bash for <(cmd) + # # Find file descriptors that are already opened (by the shell) + # Only focus on stdout+stderr for now + for my $fdno (1..2) { + my $fh; + # 2-argument-open is used to be compatible with old perl 5.8.0 + # bug #43570: Perl 5.8.0 creates 61 files + if(open($fh,">&=$fdno")) { + $Global::fd{$fdno}=$fh; + } } open $Global::original_stderr, ">&", "STDERR" or ::die_bug("Can't dup STDERR: $!"); diff --git a/src/parallel.pod b/src/parallel.pod index 3983188a..d51aa31e 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -285,7 +285,7 @@ positional replacement strings see B<{>IB<}>. See also: B<{=perl expression=}> B<{>IB<}>. -=item B<:::> I (beta testing) +=item B<:::> I Use arguments from the command line as input source instead of stdin (standard input). Unlike other options for GNU B B<:::> is @@ -326,7 +326,7 @@ B<:::> and B<::::> can be mixed. So these are equivalent: ::: 1 2 3 -=item B<:::+> I (beta testing) +=item B<:::+> I Like B<:::> but linked like B<--xapply> to the previous input source. @@ -338,7 +338,7 @@ Example: parallel echo ::: a b c :::+ 1 2 3 ::: X Y :::+ 11 22 -=item B<::::> I (beta testing) +=item B<::::> I Another way to write B<-a> I B<-a> I ... @@ -347,7 +347,7 @@ B<:::> and B<::::> can be mixed. See B<-a>, B<:::> and B<--xapply>. -=item B<::::+> I (beta testing) +=item B<::::+> I Like B<::::+> but linked like B<--xapply> to the previous input source. @@ -611,7 +611,7 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I (beta testing) +=item B<--env> I Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for diff --git a/src/sql b/src/sql index 8842da84..880f758c 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20160522; + $Global::version = 20160523; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-mem.sh b/testsuite/tests-to-run/parallel-local-mem.sh index 3a2c7fc7..0db58b78 100755 --- a/testsuite/tests-to-run/parallel-local-mem.sh +++ b/testsuite/tests-to-run/parallel-local-mem.sh @@ -6,6 +6,7 @@ mkdir -p $TMPDIR cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj1 -k --joblog /tmp/jl-`basename $0` -L1 echo '### bug #44358: 2 GB records cause problems for -N' echo '5 GB version: Eats 12.5 GB' + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | @@ -13,6 +14,7 @@ echo '5 GB version: Eats 12.5 GB' `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c echo '2 GB version: eats 10 GB' + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | @@ -34,9 +36,11 @@ echo 'Eats 4 GB' perl -e '$buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print ((length $buf)."\n")' echo 'Eats 4.7 GB' + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum echo 'Eats 4.7 GB' + PATH=input-files/perl-v5.14.2:$PATH; (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c