From 69188b0fa662ea45b1b57fe2e99a2ef6d347c2cf Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 22 Mar 2015 02:27:01 +0100 Subject: [PATCH] Builds on OBS. --- NEWS | 99 +++++++++++++++++++ README | 12 +-- configure | 20 ++-- configure.ac | 2 +- doc/release_new_version | 7 -- .../obs/home:tange/parallel/parallel.spec | 2 +- src/niceload | 2 +- src/parallel | 4 +- src/parallel.pod | 31 +++--- src/sql | 2 +- 10 files changed, 138 insertions(+), 43 deletions(-) diff --git a/NEWS b/NEWS index 425db9f0..6b2acbdf 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,102 @@ +20150322 + +New in this release: + +* --number-of-cores respects 'taskset' on GNU/Linux. + +* --joblog --pipe gives the data send and received in the log. + +* GNU Parallel was tested to support 100 GB sized records in --pipe. A + few bugs was fixed to support >2 GB records. It works, but is rather + slow. + +* GNU Parallel was cited in: RIG: Recalibration and Interrelation of + genomic sequence data with the GATK + http://www.g3journal.org/content/early/2015/02/13/g3.115.017012.full.pdf+html + +* GNU Parallel was cited in: MPI-blastn and NCBI-TaxCollector: + Improving metagenomic analysis with high performance classification + and wide taxonomic attachment + http://www.worldscientific.com/doi/abs/10.1142/S0219720014500139?af=R& + +* GNU Parallel was cited in: Recent evolution in Rattus norvegicus is + shaped by declining effective population size + http://biorxiv.org/content/biorxiv/early/2015/03/01/015818.full.pdf + +* GNU Parallel was cited in: Evidence for DCO+ as a probe of + ionization in the warm disk surface + http://arxiv.org/pdf/1503.02659.pdf + +* GNU Parallel was cited in: De novo assembly and annotation of the + Asian tiger mosquito (Aedes albopictus) repeatome with dnaPipeTE + from raw genomic reads and comparative analysis with the yellow + fever mosquito (Aedes aegypti) + http://gbe.oxfordjournals.org/content/early/2015/03/11/gbe.evv050.full.pdf + +* GNU Parallel was cited in: A General Approach to Network + Configuration Analysis + http://research.microsoft.com/en-us/um/people/ratul/papers/nsdi2015-batfish.pdf + +* GNU Parallel was cited in: Scrimer: designing primers from + transcriptome data + http://onlinelibrary.wiley.com/doi/10.1111/1755-0998.12403/pdf + +* GNU Parallel was cited in: Efficient Retrieval of Key Material for + Inspecting Potentially Malicious Traffic in the Cloud + http://sacko.uk/pdf/2015.1.pdf + +* GNU Parallel will be presented at Strataconf: Poor Man's Parallel + Pipelines + http://strataconf.com/big-data-conference-uk-2015/public/schedule/detail/40031 + +* GNU Parallel was used in: + https://github.com/alexbyrnes/FCC-Political-Ads_The-Code + +* GNU Parallel was used in: https://github.com/martymac/fpart + +* GNU Parallel was used in: + https://github.com/hoytak/diabetic-retinopathy-code + +* GNU Parallel was used in: + https://github.com/mehmattski/HybSeqPipeline + +* GNU Parallel was used in: + http://search.cpan.org/~ajpage/Bio-Roary-2.0.0/lib/Bio/Roary/JobRunner/Parallel.pm + +* Using GNU Parallel on a Raspberry Pi cluster: + http://www.dcglug.org.uk/cluster-progress/ + +* An introduction to vector tiling and map reduce in postgis: + http://dimensionaledge.com/intro-vector-tiling-map-reduce-postgis/ + +* Running scripts in parallel with GNU Parallel: + http://code.jasonbhill.com/2015/03/ + +* Mahout: Parallelising the creation of DecisionTrees: + http://www.markhneedham.com/blog/2012/12/27/mahout-parallelising-the-creation-of-decisiontrees/ + +* High-Performance Scientific Computing: Running serial jobs in + parallel + https://support.scinet.utoronto.ca/education/staticpublic/course178/serial.pdf + +* Parallel cardinality on your laptop + https://highonscience.wordpress.com/2015/03/15/parallel-cardinality-on-your-laptop/ + +* Experiments about a better locate using grep + http://a3nm.net/blog/better_locate.html + +* Homework with GNU Parallel: + https://support.scinet.utoronto.ca/education/staticpublic/course178content349.html + +* Parallel - Jobs in Skripten parallelisieren: + https://slzm.de/blog/linux-tool-des-tages-parallel-jobs-in-skripten-parallelisieren/#more-612 + +* GNU Parallel, czyli 100% użycia procesora + http://matmatyk.blogspot.dk/2015/03/gnu-parallel-czyli-100-uzycia-procesora.html + +* Bug fixes and man page updates. + + 20150222 * --tmux has gotten a major overhaul. diff --git a/README b/README index 262719e3..0cb356e3 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20150222.tar.bz2 - bzip2 -dc parallel-20150222.tar.bz2 | tar xvf - - cd parallel-20150222 + wget http://ftpmirror.gnu.org/parallel/parallel-20150322.tar.bz2 + bzip2 -dc parallel-20150322.tar.bz2 | tar xvf - + cd parallel-20150322 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20150222.tar.bz2 - bzip2 -dc parallel-20150222.tar.bz2 | tar xvf - - cd parallel-20150222 + wget http://ftpmirror.gnu.org/parallel/parallel-20150322.tar.bz2 + bzip2 -dc parallel-20150322.tar.bz2 | tar xvf - + cd parallel-20150322 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index d561304d..5945e8a3 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20150222. +# Generated by GNU Autoconf 2.69 for parallel 20150322. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20150222' -PACKAGE_STRING='parallel 20150222' +PACKAGE_VERSION='20150322' +PACKAGE_STRING='parallel 20150322' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20150222 to adapt to many kinds of systems. +\`configure' configures parallel 20150322 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20150222:";; + short | recursive ) echo "Configuration of parallel 20150322:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20150222 +parallel configure 20150322 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20150222, which was +It was created by parallel $as_me 20150322, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20150222' + VERSION='20150322' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20150222, which was +This file was extended by parallel $as_me 20150322, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20150222 +parallel config.status 20150322 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 70cb7bca..73fcbcbe 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20150222], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20150322], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index a4d4342d..5942b54f 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -244,13 +244,6 @@ New in this release: * GNU Parallel was cited in: Efficient Retrieval of Key Material for Inspecting Potentially Malicious Traffic in the Cloud http://sacko.uk/pdf/2015.1.pdf - -* <> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf - -* <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 - -taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) - * GNU Parallel will be presented at Strataconf: Poor Man's Parallel Pipelines http://strataconf.com/big-data-conference-uk-2015/public/schedule/detail/40031 * GNU Parallel was used in: https://github.com/alexbyrnes/FCC-Political-Ads_The-Code diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 8a5ec9af..8c1b855e 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20150222 +Version: 20150322 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index ed5c95ba..0138e5b0 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20150222; +$Global::version = 20150322; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 7a0b9bad..369a59e9 100755 --- a/src/parallel +++ b/src/parallel @@ -1052,7 +1052,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150311; + $Global::version = 20150322; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -6021,7 +6021,7 @@ sub wrapped { ::shell_quote_scalar($command); } else { # Remote systems use $SHELL - $command = '\nice'. " -n". $opt::nice. + $command = '\nice'. " -n". $opt::nice. ' $SHELL -c '. ::shell_quote_scalar($command); } diff --git a/src/parallel.pod b/src/parallel.pod index 5e666b8c..1715e054 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -488,7 +488,7 @@ I is a Perl Regular Expression: http://perldoc.perl.org/perlre.html -=item B<--compress> +=item B<--compress> (alpha testing) Compress temporary files. If the output is big and very compressible this will take up less disk space in $TMPDIR and possibly be faster @@ -499,9 +499,9 @@ B, B, B, B, B in that order, and use the first available. -=item B<--compress-program> I +=item B<--compress-program> I (alpha testing) -=item B<--decompress-program> I +=item B<--decompress-program> I (alpha testing) Use I for (de)compressing temporary files. It is assumed that I will decompress stdin (standard input) to stdout (standard @@ -514,9 +514,9 @@ If receiving SIGNING, GNU B will send SIGINT to tasks running on remote computers thus killing them. -=item B<--delimiter> I +=item B<--delimiter> I (alpha testing) -=item B<-d> I +=item B<-d> I (alpha testing) Input items are terminated by I. Quotes and backslash are not special; every character in the input is taken literally. Disables @@ -780,7 +780,7 @@ specified, and for B<-I>{} otherwise. This option is deprecated; use B<-I> instead. -=item B<--joblog> I +=item B<--joblog> I (alpha testing) Logfile for executed jobs. Save a list of the executed jobs to I in the following TAB separated format: sequence number, @@ -788,6 +788,9 @@ sshlogin, start time as seconds since epoch, run time in seconds, bytes in files transferred, bytes in files returned, exit status, signal, and command run. +For B<--pipe> bytes transferred and bytes returned are number of input +and output of bytes. + To convert the times into ISO-8601 strict do: perl -a -F"\t" -ne \ @@ -1034,9 +1037,9 @@ of each job is saved in a file and the filename is then printed. See also: B<--results> -=item B<--pipe> +=item B<--pipe> (alpha testing) -=item B<--spreadstdin> +=item B<--spreadstdin> (alpha testing) Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -1157,13 +1160,13 @@ exit (used by GNU B itself to determine the line length on remote computers). -=item B<--number-of-cpus> +=item B<--number-of-cpus> (alpha testing) Print the number of physical CPUs and exit (used by GNU B itself to determine the number of physical CPUs on remote computers). -=item B<--number-of-cores> +=item B<--number-of-cores> (alpha testing) Print the number of CPU cores and exit (used by GNU B itself to determine the number of CPU cores on remote computers). @@ -1175,7 +1178,7 @@ Overrides an earlier B<--keep-order> (e.g. if set in B<~/.parallel/config>). -=item B<--nice> I +=item B<--nice> I (alpha testing) Run the command at this niceness. For simple commands you can just add B in front of the command. But if the command consists of more @@ -1432,7 +1435,7 @@ B<--keep-order> will not work with B<--round-robin> as it is impossible to track which input block corresponds to which output. -=item B<--rpl> 'I I' +=item B<--rpl> 'I I' (alpha testing) Use I as a replacement string for I. This makes it possible to define your own replacement strings. GNU B's @@ -1760,7 +1763,7 @@ the lines will be prepended with the sshlogin instead. B<--tag> is ignored when using B<-u>. -=item B<--tagstring> I (beta testing) +=item B<--tagstring> I (alpha testing) Tag lines with a string. Each output line will be prepended with I and TAB (\t). I can contain replacement strings such as @@ -1777,7 +1780,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--tmux> (beta testing) +=item B<--tmux> (alpha testing) Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. diff --git a/src/sql b/src/sql index 2a7b9ad6..0a271936 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20150222; + $Global::version = 20150322; $Global::progname = 'sql'; # This must be done first as this may exec myself