diff --git a/NEWS b/NEWS index 02ac3ff9..f892031f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,67 @@ +20130822 + +* A tutorial showing much of GNU Parallel's functionality. Spend an + hour walking through the tutorial. Your commandline will love you + for it. man parallel_tutorial + +* --line-buffer will buffer output on line basis. --group keeps the + output together for a whole job. --ungroup allows output to mixup + with half a line coming from one job and half a line coming from + another job. --line-buffer fits between these two; it prints a full + line, but will allow for mixing lines of different jobs. + +* --record-env records all variables and functions set. Useful to + record the normal environment before using --env _. + +* --env _ exports all functions and variables to the remote server + except for the ones recorded. + +* New signing key. Due to recommendations from NIST + http://www.keylength.com/en/4/ the signing key was changed from + 1024D/ID:FFFFFFF1 to 9888R/ID:88888888. + +* Agalma: an automated phylogenomics workflow + http://arxiv.org/pdf/1307.6432 + +* Job ad asking for GNU Parallel expertise + http://seattle.craigslist.org/est/sof/4006079352.html + +* Transient Beowulf Clustering with GNU Parallel and SSHfs + http://www.reddit.com/r/linux/comments/1ka8mn/transient_beowulf_clustering_with_gnu_parallel/ + +* Aligning to unique regions + http://davetang.org/muse/2013/07/22/aligning-to-unique-regions/ + +* Top 10 scripting tricks for basic bioinformatics + http://students.washington.edu/bowmanjs/wordpress/?p=873 + +* Using GNU Parallel to speed up and simplify data analyzes + http://www.sergeymarkov.com/blog/2013/07/using-gnu-parallel-to-speed-up-and-simplify-data-analyzes/ + +* Compression of files in parallel using GNU parallel + http://codextechnicanum.blogspot.dk/2013/07/compression-of-files-in-parallel-using.html + +* Using GNU Parallel to roll-your-own Map Reduce! + http://www.rankfocus.com/?p=1 + +* Using GNU Parallel with Amazon + media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf + +* Some useful comments on GNU Parallel + https://news.ycombinator.com/item?id=6209767 + +* Using GNU Parallel to count reads mapping to genes for multiple BAMs + http://drduanehassane.com/blog/sequencing-resources + +* TimeMachineっぽいバックアップスクリプト + http://rio.tc/2013/07/timemachine-1.html + +* GNU ParallelでAndroid NDKの全バージョンを一括ダウンロード + http://qiita.com/mazgi/items/b10bf0ff3da2045d19ab + +* Bug fixes and man page updates. + + 20130722 * --round-robin with --pipe will write all blocks to already running diff --git a/README b/README index cc059a10..404338bd 100644 --- a/README +++ b/README @@ -44,9 +44,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20130817.tar.bz2 - bzip2 -dc parallel-20130817.tar.bz2 | tar xvf - - cd parallel-20130817 + wget http://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2 + bzip2 -dc parallel-20130822.tar.bz2 | tar xvf - + cd parallel-20130822 ./configure && make && make install @@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20130817.tar.bz2 - bzip2 -dc parallel-20130817.tar.bz2 | tar xvf - - cd parallel-20130817 + wget http://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2 + bzip2 -dc parallel-20130822.tar.bz2 | tar xvf - + cd parallel-20130822 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index dfd6f971..b7a6ca22 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.68 for parallel 20130817. +# Generated by GNU Autoconf 2.68 for parallel 20130822. # # Report bugs to . # @@ -559,8 +559,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20130817' -PACKAGE_STRING='parallel 20130817' +PACKAGE_VERSION='20130822' +PACKAGE_STRING='parallel 20130822' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1176,7 +1176,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20130817 to adapt to many kinds of systems. +\`configure' configures parallel 20130822 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1242,7 +1242,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20130817:";; + short | recursive ) echo "Configuration of parallel 20130822:";; esac cat <<\_ACEOF @@ -1309,7 +1309,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20130817 +parallel configure 20130822 generated by GNU Autoconf 2.68 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1326,7 +1326,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20130817, which was +It was created by parallel $as_me 20130822, which was generated by GNU Autoconf 2.68. Invocation command line was $ $0 $@ @@ -2141,7 +2141,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20130817' + VERSION='20130822' cat >>confdefs.h <<_ACEOF @@ -2704,7 +2704,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20130817, which was +This file was extended by parallel $as_me 20130822, which was generated by GNU Autoconf 2.68. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2766,7 +2766,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20130817 +parallel config.status 20130822 configured by $0, generated by GNU Autoconf 2.68, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index d8c6f108..9bcc1c4f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20130817], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20130822], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index e468541c..67d83ee0 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -138,6 +138,7 @@ http://www.gnu.org/software/parallel/sem.html http://www.gnu.org/software/parallel/niceload.html pod2html src/parallel.pod > ../parallel-web/parallel/man.html +pod2html src/parallel_tutorial.pod > ../parallel-web/parallel/parallel_tutorial.html pod2html src/sql > ../parallel-web/parallel/sql.html pod2html src/niceload.pod > ../parallel-web/parallel/niceload.html pod2html src/sem.pod > ../parallel-web/parallel/sem.html @@ -203,8 +204,27 @@ Subject: GNU Parallel 20130822 ('Compostela') released GNU Parallel 20130822 ('Compostela') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +It includes a big code reorganization making this release beta +quality. + New in this release: +* A tutorial showing much of GNU Parallel's functionality. Spend an + hour walking through the tutorial. Your commandline will love you + for it. man parallel_tutorial + +* --line-buffer will buffer output on line basis. --group keeps the + output together for a whole job. --ungroup allows output to mixup + with half a line coming from one job and half a line coming from + another job. --line-buffer fits between these two; it prints a full + line, but will allow for mixing lines of different jobs. + +* --record-env records all variables and functions set. Useful to + record the normal environment before using --env _. + +* --env _ exports all functions and variables to the remote server + except for the ones recorded. + * New signing key. Due to recommendations from NIST http://www.keylength.com/en/4/ the signing key was changed from 1024D/ID:FFFFFFF1 to 9888R/ID:88888888. diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 1f533e52..181ef016 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20130722 +Version: 20130822 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index 4830e0eb..234dc56a 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20130817; +$Global::version = 20130822; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index bed61225..46b912c0 100755 --- a/src/parallel +++ b/src/parallel @@ -266,22 +266,33 @@ if($opt::nonall or $opt::onall) { $Global::JobQueue = JobQueue->new( $command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files); if($opt::pipe and @opt::a) { - # Disabled for now # ... | parallel --pipe cmd ::: arg1 arg2 # The command to run is: # tee >((cmd arg1) >/tmp/tmp1 2>/tmp/err1) >((cmd arg2) >/tmp/tmp2 2>/tmp/err2) >/dev/null my @jobs; # Get all jobs + my $sshlogin = SSHLogin->new(":"); while(not $Global::JobQueue->empty()) { - push @jobs, $Global::JobQueue->get(); + my $job = $Global::JobQueue->get(); + $job->openoutputfiles(); # Do not delete + $job->set_virgin(0); + $job->set_sshlogin($sshlogin); + push @jobs, $job; } - $command = "tee ".join("",map {">((".$_->replaced().") >/tmp/tmp) "} @jobs) + @Global::tee_jobs = @jobs; + $command = "tee".join("",map {" >((".$_->replaced().")". + " >".$_->fd_file_name(1). + " 2>".$_->fd_file_name(2).")" } @jobs) ." >/dev/null"; $Global::JobQueue = JobQueue->new( $command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files); + #print ::my_dump($jobs[0]); + #print ::my_dump($Global::JobQueue); + $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget([Arg->new("")]); + #my $cmd = $Global::JobQueue->get(); + #print ::my_dump($cmd); } - if($opt::eta) { # Count the number of jobs before starting any $Global::JobQueue->total_jobs(); @@ -304,6 +315,14 @@ if($opt::pipe) { drain_job_queue(); ::debug("Done draining\n"); reaper(); +if($opt::pipe and @opt::a) { + for my $job (@Global::tee_jobs) { + unlink $job->fd_file_name(2); + $job->set_fd_file_name(2,""); + $job->print(); + unlink $job->fd_file_name(1); + } +} cleanup(); if($Global::semaphore) { $sem->release(); @@ -746,7 +765,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20130819; + $Global::version = 20130822; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3715,14 +3734,21 @@ sub openoutputfiles { # To group we create temporary files for STDOUT and STDERR # To avoid the cleanup unlink the files immediately (but keep them open) ($outfh, $outname) = ::tempfile(SUFFIX => ".par"); - $opt::files or unlink $outname; ($errfh, $errname) = ::tempfile(SUFFIX => ".par"); - unlink $errname; - $errname = ""; + if(@Global::tee_jobs) { + # files must be removed when the tee is done + } elsif($opt::files) { + # --files => only remove stderr + unlink $errname; + } else { + unlink $outname; + unlink $errname; + } } else { # --ungroup $outfh = *STDOUT; $errfh = *STDERR; + # File name must be empty as it will otherwise be printed $outname = ""; $errname = ""; } @@ -4525,7 +4551,9 @@ sub print { } ::debug("File descriptor $fdno:"); if($opt::files) { - $self->fd_file_name($fdno) and print $out_fd $self->fd_file_name($fdno),"\n"; + if($fdno == 1 and $self->fd_file_name($fdno)) { + print $out_fd $self->fd_file_name($fdno),"\n"; + } } elsif($opt::linebuffer) { # Line buffered print out my $partial = \$self->{'partial_line',$fdno}; @@ -5181,6 +5209,7 @@ sub context_replace_placeholders { # Only one input source: Treat it as a set @argset = [ @args ]; } else { + # Multiple input sources @argset = @{$self->{'arg_list'}}; } # Match -n..-1,1..n where n = max args in a argset diff --git a/src/parallel.pod b/src/parallel.pod index 7d3cdc23..9dd61bd2 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -430,7 +430,7 @@ occurs as a line of input, the rest of the input is ignored. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--delay> I +=item B<--delay> I (alpha testing) Delay starting next job I seconds. GNU B will pause I seconds after starting each job. I can be less than 1 @@ -489,7 +489,7 @@ See also B<--bg>, B. Implies B<--semaphore>. -=item B<--filter-hosts> (beta testing) +=item B<--filter-hosts> (alpha testing) Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -579,7 +579,7 @@ specified, and for B<-I>{} otherwise. This option is deprecated; use B<-I> instead. -=item B<--joblog> I +=item B<--joblog> I (alpha testing) Logfile for executed jobs. Save a list of the executed jobs to I in the following TAB separated format: sequence number, @@ -705,6 +705,17 @@ B<-l 0> is an alias for B<-l 1>. Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set. +=item B<--line-buffer> (alpha testing) + +Buffer output on line basis. B<--group> will keep the output together +for a whole job. B<--ungroup> allows output to mixup with half a line +coming from one job and half a line coming from another +job. B<--line-buffer> fits between these two: GNU B will +print a full line, but will allow for mixing lines of different jobs. + +B<--line-buffer> is slower than both B<--group> and B<--ungroup>. + + =item B<--load> I Do not start new jobs on a given computer unless the number of running @@ -786,19 +797,19 @@ When using B<--group> the output will be grouped by each server, so all the output from one server will be grouped together. -=item B<--output-as-files> +=item B<--output-as-files> (alpha testing) -=item B<--outputasfiles> +=item B<--outputasfiles> (alpha testing) -=item B<--files> +=item B<--files> (alpha testing) Instead of printing the output to stdout (standard output) the output of each job is saved in a file and the filename is then printed. -=item B<--pipe> +=item B<--pipe> (alpha testing) -=item B<--spreadstdin> +=item B<--spreadstdin> (alpha testing) Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -1004,9 +1015,9 @@ it to the command. Only used with B<--pipe>. -=item B<--results> I +=item B<--results> I (alpha testing) -=item B<--res> I +=item B<--res> I (alpha testing) Save the output into files. The files will be stored in a directory tree rooted at I. Within this directory tree, each command will result @@ -1307,9 +1318,9 @@ B<--sshlogin> is often used with B<--transfer>, B<--return>, B<--cleanup>, and B<--trc>. -=item B<--sshloginfile> I +=item B<--sshloginfile> I (alpha testing) -=item B<--slf> I +=item B<--slf> I (alpha testing) File with sshlogins. The file consists of sshlogins on separate lines. Empty lines and lines starting with '#' are ignored. Example: @@ -1334,6 +1345,9 @@ When using a different ssh program the last argument must be the hostname. Multiple B<--sshloginfile> are allowed. +GNU B will first look for the file in current dir; if that +fails it look for the file in ~/.parallel. + The sshloginfile '..' is special, it read sshlogins from ~/.parallel/sshloginfile @@ -1398,7 +1412,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--timeout> I +=item B<--timeout> I (alpha testing) Time out for command. If the command runs for longer than I seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms @@ -1543,9 +1557,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely. Print the version GNU B and exit. -=item B<--workdir> I +=item B<--workdir> I (alpha testing) -=item B<--wd> I +=item B<--wd> I (alpha testing) Files transferred using B<--transfer> and B<--return> will be relative to I on remote computers, and the command will be executed in @@ -1607,7 +1621,7 @@ See also B<-m>. Exit if the size (see the B<-s> option) is exceeded. -=item B<--xapply> (beta testing) +=item B<--xapply> Read multiple input sources like B. If multiple input sources are given, one argument will be read from each of the input diff --git a/src/parallel.texi b/src/parallel.texi index 35def3b2..f1c93b7f 100644 --- a/src/parallel.texi +++ b/src/parallel.texi @@ -456,8 +456,8 @@ Set the end of file string to eof-str. If the end of file string occurs as a line of input, the rest of the input is ignored. If neither @strong{-E} nor @strong{-e} is used, no end of file string is used. -@item @strong{--delay} @emph{secs} -@anchor{@strong{--delay} @emph{secs}} +@item @strong{--delay} @emph{secs} (alpha testing) +@anchor{@strong{--delay} @emph{secs} (alpha testing)} Delay starting next job @emph{secs} seconds. GNU @strong{parallel} will pause @emph{secs} seconds after starting each job. @emph{secs} can be less than 1 @@ -516,8 +516,8 @@ See also @strong{--bg}, @strong{man sem}. Implies @strong{--semaphore}. -@item @strong{--filter-hosts} (beta testing) -@anchor{@strong{--filter-hosts} (beta testing)} +@item @strong{--filter-hosts} (alpha testing) +@anchor{@strong{--filter-hosts} (alpha testing)} Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -611,8 +611,8 @@ This option is a synonym for @strong{-I}@emph{replace-str} if @emph{replace-str} specified, and for @strong{-I}@{@} otherwise. This option is deprecated; use @strong{-I} instead. -@item @strong{--joblog} @emph{logfile} -@anchor{@strong{--joblog} @emph{logfile}} +@item @strong{--joblog} @emph{logfile} (alpha testing) +@anchor{@strong{--joblog} @emph{logfile} (alpha testing)} Logfile for executed jobs. Save a list of the executed jobs to @emph{logfile} in the following TAB separated format: sequence number, @@ -757,6 +757,17 @@ standard specifies @strong{-L} instead. Implies @strong{-X} unless @strong{-m}, @strong{--xargs}, or @strong{--pipe} is set. +@item @strong{--line-buffer} (alpha testing) +@anchor{@strong{--line-buffer} (alpha testing)} + +Buffer output on line basis. @strong{--group} will keep the output together +for a whole job. @strong{--ungroup} allows output to mixup with half a line +coming from one job and half a line coming from another +job. @strong{--line-buffer} fits between these two: GNU @strong{parallel} will +print a full line, but will allow for mixing lines of different jobs. + +@strong{--line-buffer} is slower than both @strong{--group} and @strong{--ungroup}. + @item @strong{--load} @emph{max-load} @anchor{@strong{--load} @emph{max-load}} @@ -839,23 +850,23 @@ adjusts how many computers to log into in parallel. When using @strong{--group} the output will be grouped by each server, so all the output from one server will be grouped together. -@item @strong{--output-as-files} -@anchor{@strong{--output-as-files}} +@item @strong{--output-as-files} (alpha testing) +@anchor{@strong{--output-as-files} (alpha testing)} -@item @strong{--outputasfiles} -@anchor{@strong{--outputasfiles}} +@item @strong{--outputasfiles} (alpha testing) +@anchor{@strong{--outputasfiles} (alpha testing)} -@item @strong{--files} -@anchor{@strong{--files}} +@item @strong{--files} (alpha testing) +@anchor{@strong{--files} (alpha testing)} Instead of printing the output to stdout (standard output) the output of each job is saved in a file and the filename is then printed. -@item @strong{--pipe} -@anchor{@strong{--pipe}} +@item @strong{--pipe} (alpha testing) +@anchor{@strong{--pipe} (alpha testing)} -@item @strong{--spreadstdin} -@anchor{@strong{--spreadstdin}} +@item @strong{--spreadstdin} (alpha testing) +@anchor{@strong{--spreadstdin} (alpha testing)} Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -1069,11 +1080,11 @@ it to the command. Only used with @strong{--pipe}. -@item @strong{--results} @emph{prefix} -@anchor{@strong{--results} @emph{prefix}} +@item @strong{--results} @emph{prefix} (alpha testing) +@anchor{@strong{--results} @emph{prefix} (alpha testing)} -@item @strong{--res} @emph{prefix} -@anchor{@strong{--res} @emph{prefix}} +@item @strong{--res} @emph{prefix} (alpha testing) +@anchor{@strong{--res} @emph{prefix} (alpha testing)} Save the output into files. The files will be stored in a directory tree rooted at @emph{prefix}. Within this directory tree, each command will result @@ -1408,11 +1419,11 @@ The remote host must have GNU @strong{parallel} installed. @strong{--sshlogin} is often used with @strong{--transfer}, @strong{--return}, @strong{--cleanup}, and @strong{--trc}. -@item @strong{--sshloginfile} @emph{filename} -@anchor{@strong{--sshloginfile} @emph{filename}} +@item @strong{--sshloginfile} @emph{filename} (alpha testing) +@anchor{@strong{--sshloginfile} @emph{filename} (alpha testing)} -@item @strong{--slf} @emph{filename} -@anchor{@strong{--slf} @emph{filename}} +@item @strong{--slf} @emph{filename} (alpha testing) +@anchor{@strong{--slf} @emph{filename} (alpha testing)} File with sshlogins. The file consists of sshlogins on separate lines. Empty lines and lines starting with '#' are ignored. Example: @@ -1439,6 +1450,9 @@ When using a different ssh program the last argument must be the hostname. Multiple @strong{--sshloginfile} are allowed. +GNU @strong{parallel} will first look for the file in current dir; if that +fails it look for the file in ~/.parallel. + The sshloginfile '..' is special, it read sshlogins from ~/.parallel/sshloginfile @@ -1502,8 +1516,8 @@ into temporary files in /tmp. By setting @strong{--tmpdir} you can use a different dir for the files. Setting @strong{--tmpdir} is equivalent to setting $TMPDIR. -@item @strong{--timeout} @emph{val} -@anchor{@strong{--timeout} @emph{val}} +@item @strong{--timeout} @emph{val} (alpha testing) +@anchor{@strong{--timeout} @emph{val} (alpha testing)} Time out for command. If the command runs for longer than @emph{val} seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms @@ -1659,11 +1673,11 @@ Use @strong{-v} @strong{-v} to print the wrapping ssh command when running remot Print the version GNU @strong{parallel} and exit. -@item @strong{--workdir} @emph{mydir} -@anchor{@strong{--workdir} @emph{mydir}} +@item @strong{--workdir} @emph{mydir} (alpha testing) +@anchor{@strong{--workdir} @emph{mydir} (alpha testing)} -@item @strong{--wd} @emph{mydir} -@anchor{@strong{--wd} @emph{mydir}} +@item @strong{--wd} @emph{mydir} (alpha testing) +@anchor{@strong{--wd} @emph{mydir} (alpha testing)} Files transferred using @strong{--transfer} and @strong{--return} will be relative to @emph{mydir} on remote computers, and the command will be executed in @@ -1724,8 +1738,8 @@ See also @strong{-m}. Exit if the size (see the @strong{-s} option) is exceeded. -@item @strong{--xapply} (beta testing) -@anchor{@strong{--xapply} (beta testing)} +@item @strong{--xapply} +@anchor{@strong{--xapply}} Read multiple input sources like @strong{xapply}. If multiple input sources are given, one argument will be read from each of the input diff --git a/src/parallel_tutorial.1 b/src/parallel_tutorial.1 index 58cb4c25..aa6720fa 100644 --- a/src/parallel_tutorial.1 +++ b/src/parallel_tutorial.1 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL_TUTORIAL 1" -.TH PARALLEL_TUTORIAL 1 "2013-08-17" "20130817" "parallel" +.TH PARALLEL_TUTORIAL 1 "2013-08-22" "20130822" "parallel" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -135,7 +135,7 @@ This tutorial shows off much of \s-1GNU\s0 Parallel's functionality. The tutorial is meant to learn the options in \s-1GNU\s0 Parallel. The tutorial is not to show realistic examples from the real world. .PP -Spend an hour on walking through the tutorial. Your commandline will +Spend an hour walking through the tutorial. Your commandline will love you for it. .SH "Prerequisites" .IX Header "Prerequisites" @@ -311,7 +311,7 @@ Output: Same as above. \fIMatching arguments from all input sources\fR .IX Subsection "Matching arguments from all input sources" .PP -With \fB\-\-xapply\fR you can get one argument from each input source: +With \-\-xapply you can get one argument from each input source: .PP .Vb 1 \& parallel \-\-xapply echo ::: A B C ::: D E F @@ -1717,16 +1717,16 @@ To remove it from the remote host after the last job use \-\-cleanup. The default working dir on the remote machines is the login dir. This can be changed with \-\-workdir \fImydir\fR. .PP -Files transferred using \fB\-\-transfer\fR and \fB\-\-return\fR will be relative +Files transferred using \-\-transfer and \-\-return will be relative to \fImydir\fR on remote computers, and the command will be executed in the dir \fImydir\fR. .PP -The special \fImydir\fR value \fB...\fR will create working dirs under -\&\fB~/.parallel/tmp/\fR on the remote computers. If \fB\-\-cleanup\fR is given +The special \fImydir\fR value ... will create working dirs under +~/.parallel/tmp/ on the remote computers. If \-\-cleanup is given these dirs will be removed. .PP -The special \fImydir\fR value \fB.\fR uses the current working dir. If the -current working dir is beneath your home dir, the value \fB.\fR is +The special \fImydir\fR value . uses the current working dir. If the +current working dir is beneath your home dir, the value . is treated as the relative path to your home dir. This means that if your home dir is different on remote computers (e.g. if your login is different) the relative path will still be relative to your home dir. @@ -1932,19 +1932,15 @@ Output will be similar to: \& \*(Aq\e"\e\e\e}\e \e|\e|\e export\e my_func3=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e\e\e \e\e\e>\e\e\e \e\e\e$1.out\e"\*(Aq \& \*(Aq\e"\e\e\e}\e \e&\e&\e export\e VAR=foo\e \e&\e&\e export\e my_func2=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq \& \*(Aq\e"\e\e\e}\e \e&\e&\e eval\e my_func3\e"\e$my_func3\e"\e \e&\e&\e eval\e my_func2\e"\e$my_func2\e"\e;\e\enice\e \-n17\e /bin/bash\e \-c\e my_func3\e\e\e abc\-file;_EXIT_status=$?; mkdir \-p .; rsync \-\-rsync\-path=cd\e .parallel/tmp/hk\-31483\-1/.\e;\e rsync \-rlDzR \-essh server:abc\-file.out .;ssh server rm\e \-f\e .parallel/tmp/hk\-31483\-1/abc\-file\e;rm\e \-f\e .parallel/tmp/hk\-31483\-1/abc\-file.out\e;rm \-rf .parallel/tmp/hk\-31483\-1\e;; exit $_EXIT_status; -\& -\& ssh lo mkdir \-p .parallel/tmp/hk\-20978\-1; rsync \-rlDzR \-essh ./abc\-file lo:.parallel/tmp/hk\-20978\-1;ssh \-tt \-oLogLevel=quiet lo \*(Aqeval \`echo $SHELL | grep "/t\e{0,1\e}csh" > /dev/null && echo setenv PARALLEL_SEQ \*(Aq$PARALLEL_SEQ\*(Aq\e; setenv PARALLEL_PID \*(Aq$PARALLEL_PID\*(Aq || echo PARALLEL_SEQ=\*(Aq$PARALLEL_SEQ\*(Aq\e;export PARALLEL_SEQ\e; PARALLEL_PID=\*(Aq$PARALLEL_PID\*(Aq\e;export PARALLEL_PID\` ;\*(Aq tty\e \e>/dev/null\e \e&\e&\e stty\e isig\e \-onlcr\e \-echo\e;mkdir\e \-p\e .parallel/tmp/hk\-20978\-1\e;\e cd\e .parallel/tmp/hk\-20978\-1\e \e&\e&\e echo\e \e$SHELL\e \e|\e grep\e \e"/t\e\e\e{0,1\e\e\e}csh\e"\e \e>\e /dev/null\e \e&\e&\e setenv\e my_func\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e\e\e \e\e\e>\e\e\e \e\e\e$1.out\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e|\e|\e export\e my_func=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e\e\e \e\e\e>\e\e\e \e\e\e$1.out\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e eval\e my_func\e"\e$my_func\e"\e;\e\enice\e \-n17\e /bin/bash\e \-c\e my_func\e\e\e abc\-file;_EXIT_status=$?; mkdir \-p .; rsync \-\-rsync\-path=cd\e .parallel/tmp/hk\-20978\-1/.\e;\e rsync \-rlDzR \-essh lo:abc\-file.out .;ssh lo rm\e \-f\e .parallel/tmp/hk\-20978\-1/abc\-file\e;rm\e \-f\e .parallel/tmp/hk\-20978\-1/abc\-file.out\e;rm \-rf .parallel/tmp/hk\-20978\-1\e;; exit $_EXIT_status; .Ve .SH "\-\-pipe" .IX Header "--pipe" The \-\-pipe functionality puts \s-1GNU\s0 Parallel in a different mode: -Instead of treating the input sources as arguments for a command to -run, they will be sent to stdin (standard input) of the command. +Instead of treating the data on stdin (standard input) as arguments +for a command to run, the data will be sent to stdin (standard input) +of the command. .PP -The normal situation is that the input for \s-1GNU\s0 Parallel in \-\-pipe mode -is on stdin (standard input), so you typically have a situation like: +The typical situation is: .PP .Vb 1 \& command_A | command_B | command_C diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index 5a7b998c..9b308457 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -120,7 +120,7 @@

This tutorial shows off much of GNU Parallel's functionality. The tutorial is meant to learn the options in GNU Parallel. The tutorial is not to show realistic examples from the real world.

-

Spend an hour on walking through the tutorial. Your commandline will +

Spend an hour walking through the tutorial. Your commandline will love you for it.

@@ -279,7 +279,7 @@ sources:

Matching arguments from all input sources

-

With --xapply you can get one argument from each input source:

+

With --xapply you can get one argument from each input source:

   parallel --xapply echo ::: A B C ::: D E F

Output (the order may be different):

@@ -1190,14 +1190,14 @@ Output:

Working dir

The default working dir on the remote machines is the login dir. This can be changed with --workdir mydir.

-

Files transferred using --transfer and --return will be relative +

Files transferred using --transfer and --return will be relative to mydir on remote computers, and the command will be executed in the dir mydir.

-

The special mydir value ... will create working dirs under -~/.parallel/tmp/ on the remote computers. If --cleanup is given +

The special mydir value ... will create working dirs under +~/.parallel/tmp/ on the remote computers. If --cleanup is given these dirs will be removed.

-

The special mydir value . uses the current working dir. If the -current working dir is beneath your home dir, the value . is +

The special mydir value . uses the current working dir. If the +current working dir is beneath your home dir, the value . is treated as the relative path to your home dir. This means that if your home dir is different on remote computers (e.g. if your login is different) the relative path will still be relative to your home dir.

@@ -1339,19 +1339,15 @@ the workdir, and setting --nice value. -vv shows all of this.

'\"\\\}\ \|\|\ export\ my_func3=\\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func\\\ \\\$1\\\ \\\>\\\ \\\$1.out\"' '\"\\\}\ \&\&\ export\ VAR=foo\ \&\&\ export\ my_func2=\\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func2\\\ \\\$VAR\\\ \\\$1\"' '\"\\\}\ \&\&\ eval\ my_func3\"\$my_func3\"\ \&\&\ eval\ my_func2\"\$my_func2\"\;\\nice\ -n17\ /bin/bash\ -c\ my_func3\\\ abc-file;_EXIT_status=$?; mkdir -p .; rsync --rsync-path=cd\ .parallel/tmp/hk-31483-1/.\;\ rsync -rlDzR -essh server:abc-file.out .;ssh server rm\ -f\ .parallel/tmp/hk-31483-1/abc-file\;rm\ -f\ .parallel/tmp/hk-31483-1/abc-file.out\;rm -rf .parallel/tmp/hk-31483-1\;; exit $_EXIT_status; -
-  ssh lo mkdir -p .parallel/tmp/hk-20978-1; rsync -rlDzR -essh ./abc-file lo:.parallel/tmp/hk-20978-1;ssh -tt -oLogLevel=quiet lo  'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\;  setenv PARALLEL_PID '$PARALLEL_PID'  || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;  PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;'  tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;mkdir\ -p\ .parallel/tmp/hk-20978-1\;\ cd\ .parallel/tmp/hk-20978-1\ \&\&\ echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ my_func\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func\\\ \\\$1\\\ \\\>\\\ \\\$1.out\"'
-  '\"\\\}\ \|\|\ export\ my_func=\\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func\\\ \\\$1\\\ \\\>\\\ \\\$1.out\"'
-  '\"\\\}\ \&\&\ eval\ my_func\"\$my_func\"\;\\nice\ -n17\ /bin/bash\ -c\ my_func\\\ abc-file;_EXIT_status=$?; mkdir -p .; rsync --rsync-path=cd\ .parallel/tmp/hk-20978-1/.\;\ rsync -rlDzR -essh lo:abc-file.out .;ssh lo rm\ -f\ .parallel/tmp/hk-20978-1/abc-file\;rm\ -f\ .parallel/tmp/hk-20978-1/abc-file.out\;rm -rf .parallel/tmp/hk-20978-1\;; exit $_EXIT_status;

--pipe

The --pipe functionality puts GNU Parallel in a different mode: -Instead of treating the input sources as arguments for a command to -run, they will be sent to stdin (standard input) of the command.

-

The normal situation is that the input for GNU Parallel in --pipe mode -is on stdin (standard input), so you typically have a situation like:

+Instead of treating the data on stdin (standard input) as arguments +for a command to run, the data will be sent to stdin (standard input) +of the command.

+

The typical situation is:

   command_A | command_B | command_C

where command_B is slow, and you want to speed up command_B.

diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index cae7661d..a8c20494 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -6,7 +6,7 @@ This tutorial shows off much of GNU Parallel's functionality. The tutorial is meant to learn the options in GNU Parallel. The tutorial is not to show realistic examples from the real world. -Spend an hour on walking through the tutorial. Your commandline will +Spend an hour walking through the tutorial. Your commandline will love you for it. =head1 Prerequisites @@ -161,7 +161,7 @@ Output: Same as above. =head3 Matching arguments from all input sources -With B<--xapply> you can get one argument from each input source: +With --xapply you can get one argument from each input source: parallel --xapply echo ::: A B C ::: D E F @@ -1245,16 +1245,16 @@ To remove it from the remote host after the last job use --cleanup. The default working dir on the remote machines is the login dir. This can be changed with --workdir I. -Files transferred using B<--transfer> and B<--return> will be relative +Files transferred using --transfer and --return will be relative to I on remote computers, and the command will be executed in the dir I. -The special I value B<...> will create working dirs under -B<~/.parallel/tmp/> on the remote computers. If B<--cleanup> is given +The special I value ... will create working dirs under +~/.parallel/tmp/ on the remote computers. If --cleanup is given these dirs will be removed. -The special I value B<.> uses the current working dir. If the -current working dir is beneath your home dir, the value B<.> is +The special I value . uses the current working dir. If the +current working dir is beneath your home dir, the value . is treated as the relative path to your home dir. This means that if your home dir is different on remote computers (e.g. if your login is different) the relative path will still be relative to your home dir. @@ -1418,19 +1418,14 @@ Output will be similar to: '\"\\\}\ \&\&\ export\ VAR=foo\ \&\&\ export\ my_func2=\\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func2\\\ \\\$VAR\\\ \\\$1\"' '\"\\\}\ \&\&\ eval\ my_func3\"\$my_func3\"\ \&\&\ eval\ my_func2\"\$my_func2\"\;\\nice\ -n17\ /bin/bash\ -c\ my_func3\\\ abc-file;_EXIT_status=$?; mkdir -p .; rsync --rsync-path=cd\ .parallel/tmp/hk-31483-1/.\;\ rsync -rlDzR -essh server:abc-file.out .;ssh server rm\ -f\ .parallel/tmp/hk-31483-1/abc-file\;rm\ -f\ .parallel/tmp/hk-31483-1/abc-file.out\;rm -rf .parallel/tmp/hk-31483-1\;; exit $_EXIT_status; - ssh lo mkdir -p .parallel/tmp/hk-20978-1; rsync -rlDzR -essh ./abc-file lo:.parallel/tmp/hk-20978-1;ssh -tt -oLogLevel=quiet lo 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;mkdir\ -p\ .parallel/tmp/hk-20978-1\;\ cd\ .parallel/tmp/hk-20978-1\ \&\&\ echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ my_func\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func\\\ \\\$1\\\ \\\>\\\ \\\$1.out\"' - '\"\\\}\ \|\|\ export\ my_func=\\\(\\\)\\\ \\\{\\\ \\\ echo\\\ in\\\ my_func\\\ \\\$1\\\ \\\>\\\ \\\$1.out\"' - '\"\\\}\ \&\&\ eval\ my_func\"\$my_func\"\;\\nice\ -n17\ /bin/bash\ -c\ my_func\\\ abc-file;_EXIT_status=$?; mkdir -p .; rsync --rsync-path=cd\ .parallel/tmp/hk-20978-1/.\;\ rsync -rlDzR -essh lo:abc-file.out .;ssh lo rm\ -f\ .parallel/tmp/hk-20978-1/abc-file\;rm\ -f\ .parallel/tmp/hk-20978-1/abc-file.out\;rm -rf .parallel/tmp/hk-20978-1\;; exit $_EXIT_status; - - =head1 --pipe The --pipe functionality puts GNU Parallel in a different mode: -Instead of treating the input sources as arguments for a command to -run, they will be sent to stdin (standard input) of the command. +Instead of treating the data on stdin (standard input) as arguments +for a command to run, the data will be sent to stdin (standard input) +of the command. -The normal situation is that the input for GNU Parallel in --pipe mode -is on stdin (standard input), so you typically have a situation like: +The typical situation is: command_A | command_B | command_C diff --git a/src/sql b/src/sql index d147619b..751b5b82 100755 --- a/src/sql +++ b/src/sql @@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20130817; + $Global::version = 20130822; $Global::progname = 'sql'; # This must be done first as this may exec myself