parallel: better support for --sql csv://...

env_parallel.* added.

doc/release_new_version

@@ -213,9 +213,10 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
    Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
    Jesse Alama <jesse.alama@gmail.com>
 
-Subject: GNU Parallel 20160222 ('N977GA') released <<[stable]>>
+Subject: GNU Parallel 20160322 ('Gravitational') released <<[stable]>>
+Scott Kelly, Hillary Donald
 
-GNU Parallel 20160222 ('N977GA') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
+GNU Parallel 20160322 ('Gravitational') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -225,8 +226,6 @@ Haiku of the month:
 
 New in this release:
 
-* Vote for new community on Superuser, Ask Ubuntu, Unix&Linux http://meta.superuser.com/questions/11127/community-promotion-ads-2016/11225#11225 http://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046 http://meta.unix.stackexchange.com/questions/3854/community-promotion-ads-2016/3893#3893
-
 * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>>
 
 * <<Har angiveligt submittet ny version - afventer opdatering>> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
@@ -243,19 +242,14 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
 
 * <<Citation needed: Introspecting for RSA Key Material to Assist Intrusion Detection http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=7331177&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7331177>>
 
-* Daily Quick Tip: parallel https://medium.com/@jacaetevha/daily-quick-tip-parallel-8772435f6582
+* GNU Parallel was cited in: Random Forest DBSCAN for USPTO Inventor Name Disambiguation http://arxiv.org/pdf/1602.01792.pdf
 
-* Downloading a list of URLs http://blog.gypsydave5.com/2016/02/04/xargs-and-curl/
+* GNU Parallel was mentioned with wrong citation in: Dual Level Parallel Computations for LargeScale High-Fidelity Database to Design Aerospace Vehicles http://www.nas.nasa.gov/assets/pdf/papers/Guruswamy_2013_DualLevelParallelComputations.pdf
 
-* Using GNU Parallel and ImageMagick for Conversions http://blog.fractalcomic.com/tutorials/using-gnu-parallel-and-imagemagick-for-conversions/
+* Using ‘Parallel’ in Unix https://shearnrylan.wordpress.com/2016/02/22/using-parallel-in-unix/
 
-* Running bash commands in parallel http://linuxcommando.blogspot.dk/2016/01/running-bash-commands-in-parallel.html
+* JPEG Squish uses (and co-distributes) GNU Parallel: http://dantidswell.co.uk/jpeg-squish/
 
-*  parallel processing in PDAL https://smathermather.wordpress.com/2016/01/28/parallel-processing-in-pdal/
-
-* A Million Text Files And A Single Laptop http://randyzwitch.com/gnu-parallel-medium-data/
-
-* 「今日から使い切る」ための GNU Parallelによる並列処理入門 http://www.slideshare.net/koji_matsuda/gnu-parallel
 
 * Bug fixes and man page updates.
 

src/env_parallel.bash

@@ -0,0 +1,33 @@
+# This file must be sourced in bash:
+#
+#   source `which env_parallel.bash`
+#
+# after which 'env_parallel' works
+#
+#
+# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Ole Tange and Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+# or write to the Free Software Foundation, Inc., 51 Franklin St,
+# Fifth Floor, Boston, MA 02110-1301 USA
+
+env_parallel() {
+  export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |
+    grep -vFf <(readonly) |
+    grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) ';
+    typeset -f)";
+  `which parallel` "$@";
+  unset PARALLEL_ENV;
+}

src/env_parallel.fish

@@ -0,0 +1,30 @@
+# This file must be sourced in fish:
+#
+#   source (which env_parallel.fish)
+#
+# after which 'env_parallel' works
+#
+#
+# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Ole Tange and Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+# or write to the Free Software Foundation, Inc., 51 Franklin St,
+# Fifth Floor, Boston, MA 02110-1301 USA
+
+function env_parallel
+  setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
+  parallel $argv;
+  set -e PARALLEL_ENV
+end

src/env_parallel.ksh

@@ -0,0 +1,30 @@
+# This file must be sourced in ksh:
+#
+#   source `which env_parallel.ksh`
+#
+# after which 'env_parallel' works
+#
+#
+# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Ole Tange and Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+# or write to the Free Software Foundation, Inc., 51 Franklin St,
+# Fifth Floor, Boston, MA 02110-1301 USA
+
+env_parallel() {
+  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";
+  `which parallel` "$@";
+  unset PARALLEL_ENV;
+}

src/env_parallel.zsh

@@ -0,0 +1,34 @@
+# This file must be sourced in zsh:
+#
+#   source =env_parallel.zsh
+#
+# after which 'env_parallel' works
+#
+#
+# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Ole Tange and Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+# or write to the Free Software Foundation, Inc., 51 Franklin St,
+# Fifth Floor, Boston, MA 02110-1301 USA
+
+env_parallel() {
+  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
+    grep -aFvf <(typeset -pr) |
+    egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
+    egrep -av '^(typeset -A (commands|functions|options)|typeset IFS=|..$)|cyan';
+    typeset -f)";
+  parallel "$@";
+  unset PARALLEL_ENV;
+}

src/parallel

@@ -1176,7 +1176,7 @@ sub check_invalid_option_combinations {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20160222;
+    $Global::version = 20160223;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -9915,6 +9915,13 @@ sub uri_unescape {
 sub run {
     my $self = shift;
     my $stmt = shift;
+    if($self->{'driver'} eq "CSV") {
+	$stmt=~ s/;$//;
+	if($stmt eq "BEGIN" or
+	   $stmt eq "COMMIT") {
+	    return undef;
+	}
+    }
     my @retval;
     my $dbh = $self->{'dbh'};
     ::debug("sql","$opt::sql$opt::sqlworker run $stmt\n");
@@ -9999,10 +10006,12 @@ sub create_table {
 	"Oracle" => { "BIGINT" => "NUMBER(19,0)",
 		      "TEXT" => "CLOB", },
 	"mysql" => { "TEXT" => "LONGTEXT", },
+	"CSV" => { "BIGINT" => "INT",
+		   "FLOAT" => "REAL", },
 	);
     my $BIGINT = $vartype{$self->{'driver'}}{"BIGINT"} || "BIGINT";
     my $TEXT = $vartype{$self->{'driver'}}{"TEXT"} || "TEXT";
-    my $FLOAT = "FLOAT(44)";
+    my $FLOAT = $vartype{$self->{'driver'}}{"FLOAT"} || "FLOAT(44)";
     my $v_def = join "", map { "V$_ $TEXT," } (1..$self->max_number_of_args());
     $self->run(qq{CREATE TABLE $table
 		(Seq $BIGINT,

src/parallel.pod

@@ -1136,7 +1136,7 @@ This is useful for running the same command (e.g. uptime) on a list of
 servers.
 
 
-=item B<--onall> (beta testing)
+=item B<--onall>
 
 Run all the jobs on all computers given with B<--sshlogin>. GNU
 B<parallel> will log into B<--jobs> number of computers in parallel
@@ -1791,7 +1791,7 @@ Do not use the first line of input (used by GNU B<parallel> itself
 when called with B<--shebang>).
 
 
-=item B<--sql> I<DBURL> (beta testing)
+=item B<--sql> I<DBURL>
 
 Submit jobs via SQL server. I<DBURL> must point to a table, which will
 contain the same information as B<--joblog>, the values from the input
@@ -1823,12 +1823,12 @@ It can also be an alias from ~/.sql/aliases:
   :myalias mysql:///mydb/paralleljobs
 
 
-=item B<--sqlandworker> I<DBURL> (beta testing)
+=item B<--sqlandworker> I<DBURL>
 
 Shorthand for: B<--sql> I<DBURL> B<--sqlworker> I<DBURL>.
 
 
-=item B<--sqlworker> I<DBURL> (beta testing)
+=item B<--sqlworker> I<DBURL>
 
 Execute jobs via SQL server. Read the input sources variables from the
 table pointed to by I<DBURL>. The I<command> on the command line
@@ -2187,9 +2187,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely.
 Print the version GNU B<parallel> and exit.
 
 
-=item B<--workdir> I<mydir> (beta testing)
+=item B<--workdir> I<mydir>
 
-=item B<--wd> I<mydir> (beta testing)
+=item B<--wd> I<mydir>
 
 Files transferred using B<--transferfile> and B<--return> will be
 relative to I<mydir> on remote computers, and the command will be

src/parallel_tutorial.pod

@@ -15,9 +15,9 @@ To run this tutorial you must have the following:
 
 =over 9
 
-=item parallel >= version 20140622
+=item parallel >= version 20160222
 
-Install the newest version with:
+Install the newest version using your package manager or with:
 
   (wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
 

testsuite/tests-to-run/parallel-polarhome.sh

@@ -6,7 +6,7 @@ P_ALL="vax freebsd solaris openbsd netbsd debian alpha aix redhat hpux ultrix mi
 P_NOTWORKING="vax alpha openstep"
 P_NOTWORKING_YET="ultrix irix"
 
-P_WORKING="freebsd solaris openbsd netbsd debian aix redhat hpux qnx openindiana suse solaris-x86 mandriva ubuntu scosysv unixware centos miros raspbian macosx hpux-ia64 pidora"
+P_WORKING="freebsd solaris openbsd netbsd debian aix redhat hpux qnx openindiana suse solaris-x86 mandriva ubuntu scosysv unixware centos miros raspbian macosx hpux-ia64 pidora debian-ppc"
 P_TEMPORARILY_BROKEN="minix syllable tru64 hurd dragonfly"
 
 P="$P_WORKING"

testsuite/wanted-results/parallel-polarhome

@@ -95,8 +95,13 @@ copy_and_test pidora.polarhome.com
 pidora.polarhome.com	### Run the test on pidora.polarhome.com
 pidora.polarhome.com	Works on pidora.polarhome.com
 pidora.polarhome.com	Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000.
+copy_and_test debian-ppc.polarhome.com
+debian-ppc.polarhome.com	### Run the test on debian-ppc.polarhome.com
+debian-ppc.polarhome.com	Works on debian-ppc.polarhome.com
+debian-ppc.polarhome.com	Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000.
 aix7
 centos.polarhome.com
+debian-ppc
 debian
 freebsd.polarhome.com
 hpux-ia64