From f4c20c13b09f35427f115ba32599134eeeefb316 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 1 Jan 2016 15:12:43 +0100 Subject: [PATCH] Released as 20160101 ('20160101alpha') --- README | 12 +- configure | 20 +- configure.ac | 2 +- doc/release_new_version | 30 +- src/Makefile.in | 2 +- src/niceload | 2 +- src/parallel | 583 +++++++++++- src/parallel.pod | 110 ++- src/parallel_design.pod | 277 +++--- src/parallel_tutorial.html | 769 ++++++++++------ src/parallel_tutorial.pod | 889 +++++++++++++------ src/sql | 2 +- testsuite/tests-to-run/parallel-local-sql.sh | 75 ++ 13 files changed, 2015 insertions(+), 758 deletions(-) create mode 100644 testsuite/tests-to-run/parallel-local-sql.sh diff --git a/README b/README index 250beba5..affecffd 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20151222.tar.bz2 - bzip2 -dc parallel-20151222.tar.bz2 | tar xvf - - cd parallel-20151222 + wget http://ftpmirror.gnu.org/parallel/parallel-20160101.tar.bz2 + bzip2 -dc parallel-20160101.tar.bz2 | tar xvf - + cd parallel-20160101 ./configure && make && sudo make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20151222.tar.bz2 - bzip2 -dc parallel-20151222.tar.bz2 | tar xvf - - cd parallel-20151222 + wget http://ftpmirror.gnu.org/parallel/parallel-20160101.tar.bz2 + bzip2 -dc parallel-20160101.tar.bz2 | tar xvf - + cd parallel-20160101 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 01b791ed..58a12fe9 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20151222. +# Generated by GNU Autoconf 2.69 for parallel 20160101. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20151222' -PACKAGE_STRING='parallel 20151222' +PACKAGE_VERSION='20160101' +PACKAGE_STRING='parallel 20160101' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20151222 to adapt to many kinds of systems. +\`configure' configures parallel 20160101 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20151222:";; + short | recursive ) echo "Configuration of parallel 20160101:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20151222 +parallel configure 20160101 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20151222, which was +It was created by parallel $as_me 20160101, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20151222' + VERSION='20160101' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20151222, which was +This file was extended by parallel $as_me 20160101, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20151222 +parallel config.status 20160101 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 0edb3d93..0bc961f3 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20151222], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20160101], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index ef44512c..3db0601c 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -212,9 +212,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20151222 ('') released <<[stable]>> +Subject: GNU Parallel 20160122 ('') released <<[stable]>> -GNU Parallel 20151222 ('') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20160122 ('') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> @@ -227,11 +227,19 @@ Haiku of the month: New in this release: -* --transfer is now an alias for --transferfile {}. +* --sql DBURL uses DBURL as storage for jobs and output. It does not run any jobs so it requires at least one --sqlworker. DBURL must point to a table. -* --transferfile works like --transfer, but takes an argument like --return. This makes it possible to combine transferring files with multiple input sources: parallel -S server --tf {1} wc {2} {1} ::: * ::: -l -w -c +* --sqlworker DBURL gets jobs from DBURL and stores the result back to DBURL. -* total_jobs() can now be used in {= =}: parallel echo job {#} of '{= $_=total_jobs() =}' ::: {1..50} +* --sqlandworker is a shorthand for --sql and --sqlworker. + +* --sqlworker requires the output of a single job to fit in memory. + +* --results now also saves a file called 'seq' containing the sequence number. + +* If $PARALLEL_ENV is a file, then that file will be read into $PARALLEL_ENV. + +* man parallel_tutorial has been given an overhaul. * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>> @@ -249,17 +257,13 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> -* GNU Parallel was cited in: Evolution and Learning in Heterogeneous Environments http://research.gold.ac.uk/15078/1/COM_thesis_JonesD_2015.pdf +* GNU Parallel is used in LAST: http://last.cbrc.jp/ -* GNU Parallel was cited in: Contrasting genetic architectures of schizophrenia and other complex diseases using fast variance-components analysis http://www.nature.com/ng/journal/v47/n12/full/ng.3431.html +* GNU Parallel was cited in: Possum - A Framework for Three-Dimensional Reconstruction of Brain Images rfom Serial Sections http://link.springer.com/article/10.1007/s12021-015-9286-1 -* GNU Parallel was cited in: Efficient Retrieval of Key Material for Inspecting Potentially Malicious Traffic in the Cloud http://www.cs.bham.ac.uk/~bxb/Papres/2015.1.pdf +* GNU Parallel was used in: Mission Impossible: you have 1 minute to analyze the Ebola Genome https://www.biostars.org/p/119397 -* GNU Parallel was cited in: Achieving Consistent Doppler Measurements from SDO/HMI Vector Field Inversions http://arxiv.org/pdf/1511.06500.pdf - -* Flo uses GNU Parallel: https://github.com/wurmlab/flo - -* 使用 GNU parallel 來平行運算http://mutolisp.logdown.com/posts/316959-using-gnu-parallel-to-parallel-computing +* Distributed Log Search Using GNU Parallel http://blog.codehate.com/post/134320079974/distributed-log-search-using-gnu-parallel * Bug fixes and man page updates. diff --git a/src/Makefile.in b/src/Makefile.in index aa536945..d2675532 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -78,7 +78,7 @@ NORMAL_UNINSTALL = : PRE_UNINSTALL = : POST_UNINSTALL = : subdir = src -DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am README +DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ diff --git a/src/niceload b/src/niceload index ff0f4847..9679af82 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20151222; +$Global::version = 20160101; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 73c52f56..1e945f00 100755 --- a/src/parallel +++ b/src/parallel @@ -40,7 +40,7 @@ parse_options(); ::debug("init", "Open file descriptors: ", join(" ",keys %Global::fd), "\n"); my $number_of_args; if($Global::max_number_of_args) { - $number_of_args=$Global::max_number_of_args; + $number_of_args = $Global::max_number_of_args; } elsif ($opt::X or $opt::m or $opt::xargs) { $number_of_args = undef; } else { @@ -59,6 +59,15 @@ if($opt::pipepart) { @input_source_fh = (*STDIN); } } +if($opt::sql) { + # Create SQL table to hold joblog + output + $Global::sql->create_table($#input_source_fh+1); + if($opt::sqlworker) { + # Start a real --sqlworker in the background later + $Global::sqlworker = 1; + $opt::sqlworker = undef; + } +} if($opt::skip_first_line) { # Skip the first line for the first file handle @@ -751,6 +760,9 @@ sub options_hash { "m" => \$opt::m, "X" => \$opt::X, "v" => \@opt::v, + "sql=s" => \$opt::sql, + "sqlworker=s" => \$opt::sqlworker, + "sqlandworker=s" => \$opt::sqlandworker, "joblog=s" => \$opt::joblog, "results|result|res=s" => \$opt::results, "resume" => \$opt::resume, @@ -935,7 +947,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A init_globals(); - @ARGV=read_options(); + @ARGV = read_options(); # no-* overrides * if($opt::nokeeporder) { $opt::keeporder = undef; } @@ -960,6 +972,7 @@ sub parse_options { if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; } $opt::nice ||= 0; if(defined $opt::help) { die_usage(); } + if(defined $opt::sqlandworker) { $opt::sql = $opt::sqlworker = $opt::sqlandworker; } if(defined $opt::colsep) { $Global::trim = 'lr'; } if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; } if(defined $opt::trim) { $Global::trim = $opt::trim; } @@ -1108,6 +1121,7 @@ sub parse_options { $opt::jobs = "100%"; } open_joblog(); + ($opt::sql or $opt::sqlworker) and $Global::sql = SQL->new($opt::sql || $opt::sqlworker); } sub check_invalid_option_combinations { @@ -1162,7 +1176,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20151222; + $Global::version = 20160101; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1595,6 +1609,7 @@ sub read_options { Getopt::Long::Configure("bundling","require_order"); my @ARGV_copy = @ARGV; + my @ARGV_orig = @ARGV; # Check if there is a --profile to set @opt::profile get_options_from_array(\@ARGV_copy,"profile|J=s","plain") || die_usage(); my @ARGV_profile = (); @@ -1644,12 +1659,29 @@ sub read_options { get_options_from_array(\@ARGV_profile) || die_usage(); get_options_from_array(\@ARGV_env) || die_usage(); get_options_from_array(\@ARGV) || die_usage(); - + # What were the options given on the command line? + # Used to start --sqlworker + my $ai = arrayindex(\@ARGV_orig, \@ARGV); + @Global::options_in_argv = @ARGV_orig[0..$ai-1]; # Prepend non-options to @ARGV (such as commands like 'nice') unshift @ARGV, @ARGV_profile, @ARGV_env; return @ARGV; } +sub arrayindex { + # Similar to Perl's index function, but for arrays + # Input: + # $arr_ref1 = ref to @array1 to search in + # $arr_ref2 = ref to @array2 to search for + my ($arr_ref1,$arr_ref2) = @_; + my $array1_as_string = join "", map { "\257\257".$_ } @$arr_ref1; + my $array2_as_string = join "", map { "\257\257".$_ } @$arr_ref2; + my $i = index($array1_as_string,$array2_as_string,0); + if($i == -1) { return -1 } + my @before = split /\257\257/, substr($array1_as_string,0,$i); + return $#before; +} + sub read_args_from_command_line { # Arguments given on the command line after: # ::: ($Global::arg_sep) @@ -2322,9 +2354,27 @@ sub drain_job_queue { ::warning("There are no job slots available. Increase --jobs."); } } + while($opt::sql and not $Global::sql->finished()) { + # SQL master + $sleep = ::reap_usleep($sleep); + if($Global::sqlworker) { + # Start an SQL worker as we are now sure there is work to do + $Global::sqlworker = 0; + if(fork()) { + # skip + } else { + # Replace --sql/--sqlandworker with --sqlworker + my @ARGV = map { s/^--sql(andworker)?$/--sqlworker/; $_ } @Global::options_in_argv; + # exec the --sqlworker + exec($0,::shell_quote(@ARGV),@command); + } + } + } } while ($Global::total_running > 0 or - not $Global::start_no_new_jobs and not $Global::JobQueue->empty()); + not $Global::start_no_new_jobs and not $Global::JobQueue->empty() + or + $opt::sql and not $Global::sql->finished()); if($opt::progress) { my %progress = progress(); ::status("\r", $progress{'status'}, "\n"); @@ -3291,6 +3341,8 @@ sub reaper { my $stiff; my @pids_reaped; debug("run", "Reaper "); + # For efficiency surround with BEGIN/COMMIT when using $opt::sql + $opt::sql and $Global::sql->run("BEGIN;"); while (($stiff = waitpid(-1, &WNOHANG)) > 0) { # $stiff = pid of dead process push(@pids_reaped,$stiff); @@ -3348,6 +3400,7 @@ sub reaper { ::status("\r",$progress{'status'}); } } + $opt::sql and $Global::sql->run("COMMIT;"); debug("run", "done "); return @pids_reaped; } @@ -3741,6 +3794,13 @@ sub undef_as_empty { return $a ? $a : ""; } +sub undef_if_empty { + if(defined($_[0]) and $_[0] eq "") { + return undef; + } + return $_[0]; +} + sub multiply_binary_prefix { # Evalualte numbers with binary prefix # Ki=2^10, Mi=2^20, Gi=2^30, Ti=2^40, Pi=2^50, Ei=2^70, Zi=2^80, Yi=2^80 @@ -6092,6 +6152,15 @@ sub openoutputfiles { $dir = $opt::results."/".$args_as_dirname; File::Path::mkpath($dir); } + # prefix/name1/val1/name2/val2/seq + my $seqname = "$dir/seq"; + my $seqfhw; + if(not open($seqfhw, "+>", $seqname)) { + ::error("Cannot write to `$seqname'."); + ::wait_and_exit(255); + } + print $seqfhw $self->seq(); + close $seqfhw; # prefix/name1/val1/name2/val2/stdout $outname = "$dir/stdout"; if(not open($outfhw, "+>", $outname)) { @@ -6106,6 +6175,13 @@ sub openoutputfiles { } $self->set_fh(1,"unlink",""); $self->set_fh(2,"unlink",""); + if($opt::sqlworker) { + # Save the filenames in SQL table + $Global::sql->update("SET Stdout = ? WHERE Seq = ".$self->seq(), + $outname); + $Global::sql->update("SET Stderr = ? WHERE Seq = ".$self->seq(), + $errname); + } } elsif(not $opt::ungroup) { # To group we create temporary files for STDOUT and STDERR # To avoid the cleanup unlink the files immediately (but keep them open) @@ -6404,6 +6480,9 @@ sub set_starttime { my $self = shift; my $starttime = shift || ::now(); $self->{'starttime'} = $starttime; + $opt::sqlworker and + $Global::sql->update("SET Starttime = ? WHERE Seq = ".$self->seq(), + $starttime); } sub runtime { @@ -6425,6 +6504,9 @@ sub set_endtime { my $self = shift; my $endtime = shift; $self->{'endtime'} = $endtime; + $opt::sqlworker and + $Global::sql->update("SET JobRuntime = ? WHERE Seq = ".$self->seq(), + $self->runtime()); } sub is_timedout { @@ -6635,6 +6717,14 @@ sub wrapped { ';'; } if($ENV{'PARALLEL_ENV'}) { + if(-e $ENV{'PARALLEL_ENV'}) { + # This is a file/fifo: Replace envvar with content of file + open(my $parallel_env, "<", $ENV{'PARALLEL_ENV'}) || + ::die_bug("Cannot read parallel_env from $ENV{'PARALLEL_ENV'}"); + local $/; + $ENV{'PARALLEL_ENV'} = <$parallel_env>; + close $parallel_env; + } # If $PARALLEL_ENV set, put that in front of the command # Used for importing functions for fish # Map \001 to \n to make it easer to quote \n in $PARALLEL_ENV @@ -6664,8 +6754,11 @@ sub wrapped { and length $command > 499) { # csh does not like words longer than 1000 (499 quoted) - $command = "perl -e '".base64_zip_eval()."' ". - join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"'); + # bzip2 breaks --sql mysql://... + # $command = "perl -e '".base64_zip_eval()."' ". + # join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"'); + $command = "perl -e '".base64_eval()."' ". + join" ",string_base64('exec "'.::perl_quote_scalar($command).'"'); } $self->{'wrapped'} = $command; } @@ -6678,6 +6771,9 @@ sub set_sshlogin { $self->{'sshlogin'} = $sshlogin; delete $self->{'sshlogin_wrap'}; # If sshlogin is changed the wrap is wrong delete $self->{'wrapped'}; + $opt::sqlworker and + $Global::sql->update("SET Host = ? WHERE Seq = ".$self->seq(), + $sshlogin->string()); } sub sshlogin { @@ -6685,6 +6781,18 @@ sub sshlogin { return $self->{'sshlogin'}; } +sub string_base64 { + # Base64 encode it into 1000 byte blocks. + # 1000 bytes is the largest word size csh supports + # Input: + # @strings = to be encoded + # Returns: + # @base64 = 1000 byte block + $Global::use{"MIME::Base64"} ||= eval "use MIME::Base64; 1;"; + my @base64 = unpack("(A1000)*",encode_base64((join"",@_),"")); + return @base64; +} + sub string_zip_base64 { # Pipe string through 'bzip2 -9' and base64 encode it into 1000 # byte blocks. @@ -6707,7 +6815,7 @@ sub string_zip_base64 { close $zipin_fh; exit; } - ::debug("base64","Orig:@_\nAs base64:@base64\n"); + ::debug("base64","Orig:@_\nAs bzip2 base64:@base64\n"); return @base64; } @@ -6725,11 +6833,11 @@ sub base64_zip_eval { @GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64"); eval "@GNU_Parallel"; - $SIG{CHLD}="IGNORE"; - # Search for bzip2. Not found => use default path + $SIG{CHLD}="IGNORE"; + # Search for bzip2. Not found => use default path my $zip = (grep { -x $_ } "/usr/local/bin/bzip2")[0] || "bzip2"; - # $in = stdin on $zip, $out = stdout from $zip - my($in, $out,$eval); + # $in = stdin on $zip, $out = stdout from $zip + my($in, $out,$eval); open3($in,$out,">&STDERR",$zip,"-dc"); if(my $perlpid = fork) { close $in; @@ -6742,7 +6850,27 @@ sub base64_zip_eval { close $in; exit; } - wait; + wait; + eval $eval; + }); + ::debug("base64",$script,"\n"); + return $script; +} + +sub base64_eval { + # Script that: + # * reads base64 strings from @ARGV + # * decodes them + # * evals the result + # Reverse of string_base64 + eval + # Will be wrapped in ' so single quote is forbidden + # Returns: + # $script = 1-liner for perl -e + my $script = ::spacefree(0,q{ + @GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64"); + eval "@GNU_Parallel"; + my $eval; + $eval = decode_base64(join"",@ARGV); eval $eval; }); ::debug("base64",$script,"\n"); @@ -6913,8 +7041,11 @@ sub sshlogin_wrap { $command =~ /\n/) { # csh does not deal well with > 1000 chars in one word # csh does not deal well with $ENV with \n - $env_command = "perl -e '".base64_zip_eval()."' ". - join" ",string_zip_base64($env_command); + # bzip2 breaks --sql mysql://... + # $env_command = "perl -e '".base64_zip_eval()."' ". + # join" ",string_zip_base64($env_command); + $env_command = "perl -e '".base64_eval()."' ". + join" ",string_base64($env_command); $self->{'sshlogin_wrap'} = $env_command; } else { $self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($env_command); @@ -6942,8 +7073,11 @@ sub sshlogin_wrap { $command =~ /\n/) { # csh does not deal well with > 1000 chars in one word # csh does not deal well with $ENV with \n - $quoted_remote_command = "perl -e \\''".base64_zip_eval()."'\\' ". - join" ",string_zip_base64($remote_command); + # bzip2 breaks --sql mysql://... + # $quoted_remote_command = "perl -e \\''".base64_zip_eval()."'\\' "."". + # join" ",string_zip_base64($remote_command); + $quoted_remote_command = "perl -e \\''".base64_eval()."'\\' ". + join" ",string_base64($remote_command); } else { $quoted_remote_command = $dq_remote_command; } @@ -6999,6 +7133,9 @@ sub add_transfersize { my $self = shift; my $transfersize = shift; $self->{'transfersize'} += $transfersize; + $opt::sqlworker and + $Global::sql->update("SET Send = ? WHERE Seq = ".$self->seq(), + $self->{'transfersize'}); } sub sshtransfer { @@ -7041,6 +7178,9 @@ sub add_returnsize { my $self = shift; my $returnsize = shift; $self->{'returnsize'} += $returnsize; + $opt::sqlworker and + $Global::sql->update("SET Receive = ? WHERE Seq = ".$self->seq(), + $self->{'returnsize'}); } sub sshreturn { @@ -7275,10 +7415,10 @@ sub start { } $job->openoutputfiles(); my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w")); - if($opt::ungroup) { + if($opt::ungroup or $opt::sqlworker) { print_dryrun_and_verbose($stdout_fh,$job,$command); } - if($opt::dryrun) { $command = "true"; } + if($opt::dryrun or $opt::sql) { $command = "true"; } $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_TMP'} = ::tmpname("par"); @@ -7381,6 +7521,10 @@ sub print_dryrun_and_verbose { print $stdout_fh $command,"\n"; } } + if($opt::sqlworker) { + $Global::sql->update("SET Command = ? WHERE Seq = ".$job->seq(), + $job->replaced()); + } } { @@ -7567,7 +7711,11 @@ sub print { if(($opt::dryrun or $Global::verbose) and - not $self->{'verbose_printed'}) { + not $self->{'verbose_printed'} + and + not $opt::sql + and + not $opt::sqlworker) { $self->{'verbose_printed'}++; if($Global::verbose <= 1) { print STDOUT $self->replaced(),"\n"; @@ -7638,6 +7786,10 @@ sub files_print { } } elsif($fdno == 1 and $self->fh($fdno,"name")) { print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; + if($opt::sqlworker) { + $Global::sql->update("SET Stdout = ? WHERE Seq = ".$self->seq(), + $self->tag().$self->fh($fdno,"name")); + } $self->add_returnsize(-s $self->fh($fdno,"name")); } } @@ -7781,12 +7933,25 @@ sub normal_print { seek $in_fh, 0, 0; # $in_fh is now ready for reading at position 0 my $outputlength = 0; + my @output; while(sysread($in_fh,$buf,131072)) { print $out_fd $buf; $outputlength += length $buf; + if($opt::sqlworker) { + push @output, $buf; + } } if($fdno == 1) { $self->add_returnsize($outputlength); + if($opt::sqlworker and not $opt::results) { + $Global::sql->update("SET Stdout = ? WHERE Seq = ".$self->seq(), + join("",@output)); + } + } else { + if($opt::sqlworker and not $opt::results) { + $Global::sql->update("SET Stderr = ? WHERE Seq = ".$self->seq(), + join("",@output)); + } } close $in_fh; if($? and $opt::compress) { @@ -7851,6 +8016,9 @@ sub set_exitstatus { # Status may have been set by --timeout $self->{'exitstatus'} ||= $exitstatus; } + $opt::sqlworker and + $Global::sql->update("SET Exitval = ? WHERE Seq = ".$self->seq(), + $exitstatus); } sub reset_exitstatus { @@ -7867,9 +8035,11 @@ sub set_exitsignal { my $self = shift; my $exitsignal = shift; $self->{'exitsignal'} = $exitsignal; + $opt::sqlworker and + $Global::sql->update("SET _Signal = ? WHERE Seq = ".$self->seq(), + $exitsignal); } - { my $status_printed; my $total_jobs; @@ -8092,6 +8262,10 @@ sub populate { } } } + if($opt::sql) { + # Insert the V1..Vn for this $seq in SQL table instead of generating one + $Global::sql->insert_records($self->seq(),$self->{'arg_list_flat_orig'}); + } } sub push { @@ -8715,6 +8889,10 @@ sub get { $self->{'len'}, ); $cmd_line->populate(); + if($opt::sqlworker) { + # Get the sequence number from the SQL table + $cmd_line->set_seq($SQL::next_seq); + } ::debug("init","cmd_line->number_of_args ", $cmd_line->number_of_args(), "\n"); if(not $Global::no_more_input and ($opt::pipe or $opt::pipepart)) { @@ -8893,7 +9071,10 @@ sub new { my $colsep = shift; my @unget = (); my $arg_sub_queue; - if($colsep) { + if($opt::sqlworker) { + # Open SQL table + $arg_sub_queue = SQLRecordQueue->new(); + } elsif($colsep) { # Open one file with colsep $arg_sub_queue = RecordColQueue->new($fhs); } else { @@ -8978,7 +9159,7 @@ sub get { if(@{$self->{'unget'}}) { return shift @{$self->{'unget'}}; } - my $unget_ref=$self->{'unget'}; + my $unget_ref = $self->{'unget'}; if($self->{'arg_sub_queue'}->empty()) { return undef; } @@ -9017,6 +9198,45 @@ sub empty { } +package SQLRecordQueue; + +sub new { + my $class = shift; + my @unget = (); + return bless { + 'unget' => \@unget, + }, ref($class) || $class; +} + +sub get { + # Returns: + # reference to array of Arg-objects + my $self = shift; + if(@{$self->{'unget'}}) { + return shift @{$self->{'unget'}}; + } + return $Global::sql->get_record(); +} + +sub unget { + my $self = shift; + ::debug("run", "SQLRecordQueue-unget '@_'\n"); + unshift @{$self->{'unget'}}, @_; +} + +sub empty { + my $self = shift; + if(@{$self->{'unget'}}) { return 0; } + my $get = $self->get(); + if(defined $get) { + $self->unget($get); + } + my $empty = not $get; + ::debug("run", "SQLRecordQueue->empty $empty"); + return $empty; +} + + package MultifileQueue; @Global::unget_argv=(); @@ -9463,6 +9683,323 @@ sub insert { } +package SQL; + +sub new { + my $class = shift; + my $dburl = shift; + $Global::use{"DBI"} ||= eval "use DBI; 1;"; + my %options = parse_dburl(get_alias($dburl)); + my %driveralias = ("sqlite" => "SQLite", + "sqlite3" => "SQLite", + "pg" => "Pg", + "postgres" => "Pg", + "postgresql" => "Pg"); + my $driver = $driveralias{$options{'databasedriver'}} || $options{'databasedriver'}; + my $database = $options{'database'}; + my $host = $options{'host'} ? ";host=".$options{'host'} : ""; + my $port = $options{'port'} ? ";port=".$options{'port'} : ""; + my $dsn = "DBI:$driver:dbname=$database$host$port"; + my $userid = $options{'user'}; + my $password = $options{'password'};; + my $dbh = DBI->connect($dsn, $userid, $password, { RaiseError => 1 }) + or die $DBI::errstr; + return bless { + 'dbh' => $dbh, + 'max_number_of_args' => undef, + 'table' => $options{'table'}, + }, ref($class) || $class; +} + +sub get_alias { + my $alias = shift; + $alias =~ s/^(sql:)*//; # Accept aliases prepended with sql: + if ($alias !~ /^:/) { + return $alias; + } + + # Find the alias + my $path; + if (-l $0) { + ($path) = readlink($0) =~ m|^(.*)/|; + } else { + ($path) = $0 =~ m|^(.*)/|; + } + + my @deprecated = ("$ENV{HOME}/.dburl.aliases", + "$path/dburl.aliases", "$path/dburl.aliases.dist"); + for (@deprecated) { + if(-r $_) { + print STDERR "$_ is deprecated. Use .sql/aliases instead (read man sql)\n"; + } + } + my @urlalias=(); + check_permissions("$ENV{HOME}/.sql/aliases"); + check_permissions("$ENV{HOME}/.dburl.aliases"); + my @search = ("$ENV{HOME}/.sql/aliases", + "$ENV{HOME}/.dburl.aliases", "/etc/sql/aliases", + "$path/dburl.aliases", "$path/dburl.aliases.dist"); + for my $alias_file (@search) { + if(-r $alias_file) { + push @urlalias, `cat "$alias_file"`; + } + } + my ($alias_part,$rest) = $alias=~/(:\w*)(.*)/; + # If we saw this before: we have an alias loop + if(grep {$_ eq $alias_part } @Private::seen_aliases) { + print STDERR "$alias_part is a cyclic alias\n"; + exit -1; + } else { + push @Private::seen_aliases, $alias_part; + } + + my $dburl; + for (@urlalias) { + /^$alias_part\s+(\S+.*)/ and do { $dburl = $1; last; } + } + + if($dburl) { + return get_alias($dburl.$rest); + } else { + Usage("$alias is not defined in @search"); + exit(-1); + } +} + +sub check_permissions { + my $file = shift; + + if(-e $file) { + if(not -o $file) { + my $username = (getpwuid($<))[0]; + print STDERR "$file should be owned by $username: chown $username $file\n"; + } + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, + $atime,$mtime,$ctime,$blksize,$blocks) = stat($file); + if($mode & 077) { + my $username = (getpwuid($<))[0]; + print STDERR "$file should be only be readable by $username: chmod 600 $file\n"; + } + } +} + +sub parse_dburl { + my $url = shift; + my %options = (); + # sql:mysql://[[user][:password]@][host][:port]/[database[/table][?sql query]] + + if($url=~m!(?:sql:)? # You can prefix with 'sql:' + ((?:oracle|ora|mysql|pg|postgres|postgresql)(?:s|ssl|)| + (?:sqlite|sqlite2|sqlite3)):// # Databasedriver ($1) + (?: + ([^:@/][^:@]*|) # Username ($2) + (?: + :([^@]*) # Password ($3) + )? + @)? + ([^:/]*)? # Hostname ($4) + (?: + : + ([^/]*)? # Port ($5) + )? + (?: + / + ([^/?]*)? # Database ($6) + )? + (?: + / + ([^?]*)? # Table ($7) + )? + (?: + \? + (.*)? # Query ($8) + )? + !ix) { + $options{databasedriver} = ::undef_if_empty(lc(uri_unescape($1))); + $options{user} = ::undef_if_empty(uri_unescape($2)); + $options{password} = ::undef_if_empty(uri_unescape($3)); + $options{host} = ::undef_if_empty(uri_unescape($4)); + $options{port} = ::undef_if_empty(uri_unescape($5)); + $options{database} = ::undef_if_empty(uri_unescape($6)); + $options{table} = ::undef_if_empty(uri_unescape($7)); + $options{query} = ::undef_if_empty(uri_unescape($8)); + ::debug("sql","dburl $url\n"); + ::debug("sql","databasedriver ",$options{databasedriver}, " user ", $options{user}, + " password ", $options{password}, " host ", $options{host}, + " port ", $options{port}, " database ", $options{database}, + " table ",$options{table}," query ",$options{query}, "\n"); + + } else { + ::error("$url is not a valid DBURL"); + exit 255; + } + return %options; +} + +sub uri_unescape { + # Copied from http://cpansearch.perl.org/src/GAAS/URI-1.55/URI/Escape.pm + # to avoid depending on URI::Escape + # This section is (C) Gisle Aas. + # Note from RFC1630: "Sequences which start with a percent sign + # but are not followed by two hexadecimal characters are reserved + # for future extension" + my $str = shift; + if (@_ && wantarray) { + # not executed for the common case of a single argument + my @str = ($str, @_); # need to copy + foreach (@str) { + s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg; + } + return @str; + } + $str =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg if defined $str; + $str; +} + +sub run { + my $self = shift; + my $stmt = shift; + my @retval; + my $dbh = $self->{'dbh'}; + ::debug("sql","$opt::sql$opt::sqlworker run $stmt\n"); + # Execute with the rest of the args - if any + my $rv; + my $sth; + my $lockretry = 0; + while($lockretry < 10) { + $sth = $dbh->prepare($stmt); + if($rv = $sth->execute(@_)) { + last; + } else { + if($DBI::errstr =~ /locked/) { + ::debug("sql","Lock retry: $lockretry"); + $lockretry++; + } else { + ::error($DBI::errstr); + } + } + } + if($rv < 0){ + print $DBI::errstr; + } + return $sth; +} + +sub get { + my $self = shift; + my $sth = $self->run(@_); + my @retval; + while(1) { + my @row = $sth->fetchrow_array(); + @row or last; + push @retval, \@row; + } + return \@retval; +} + +sub table { + my $self = shift; + return $self->{'table'}; +} + +sub update { + my $self = shift; + my $stmt = shift; + my $table = $self->table(); + $self->run("UPDATE $table $stmt",@_); +} + +sub max_number_of_args { + # Maximal number of args for this table + my $self = shift; + if(not $self->{'max_number_of_args'}) { + # Read the number of args from the SQL table + my $table = $self->table(); + my $v = $self->get("SELECT * FROM $table LIMIT 1;"); + my @reserved_columns = qw(Seq Host Starttime JobRuntime Send + Receive Exitval _Signal Command Stdout Stderr); + if(not $v) { + ::error("$table contains no records"); + } + # Count the number of Vx columns + $self->{'max_number_of_args'} = $#{$v->[0]} - $#reserved_columns; + } + return $self->{'max_number_of_args'}; +} + +sub set_max_number_of_args { + my $self = shift; + $self->{'max_number_of_args'} = shift; +} + +sub create_table { + my $self = shift; + my $max_number_of_args = shift; + $self->set_max_number_of_args($max_number_of_args); + my $table = $self->table(); + $self->run(qq(DROP TABLE IF EXISTS $table;)); + my $v_def = join "", map { "V$_ TEXT," } (1..$self->max_number_of_args()); + $self->run(qq{CREATE TABLE $table + (Seq INT, + Host TEXT, + Starttime REAL, + JobRuntime REAL, + Send INT, + Receive INT, + Exitval INT, + _Signal INT, + Command TEXT,}. + $v_def. + qq{Stdout TEXT, + Stderr TEXT);}); +} + +sub insert_records { + my $self = shift; + my $seq = shift; + my $record_ref = shift; + my $table = $self->table(); + my $v_cols = join ",", map { "V$_" } (1..$self->max_number_of_args()); + # Two extra value due to $seq, Exitval + my $v_vals = join ",", map { "?" } (1..$self->max_number_of_args()+2); + $self->run("INSERT INTO $table (Seq,Exitval,$v_cols) ". + "VALUES ($v_vals);", $seq, -1000, @$record_ref[1..$#$record_ref]); +} + +sub get_record { + my $self = shift; + my @retval; + my $table = $self->table(); + my $v_cols = join ",", map { "V$_" } (1..$self->max_number_of_args()); + my $v = $self->get("SELECT Seq, $v_cols FROM $table ". + "WHERE Exitval = -1000 ORDER BY Seq LIMIT 1;"); + if($v->[0]) { + my $val_ref = $v->[0]; + # Mark record as taken + my $seq = shift @$val_ref; + # Save the sequence number to use when running the job + $SQL::next_seq = $seq; + $self->update("SET Exitval = ? WHERE Seq = ".$seq, -1220); + for (@$val_ref) { + push @retval, Arg->new($_); + } + } + if(@retval) { + return \@retval; + } else { + return undef; + } +} + +sub finished { + # Check if there are any jobs left in the SQL table that do not + # have a "real" exitval + my $self = shift; + my $table = $self->table(); + my $rv = $self->get("select Seq,Exitval from $table where Exitval <= -1000 limit 1"); + return not $rv->[0]; +} + package Semaphore; # This package provides a counting semaphore diff --git a/src/parallel.pod b/src/parallel.pod index eea52090..f4ececf0 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -514,7 +514,7 @@ should feel free to use B<--will-cite>. Size of block in bytes to read at a time. The I can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply the size with 1024, 1048576, 1073741824, 1099511627776, 1125899906842624, 1000, -1000000, 1000000000, 1000000000000, or 1000000000000000 respectively. +1000000, 1000000000, 1000000000000, or 1000000000000000, respectively. GNU B tries to meet the block size but can be off by the length of one record. For performance reasons I should be bigger @@ -1088,7 +1088,7 @@ most likely do what is needed. Multiple arguments. Insert as many arguments as the command line length permits. If multiple jobs are being run in parallel: distribute -the arguments evenly among the jobs. Use B<-j1> to avoid this. +the arguments evenly among the jobs. Use B<-j1> or B<--xargs> to avoid this. If B<{}> is not used the arguments will be appended to the line. If B<{}> is used multiple times each B<{}> will be replaced @@ -1106,7 +1106,7 @@ Minimum memory free when starting another job. The I can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply the size with 1024, 1048576, 1073741824, 1099511627776, 1125899906842624, 1000, 1000000, 1000000000, 1000000000000, or -1000000000000000 respectively. +1000000000000000, respectively. If the jobs take up very different amount of RAM, GNU B will only start as many as there is memory for. If less than I bytes @@ -1791,6 +1791,50 @@ Do not use the first line of input (used by GNU B itself when called with B<--shebang>). +=item B<--sql> I (alpha testing) + +Submit jobs via SQL server. I must point to a table, which will +contain the same information as B<--joblog>, the values from the input +sources (stored in columns V1 .. Vn), and the output (stored in +columns Stdout and Stderr). + +The table will be dropped and created with the correct amount of +V-columns. + +B<--sql> does not run any jobs, but it creates the values for the jobs +to be run and wait for them to complete. One or more B<--sqlworker> +must be run to actually execute the jobs. + +The format of a DBURL is: + + [sql:]vendor://[[user][:password]@][host][:port]/[database]/table + +E.g. + + sql:mysql://hr:hr@localhost:3306/hrdb/jobs + mysql://scott:tiger@my.example.com/pardb/paralleljobs + sql:oracle://scott:tiger@ora.example.com/xe/parjob + postgresql://scott:tiger@pg.example.com/pgdb/parjob + pg:///parjob + sqlite3:///pardb/parjob + +It can also be an alias from ~/.sql/aliases: + + :myalias mysql:///mydb/paralleljobs + + +=item B<--sqlandworker> I (alpha testing) + +Shorthand for: B<--sql> I B<--sqlworker> I. + + +=item B<--sqlworker> I (alpha testing) + +Execute jobs via SQL server. Read the input sources variables from the +table pointed to by I. The I on the command line +should be the same as given by B<--sql>. + + =item B<--ssh> I GNU B defaults to using B for remote access. This can @@ -2579,7 +2623,8 @@ files. tar xvf foo.tgz | perl -ne 'print $l;$l=$_;END{print $l}' | \ parallel echo -The Perl one-liner is needed to avoid race condition. +The Perl one-liner is needed to make sure the file is complete before +handing it to GNU B. =head1 EXAMPLE: Rewriting a for-loop and a while-read-loop @@ -3410,6 +3455,35 @@ computers: true >jobqueue; tail -n+0 -f jobqueue | parallel -S .. +If you keep this running for a long time, jobqueue will grow. A way of +removing the jobs already run is by making GNU B stop when +it hits a special value and then restart. To use B<--eof> to make GNU +B exit, B also needs to be forced to exit: + + true >jobqueue; + while true; do + tail -n+0 -f jobqueue | + (parallel -E StOpHeRe -S ..; echo GNU Parallel is now done; + perl -e 'while(<>){/StOpHeRe/ and last};print <>' jobqueue > j2; + (seq 1000 >> jobqueue &); + echo Done appending dummy data forcing tail to exit) + echo tail exited; + mv j2 jobqueue + done + +In some cases you can run on more CPUs and computers during the night: + + # Day time + echo 50% > jobfile + cp day_server_list ~/.parallel/sshloginfile + # Night time + echo 100% > jobfile + cp night_server_list ~/.parallel/sshloginfile + tail -n+0 -f jobqueue | parallel --jobs jobfile -S .. + +GNU Parallel discovers if B or B<~/.parallel/sshloginfile> +changes. + There is a a small issue when using GNU B as queue system/batch manager: You have to submit JobSlot number of jobs before they will start, and after that you can submit one at a time, and job @@ -3421,14 +3495,6 @@ E.g. if you have 10 jobslots then the output from the first completed job will only be printed when job 11 has started, and the output of second completed job will only be printed when job 12 has started. -To use B<--eof> to make GNU B exit, B also needs to be -forced to exit: - - tail -n+0 -f command-list.txt | - (parallel --eof=EXIT {}; echo Parallel is now done; - (seq 1000 >> command-list.txt &); - echo Done appending dummy data forcing tail to exit) - =head1 EXAMPLE: GNU Parallel as dir processor @@ -3773,6 +3839,26 @@ More than 100 jobs failed. Other error. +=item Z<>-1 (In joblog and SQL table) + +Killed by Ctrl-C, timeout, not enough memory or similar. + +=item Z<>-2 (In joblog and SQL table) + +$job->skip() was called in {= =}. + +=item Z<>-1200 (In SQL table) + +Job is ready to run (used with --sql). + +=item Z<>-1220 (In SQL table) + +Job is taken by worker to be started (used with --sql). + +=item Z<>-1250 (In SQL table) + +Job is running (used with --sql). + =back If fail=1 is used, the exit status will be the exit status of the diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 4ff06cdc..cda28c58 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -32,7 +32,7 @@ is CentOS 3.9 and Perl 5.8.0. GNU B busy waits. This is because the reason why a job is not started may be due to load average, and thus it will not make sense to wait for a job to finish. Instead the load average must be -checked again. Load average is not the only reason: --timeout has a +checked again. Load average is not the only reason: B<--timeout> has a similar problem. To not burn up too much CPU GNU B sleeps exponentially @@ -56,14 +56,13 @@ The easiest way to explain what GNU B does is to assume that there are a number of job slots, and when a slot becomes available a job from the queue will be run in that slot. But originally GNU B did not model job slots in the code. Job slots have been -added to make it possible to use {%} as a replacement string. +added to make it possible to use B<{%}> as a replacement string. -While the job -sequence number can be computed in advance, the job slot can only be -computed the moment a slot becomes available. So it has been -implemented as a stack with lazy evaluation: Draw one from an empty -stack and the stack is extended by one. When a job is done, push the -available job slot back on the stack. +While the job sequence number can be computed in advance, the job slot +can only be computed the moment a slot becomes available. So it has +been implemented as a stack with lazy evaluation: Draw one from an +empty stack and the stack is extended by one. When a job is done, push +the available job slot back on the stack. This implementation also means that if you use remote executions, you cannot assume that a given job slot will remain on the same remote @@ -89,7 +88,7 @@ B<--compress> compresses the data in the temporary files. This is a bit tricky because there should be no files to clean up if GNU B is killed by a power outage. -GNU B first selects a compress program. If the user has not +GNU B first selects a compression program. If the user has not selected one, the first of these that are in $PATH is used: B. They are sorted by speed on a 16 core machine. @@ -102,16 +101,16 @@ Schematically the setup is as follows: The setup is duplicated for both standard output (stdout) and standard error (stderr). -GNU B pipes output from the command run into the compress +GNU B pipes output from the command run into the compression program which saves to a tmpfile. GNU B records the pid of the compress program. At the same time a small perl script (called B above) is started: It basically does B followed by B, but it also removes the tmpfile as soon as the first byte -is read, and it continously checks if the pid of the compress program -is dead. If the compress program is dead, B reads the rest of -tmpfile and exits. +is read, and it continously checks if the pid of the compression +program is dead. If the compress program is dead, B reads the +rest of tmpfile and exits. -As most compress programs write out a header when they start, the +As most compression programs write out a header when they start, the tmpfile in practice is unlinked after around 40 ms. @@ -134,16 +133,12 @@ Local: B =item --cat -cat > {}; I {}; -perl -e '$bash = shift; - $csh = shift; - for(@ARGV) { - unlink;rmdir; - } - if($bash =~ s/h//) { - exit $bash; - } - exit $csh;' "$?h" "$status" {}; + cat > {}; I {}; + perl -e '$bash = shift; + $csh = shift; + for(@ARGV) { unlink;rmdir; } + if($bash =~ s/h//) { exit $bash; } + exit $csh;' "$?h" "$status" {}; {} is set to $PARALLEL_TMP which is a tmpfile. The Perl script saves the exit value, unlinks the tmpfile, and returns the exit value - no @@ -151,22 +146,22 @@ matter if the shell is B (using $?) or B<*csh> (using $status). =item --fifo -perl -e '($s,$c,$f) = @ARGV; - # mkfifo $PARALLEL_TMP - system "mkfifo", $f; - # spawn $shell -c $command & - $pid = fork || exec $s, "-c", $c; - open($o,">",$f) || die $!; - # cat > $PARALLEL_TMP - while(sysread(STDIN,$buf,131072)){ - syswrite $o, $buf; - } - close $o; - # waitpid to get the exit code from $command - waitpid $pid,0; - # Cleanup - unlink $f; - exit $?/256;' I I $PARALLEL_TMP + perl -e '($s,$c,$f) = @ARGV; + # mkfifo $PARALLEL_TMP + system "mkfifo", $f; + # spawn $shell -c $command & + $pid = fork || exec $s, "-c", $c; + open($o,">",$f) || die $!; + # cat > $PARALLEL_TMP + while(sysread(STDIN,$buf,131072)){ + syswrite $o, $buf; + } + close $o; + # waitpid to get the exit code from $command + waitpid $pid,0; + # Cleanup + unlink $f; + exit $?/256;' I I $PARALLEL_TMP This is an elaborate way of: mkfifo {}; run I in the background using I; copying STDIN to {}; waiting for background @@ -214,19 +209,19 @@ B<$_EXIT_status>: see B<--return> above. =item --pipe -perl -e 'if(sysread(STDIN, $buf, 1)) { - open($fh, "|-", "@ARGV") || die; - syswrite($fh, $buf); - # Align up to 128k block - if($read = sysread(STDIN, $buf, 131071)) { - syswrite($fh, $buf); - } - while($read = sysread(STDIN, $buf, 131072)) { - syswrite($fh, $buf); - } - close $fh; - exit ($?&127 ? 128+($?&127) : 1+$?>>8) - }' I -c I + perl -e 'if(sysread(STDIN, $buf, 1)) { + open($fh, "|-", "@ARGV") || die; + syswrite($fh, $buf); + # Align up to 128k block + if($read = sysread(STDIN, $buf, 131071)) { + syswrite($fh, $buf); + } + while($read = sysread(STDIN, $buf, 131072)) { + syswrite($fh, $buf); + } + close $fh; + exit ($?&127 ? 128+($?&127) : 1+$?>>8) + }' I -c I This small wrapper makes sure that I will never be run if there is no data. @@ -261,8 +256,6 @@ the FIFO, and this value is used as exit value. To make it compatible with B and B the exit value is printed as: $?h/$status and this is parsed by B. -Works in B. - There is a bug that makes it necessary to print the exit value 3 times. @@ -272,17 +265,17 @@ are added to the title to force it to be outside the limits. You can map the bad limits using: -perl -e 'sub r { int(rand(shift)).($_[0] && "\t".r(@_)) } print map { r(@ARGV)."\n" } 1..10000' 1600 1500 90 | - perl -ane '$F[0]+$F[1]+$F[2] < 2037 and print ' | - parallel --colsep '\t' --tagstring '{1}\t{2}\t{3}' tmux -S /tmp/p{%}-'{=3 $_="O"x$_ =}' \ - new-session -d -n '{=1 $_="O"x$_ =}' true'\ {=2 $_="O"x$_ =};echo $?;rm -f /tmp/p{%}-O*' + perl -e 'sub r { int(rand(shift)).($_[0] && "\t".r(@_)) } print map { r(@ARGV)."\n" } 1..10000' 1600 1500 90 | + perl -ane '$F[0]+$F[1]+$F[2] < 2037 and print ' | + parallel --colsep '\t' --tagstring '{1}\t{2}\t{3}' tmux -S /tmp/p{%}-'{=3 $_="O"x$_ =}' \ + new-session -d -n '{=1 $_="O"x$_ =}' true'\ {=2 $_="O"x$_ =};echo $?;rm -f /tmp/p{%}-O*' -perl -e 'sub r { int(rand(shift)).($_[0] && "\t".r(@_)) } print map { r(@ARGV)."\n" } 1..10000' 17000 17000 90 | - parallel --colsep '\t' --tagstring '{1}\t{2}\t{3}' \ -tmux -S /tmp/p{%}-'{=3 $_="O"x$_ =}' new-session -d -n '{=1 $_="O"x$_ =}' true'\ {=2 $_="O"x$_ =};echo $?;rm /tmp/p{%}-O*' -> value.csv 2>/dev/null + perl -e 'sub r { int(rand(shift)).($_[0] && "\t".r(@_)) } print map { r(@ARGV)."\n" } 1..10000' 17000 17000 90 | + parallel --colsep '\t' --tagstring '{1}\t{2}\t{3}' \ + tmux -S /tmp/p{%}-'{=3 $_="O"x$_ =}' new-session -d -n '{=1 $_="O"x$_ =}' true'\ {=2 $_="O"x$_ =};echo $?;rm /tmp/p{%}-O*' + > value.csv 2>/dev/null -R -e 'a<-read.table("value.csv");X11();plot(a[,1],a[,2],col=a[,3]+5,cex=0.1);Sys.sleep(1000)' + R -e 'a<-read.table("value.csv");X11();plot(a[,1],a[,2],col=a[,3]+5,cex=0.1);Sys.sleep(1000)' For B 17000 can be lowered to 2100. @@ -306,23 +299,6 @@ B<--pipepart>/B<--pipe> should be done on the local machine inside B<--tmux> =back -=head2 --block-size adjustment - -Every time GNU B detects a record bigger than -B<--block-size> it increases the block size by 30%. A small -B<--block-size> gives very poor performance; by exponentially -increasing the block size performance will not suffer. - -GNU B will waste CPU power if B<--block-size> does not -contain a full record, because it tries to find a full record and will -fail to do so. The recommendation is therefore to use a -B<--block-size> > 2 records, so you always get at least one full -record when you read one block. - -If you use B<-N> then B<--block-size> should be big enough to contain -N+1 records. - - =head2 Convenience options --nice --basefile --transfer --return --cleanup --tmux --group --compress --cat --fifo --workdir @@ -360,6 +336,23 @@ that correctly for all corner cases is next to impossible to do by hand. +=head2 --block-size adjustment + +Every time GNU B detects a record bigger than +B<--block-size> it increases the block size by 30%. A small +B<--block-size> gives very poor performance; by exponentially +increasing the block size performance will not suffer. + +GNU B will waste CPU power if B<--block-size> does not +contain a full record, because it tries to find a full record and will +fail to do so. The recommendation is therefore to use a +B<--block-size> > 2 records, so you always get at least one full +record when you read one block. + +If you use B<-N> then B<--block-size> should be big enough to contain +N+1 records. + + =head2 Shell shock The shell shock bug in B did not affect GNU B, but the @@ -402,10 +395,14 @@ exec'ing a Perl wrapper to monitor the parent pid and kill the child if the parent pid becomes 1, then Ctrl-C works and stderr is kept on stderr. +To be able to kill all (grand)*children a new process group is +started. + =head3 --nice Bing the remote process is done by B. A -few old systems do not implement this and is thus unsupported. +few old systems do not implement this and B<--nice> is unsupported on +those. =head3 Setting $PARALLEL_TMP @@ -417,39 +414,39 @@ remote system. The wrapper looks like this: - $shell = $PARALLEL_SHELL || $SHELL; - $tmpdir = $TMPDIR; - $nice = $opt::nice; - # Set $PARALLEL_TMP to a non-existent file name in $TMPDIR - do { - $ENV{PARALLEL_TMP} = $tmpdir."/par". - join"", map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5); - } while(-e $ENV{PARALLEL_TMP}); - $SIG{CHLD} = sub { $done = 1; }; - $pid = fork; - unless($pid) { - # Make own process group to be able to kill HUP it later - setpgrp; - eval { setpriority(0,0,$nice) }; - exec $shell, "-c", ($bashfunc."@ARGV"); - die "exec: $!\n"; - } - do { - # Parent is not init (ppid=1), so sshd is alive - # Exponential sleep up to 1 sec - $s = $s < 1 ? 0.001 + $s * 1.03 : $s; - select(undef, undef, undef, $s); - } until ($done || getppid == 1); - # Kill HUP the process group if job not done - kill(SIGHUP, -${pid}) unless $done; - wait; - exit ($?&127 ? 128+($?&127) : 1+$?>>8) + $shell = $PARALLEL_SHELL || $SHELL; + $tmpdir = $TMPDIR; + $nice = $opt::nice; + # Set $PARALLEL_TMP to a non-existent file name in $TMPDIR + do { + $ENV{PARALLEL_TMP} = $tmpdir."/par". + join"", map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5); + } while(-e $ENV{PARALLEL_TMP}); + $SIG{CHLD} = sub { $done = 1; }; + $pid = fork; + unless($pid) { + # Make own process group to be able to kill HUP it later + setpgrp; + eval { setpriority(0,0,$nice) }; + exec $shell, "-c", ($bashfunc."@ARGV"); + die "exec: $!\n"; + } + do { + # Parent is not init (ppid=1), so sshd is alive + # Exponential sleep up to 1 sec + $s = $s < 1 ? 0.001 + $s * 1.03 : $s; + select(undef, undef, undef, $s); + } until ($done || getppid == 1); + # Kill HUP the process group if job not done + kill(SIGHUP, -${pid}) unless $done; + wait; + exit ($?&127 ? 128+($?&127) : 1+$?>>8) =head2 Transferring of variables and functions Transferring of variables and functions given by B<--env> is done by running a Perl script remotely that calls the actual command. The Perl -script sets $ENV{variable} to the correct value before exec'ing the a +script sets B<$ENV{>IB<}> to the correct value before exec'ing the a shell that runs the function definition followed by the actual command. @@ -512,9 +509,9 @@ shell is used when GNU B executes commands. GNU B tries hard to use the right shell. If GNU B is called from B it will use B. If it is called from B it will use B. It does this by looking at the -(grand*)parent process: If the (grand*)parent process is a shell, use -this shell; otherwise look at the parent of this (grand*)parent. If -none of the (grand*)parents are shells, then $SHELL is used. +(grand)*parent process: If the (grand)*parent process is a shell, use +this shell; otherwise look at the parent of this (grand)*parent. If +none of the (grand)*parents are shells, then $SHELL is used. This will do the right thing if called from: @@ -570,7 +567,7 @@ not known to B. =back If GNU B guesses wrong in these situation, set the shell using -$PARALLEL_SHELL. +B<$PARALLEL_SHELL>. =head2 Quoting @@ -596,7 +593,7 @@ implemented very differently. With B<--pipe> GNU B reads the blocks from standard input (stdin), which is then given to the command on standard input (stdin); so every block is being processed by GNU B itself. This is -the reason why B<--pipe> maxes out at around 100 MB/sec. +the reason why B<--pipe> maxes out at around 500 MB/sec. B<--pipepart>, on the other hand, first identifies at which byte positions blocks start and how long they are. It does that by seeking @@ -766,42 +763,32 @@ B and put the result in a file, which is then used next time. =head2 Killing jobs -B<--memfree>, B<--halt> and when GNU B meets a condition -from which it cannot recover, jobs are killed. This is done by finding -the (grand)*children of the jobs and killing those processes. +GNU B kills jobs. It can be due to B<--memfree>, B<--halt>, +or when GNU B meets a condition from which it cannot +recover. Every job is started as its own process group. This way any +(grand)*children will get killed, too. The process group is killed with +the specification mentioned in B<--termseq>. -More specifically GNU B maintains a list of processes to be -killed, sends a signal to all processes (first round this is a TERM). -It weeds out the processes that exited from the list then waits a -while and weeds out again. It does that until all processes are dead -or 200 ms passed. Then it does another round with TERM, and finally a -round with KILL. - pids = family_pids(jobs) - for signal in TERM, TERM, KILL: - for pid in pids: - kill signal, pid - while kill 0, pids and slept < 200 ms: - sleep sleeptime - pids = kill 0, pids - slept += sleeptime - sleeptime = sleeptime * 1.1 +=head2 SQL interface -By doing so there is a tiny risk, that GNU B will kill -processes that are not started from GNU B. It, however, -requires all of these to be true: +GNU B uses the DBURL from GNU B to give database +software, username, password, host, port, database, and table in a +single string. -* Process A is sent a signal -* It dies during a I cycle -* A new process B is spawned (by an unrelated process) -* This is done during the same I cycle -* B is owned by the same user -* B reuses the pid of the A +The DBURL must point to a table name. The table will be dropped and +created. The reason for not reusing an exising table is that the user +may have added more input sources which would require more columns in +the table. -It is considered unlikely to ever happen due to: +The table columns are similar to joblog with the addition of B +.. B which are values from the input sources, and stdout and +stderr which are the output from standard output and standard error, +respectively. + +The Signal column has been renamed to _Signal due to Signal being a +reserved word in MySQL. -* The longest I sleeps is 10 ms -* Re-use of a dead pid rarely happens within a few seconds =head1 Ideas for new design diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index 6e9bb581..f9467795 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -40,6 +40,7 @@
  • Positional perl expression replacement string
  • Input from columns
  • Header defined replacement strings
    • +
  • More pre-defined replacement strings
  • More than one argument
    • @@ -52,15 +53,19 @@
  • Saving output into files
    • -
  • Control the execution +
  • Controlling the execution
    • @@ -72,10 +77,15 @@
  • Avoid overloading sshd
  • Ignore hosts that are down
  • Running the same commands on all hosts
    • -
  • Transfer environment variables and functions
    • +
  • Transferring environment variables and functions
  • Showing what is actually run
    • +
  • Saving to an SQL base (advanced) + +
  • --pipe
    • Chunk size
      • @@ -93,6 +103,7 @@
    • Semaphore @@ -104,7 +115,7 @@

      GNU Parallel Tutorial

      -

      This tutorial shows off much of GNU Parallel's functionality. The tutorial is meant to learn the options in GNU Parallel. The tutorial is not to show realistic examples from the real world.

      +

      This tutorial shows off much of GNU parallel's functionality. The tutorial is meant to learn the options in GNU parallel. The tutorial is not to show realistic examples from the real world.

      Spend an hour walking through the tutorial. Your command line will love you for it.

      @@ -226,7 +237,7 @@

      Input sources

      -

      GNU Parallel reads input from input sources. These can be files, the command line, and stdin (standard input or a pipe).

      +

      GNU parallel reads input from input sources. These can be files, the command line, and stdin (standard input or a pipe).

      A single input source

      @@ -254,7 +265,7 @@

      Multiple input sources

      -

      GNU Parallel can take multiple input sources given on the command line. GNU Parallel then generates all combinations of the input sources:

      +

      GNU parallel can take multiple input sources given on the command line. GNU parallel then generates all combinations of the input sources:

        parallel echo ::: A B C ::: D E F
      @@ -276,13 +287,13 @@

      Output: Same as above.

      -

      STDIN (standard input) can be one of the input sources using '-':

      +

      STDIN (standard input) can be one of the input sources using -:

        cat abc-file | parallel -a - -a def-file echo

      Output: Same as above.

      -

      Instead of -a files can be given after '::::':

      +

      Instead of -a files can be given after :::::

        cat abc-file | parallel echo :::: - def-file
      @@ -296,7 +307,7 @@

      Matching arguments from all input sources

      -

      With --xapply you can get one argument from each input source:

      +

      With --xapply you can get one argument from each input source:

        parallel --xapply echo ::: A B C ::: D E F
      @@ -320,7 +331,7 @@

      Changing the argument separator.

      -

      GNU Parallel can use other separators than ::: or ::::. This is typically useful if ::: or :::: is used in the command to run:

      +

      GNU parallel can use other separators than ::: or ::::. This is typically useful if ::: or :::: is used in the command to run:

        parallel --arg-sep ,, echo ,, A B C :::: def-file
      @@ -344,7 +355,7 @@

      Changing the argument delimiter

      -

      GNU Parallel will normally treat a full line as a single argument: It uses \n as argument delimiter. This can be changed with -d:

      +

      GNU parallel will normally treat a full line as a single argument: It uses \n as argument delimiter. This can be changed with -d:

        parallel -d _ echo :::: abc_-file
      @@ -354,13 +365,13 @@ B C -

      NULL can be given as \0:

      +

      NULL can be given as \0:

        parallel -d '\0' echo :::: abc0-file

      Output: Same as above.

      -

      A shorthand for -d '\0' is -0 (this will often be used to read files from find ... -print0):

      +

      A shorthand for -d '\0' is -0 (this will often be used to read files from find ... -print0):

        parallel -0 echo :::: abc0-file
      @@ -368,7 +379,7 @@

      End-of-file value for input source

      -

      GNU Parallel can stop reading when it encounters a certain value:

      +

      GNU parallel can stop reading when it encounters a certain value:

        parallel -E stop echo ::: A B stop C D
      @@ -379,7 +390,7 @@

      Skipping empty lines

      -

      Using --no-run-if-empty GNU Parallel will skip empty lines.

      +

      Using --no-run-if-empty GNU parallel will skip empty lines.

        (echo 1; echo; echo 2) | parallel --no-run-if-empty echo
      @@ -402,7 +413,7 @@ foo [/path/to/current/working/dir] -

      The command can be a script, a binary or a Bash function if the function is exported using 'export -f':

      +

      The command can be a script, a binary or a Bash function if the function is exported using export -f:

        # Only works in Bash
   my_func() {
@@ -421,7 +432,7 @@
 
 
      The 7 predefined replacement strings
      
 
-
      GNU Parallel has several replacement strings. If no replacement strings are used the default is to append {}:
      
+
      GNU parallel has several replacement strings. If no replacement strings are used the default is to append {}:
      
 
 
        parallel echo ::: A/B.C
      
 
@@ -429,7 +440,7 @@
 
 
        A/B.C
      
 
-
      The default replacement string is {}:
      
+
      The default replacement string is {}:
      
 
 
        parallel echo {} ::: A/B.C
      
 
@@ -437,7 +448,7 @@
 
 
        A/B.C
      
 
-
      The replacement string {.} removes the extension:
      
+
      The replacement string {.} removes the extension:
      
 
 
        parallel echo {.} ::: A/B.C
      
 
@@ -445,7 +456,7 @@
 
 
        A/B
      
 
-
      The replacement string {/} removes the path:
      
+
      The replacement string {/} removes the path:
      
 
 
        parallel echo {/} ::: A/B.C
      
 
@@ -453,7 +464,7 @@
 
 
        B.C
      
 
-
      The replacement string {//} keeps only the path:
      
+
      The replacement string {//} keeps only the path:
      
 
 
        parallel echo {//} ::: A/B.C
      
 
@@ -461,7 +472,7 @@
 
 
        A
      
 
-
      The replacement string {/.} removes the path and the extension:
      
+
      The replacement string {/.} removes the path and the extension:
      
 
 
        parallel echo {/.} ::: A/B.C
      
 
@@ -469,7 +480,7 @@
 
 
        B
      
 
-
      The replacement string {#} gives the job number:
      
+
      The replacement string {#} gives the job number:
      
 
 
        parallel echo {#} ::: A B C
      
 
@@ -479,7 +490,7 @@
   2
   3
      -

      The replacement string {%} gives the job slot number (between 1 and number of jobs to run in parallel):

      +

      The replacement string {%} gives the job slot number (between 1 and number of jobs to run in parallel):

        parallel -j 2 echo {%} ::: A B C
      @@ -491,7 +502,7 @@

      Changing the replacement strings

      -

      The replacement string {} can be changed with -I:

      +

      The replacement string {} can be changed with -I:

        parallel -I ,, echo ,, ::: A/B.C
      @@ -499,7 +510,7 @@ 
        A/B.C
      -

      The replacement string {.} can be changed with --extensionreplace:

      +

      The replacement string {.} can be changed with --extensionreplace:

        parallel --extensionreplace ,, echo ,, ::: A/B.C
      @@ -507,7 +518,7 @@ 
        A/B
      -

      The replacement string {/} can be replaced with --basenamereplace:

      +

      The replacement string {/} can be replaced with --basenamereplace:

        parallel --basenamereplace ,, echo ,, ::: A/B.C
      @@ -515,7 +526,7 @@ 
        B.C
      -

      The replacement string {//} can be changed with --dirnamereplace:

      +

      The replacement string {//} can be changed with --dirnamereplace:

        parallel --dirnamereplace ,, echo ,, ::: A/B.C
      @@ -523,7 +534,7 @@ 
        A
      -

      The replacement string {/.} can be changed with --basenameextensionreplace:

      +

      The replacement string {/.} can be changed with --basenameextensionreplace:

        parallel --basenameextensionreplace ,, echo ,, ::: A/B.C
      @@ -531,7 +542,7 @@ 
        B
      -

      The replacement string {#} can be changed with --seqreplace:

      +

      The replacement string {#} can be changed with --seqreplace:

        parallel --seqreplace ,, echo ,, ::: A B C
      @@ -541,7 +552,7 @@ 2 3 -

      The replacement string {%} can be changed with --slotreplace:

      +

      The replacement string {%} can be changed with --slotreplace:

        parallel -j2 --slotreplace ,, echo ,, ::: A B C
      @@ -553,7 +564,7 @@

      Perl expression replacement string

      -

      When predefined replacement strings are not flexible enough a perl expression can be used instead. One example is to remove two extensions: foo.tar.gz -> foo

      +

      When predefined replacement strings are not flexible enough a perl expression can be used instead. One example is to remove two extensions: foo.tar.gz becomes foo

        parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
      @@ -561,25 +572,67 @@ 
        foo
      -

      If the strings {= and =} cause problems they can be replaced with --parens:

      +

      In {= =} you can access all of GNU parallel's internal functions and variables. A few are worth mentioning.

      + +

      total_jobs() returns the total number of jobs:

      + +
        parallel echo Job {#} of {= '$_=total_jobs()' =} ::: {1..5}
      + +

      Output:

      + +
        Job 1 of 5
+  Job 2 of 5
+  Job 3 of 5
+  Job 4 of 5
+  Job 5 of 5
      + +

      Q(...) shell quotes the string:

      + +
        parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$'
      + +

      $job->skip() skips the job:

      + +
        parallel echo {= 'if($_==3) { $job->skip() }' =} ::: {1..5}
      + +

      Output:

      + +
        1
+  2
+  4
+  5
      + +

      @arg contains the input source variables:

      + +
        parallel echo {= 'if($arg[1]==$arg[2]) { $job->skip() }' =} ::: {1..3} ::: {1..3}
      + +

      Output:

      + +
        1 2
+  1 3
+  2 1
+  2 3
+  3 1
+  3 2
      + +

      If the strings {= and =} cause problems they can be replaced with --parens:

        parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz

      Output: Same as above.

      -

      To define a short hand replacement string use --rpl:

      +

      To define a shorthand replacement string use --rpl:

        parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz

      Output: Same as above.

      -

      If the short hand starts with '{' it can be used as a positional replacement string, too:

      +

      If the shorthand starts with { it can be used as a positional replacement string, too:

        parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz

      Output: Same as above.

      -

      GNU parallel's 7 replacement strings are implemented as:

      +

      GNU parallel's 7 replacement strings are implemented as this:

        --rpl '{} '
   --rpl '{#} $_=$job->seq()'
@@ -591,7 +644,7 @@
 
 
      Positional replacement strings
      
 
-
      With multiple input sources the argument from the individual input sources can be access with {number}:
      
+
      With multiple input sources the argument from the individual input sources can be accessed with {number}:
      
 
 
        parallel echo {1} and {2} ::: A B ::: C D
      
 
@@ -602,7 +655,7 @@
   B and C
   B and D
      -

      The positional replacement strings can also be modified using / // /. and .:

      +

      The positional replacement strings can also be modified using /, //, /., and .:

        parallel echo /={1/} //={1//} /.={1/.} .={1.} ::: A/B.C D/E.F
      @@ -636,7 +689,7 @@ 
        foo bar
      -

      If a defined short hand starts with '{' it can be used as a positional replacement string, too:

      +

      If shorthand defined using --rpl starts with { it can be used as a positional replacement string, too:

        parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz
      @@ -644,7 +697,7 @@

      Input from columns

      -

      The columns in a file can be bound to positional replacement strings using --colsep. Here the columns are separated with TAB (\t):

      +

      The columns in a file can be bound to positional replacement strings using --colsep. Here the columns are separated by TAB (\t):

        parallel --colsep '\t' echo 1={1} 2={2} :::: tsv-file.tsv
      @@ -656,7 +709,7 @@

      Header defined replacement strings

      -

      With --header GNU Parallel will use the first value of the input source as the name of the replacement string. Only the non-modified version {} is supported:

      +

      With --header GNU parallel will use the first value of the input source as the name of the replacement string. Only the non-modified version {} is supported:

        parallel --header : echo f1={f1} f2={f2} ::: f1 A B ::: f2 C D
      @@ -667,7 +720,7 @@ f1=B f2=C f1=B f2=D -

      It is useful with --colsep for processing files with TAB separated values:

      +

      It is useful with --colsep for processing files with TAB separated values:

        parallel --header : --colsep '\t' echo f1={f1} f2={f2} :::: tsv-file.tsv
      @@ -676,9 +729,38 @@ 
        f1=A f2=B
   f1=C f2=D
      +

      More pre-defined replacement strings

      + +

      --plus adds the replacement strings {+/} {+.} {+..} {+...} {..} {...} {/..} {/...} {##}. The idea being that {+foo} matches the opposite of {foo} and {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}.

      + +
        parallel --plus echo {} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {.}.{+.} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/.}.{+.} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {..}.{+..} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/..}.{+..} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {...}.{+...} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/...}.{+...} ::: dir/sub/file.ext1.ext2.ext3
      + +

      Output:

      + +
        dir/sub/file.ext1.ext2.ext3
      + +

      {##} is simply the number of jobs:

      + +
        parallel --plus echo Job {#} of {##} ::: {1..5}
      + +

      Output:

      + +
        Job 1 of 5
+  Job 2 of 5
+  Job 3 of 5
+  Job 4 of 5
+  Job 5 of 5
      +

      More than one argument

      -

      With --xargs will GNU Parallel fit as many arguments as possible on a single line:

      +

      With --xargs GNU parallel will fit as many arguments as possible on a single line:

        cat num30000 | parallel --xargs echo | wc -l
      @@ -688,7 +770,7 @@

      The 30000 arguments fitted on 2 lines.

      -

      The maximal length of a single line can be set with -s. With a maximal line length of 10000 chars 17 commands will be run:

      +

      The maximal length of a single line can be set with -s. With a maximal line length of 10000 chars 17 commands will be run:

        cat num30000 | parallel --xargs -s 10000 echo | wc -l
      @@ -696,11 +778,11 @@ 
        17
      -

      For better parallelism GNU Parallel can distribute the arguments between all the parallel jobs when end of file is met.

      +

      For better parallelism GNU parallel can distribute the arguments between all the parallel jobs when end of file is met.

      -

      Below GNU Parallel reads the last argument when generating the second job. When GNU Parallel reads the last argument, it spreads all the arguments for the second job over 4 jobs instead, as 4 parallel jobs are requested.

      +

      Below GNU parallel reads the last argument when generating the second job. When GNU parallel reads the last argument, it spreads all the arguments for the second job over 4 jobs instead, as 4 parallel jobs are requested.

      -

      The first job will be the same as the --xargs example above, but the second job will be split into 4 evenly sized jobs, resulting in a total of 5 jobs:

      +

      The first job will be the same as the --xargs example above, but the second job will be split into 4 evenly sized jobs, resulting in a total of 5 jobs:

        cat num30000 | parallel --jobs 4 -m echo | wc -l
      @@ -719,7 +801,7 @@ 7 8 9 10 -

      A replacement string can be part of a word. -m will not repeat the context:

      +

      A replacement string can be part of a word. -m will not repeat the context:

        parallel --jobs 4 -m echo pre-{}-post ::: A B C D E F G
      @@ -730,7 +812,7 @@ pre-E F-post pre-G-post -

      To repeat the context use -X which otherwise works like -m:

      +

      To repeat the context use -X which otherwise works like -m:

        parallel --jobs 4 -X echo pre-{}-post ::: A B C D E F G
      @@ -741,7 +823,7 @@ pre-E-post pre-F-post pre-G-post -

      To limit the number of arguments use -N:

      +

      To limit the number of arguments use -N:

        parallel -N3 echo ::: A B C D E F G H
      @@ -751,7 +833,7 @@ D E F G H -

      -N also sets the positional replacement strings:

      +

      -N also sets the positional replacement strings:

        parallel -N3 echo 1={1} 2={2} 3={3} ::: A B C D E F G H
      @@ -761,7 +843,7 @@ 1=D 2=E 3=F 1=G 2=H 3= -

      -N0 reads 1 argument but inserts none:

      +

      -N0 reads 1 argument but inserts none:

        parallel -N0 echo foo ::: 1 2 3
      @@ -775,7 +857,7 @@

      Command lines that contain special characters may need to be protected from the shell.

      -

      The perl program 'print "@ARGV\n"' basically works like echo.

      +

      The perl program print "@ARGV\n" basically works like echo.

        perl -e 'print "@ARGV\n"' A
      @@ -791,7 +873,7 @@ 
        [Nothing]
      -

      To quote the command use -q:

      +

      To quote the command use -q:

        parallel -q perl -e 'print "@ARGV\n"' ::: This works
      @@ -800,7 +882,7 @@ 
        This
   works
      -

      Or you can quote the critical part using \':

      +

      Or you can quote the critical part using \':

        parallel perl -e \''print "@ARGV\n"'\' ::: This works, too
      @@ -810,7 +892,7 @@ works, too -

      GNU Parallel can also \-quote full lines. Simply run:

      +

      GNU parallel can also \-quote full lines. Simply run this:

        parallel --shellquote
   parallel: Warning: Input is read from the terminal. Only experts do this on purpose. Press CTRL-D to exit.
@@ -833,7 +915,7 @@
 
 
      Trimming space
      
 
-
      Space can be trimmed on the arguments using --trim:
      
+
      Space can be trimmed on the arguments using --trim:
      
 
 
        parallel --trim r echo pre-{}-post ::: ' A '
      
 
@@ -869,7 +951,7 @@
   B       foo-B
   C       foo-C
      -

      To prefix it with another string use --tagstring:

      +

      To prefix it with another string use --tagstring:

        parallel --tagstring {}-bar echo foo-{} ::: A B C
      @@ -879,7 +961,7 @@ B-bar foo-B C-bar foo-C -

      To see what commands will be run without running them:

      +

      To see what commands will be run without running them use --dryrun:

        parallel --dryrun echo {} ::: A B C
      @@ -889,7 +971,7 @@ echo B echo C -

      To print the command before running them use --verbose:

      +

      To print the command before running them use --verbose:

        parallel --verbose echo {} ::: A B C
      @@ -902,7 +984,7 @@ B C -

      GNU Parallel will postpone the output until the command completes:

      +

      GNU parallel will postpone the output until the command completes:

        parallel -j2 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
      @@ -918,7 +1000,7 @@ 4-middle 4-end -

      To get the output immediately use --ungroup:

      +

      To get the output immediately use --ungroup:

        parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
      @@ -934,9 +1016,9 @@ -middle 4-end -

      --ungroup is fast, but can cause half a line from one job to be mixed with half a line of another job. That has happend in the second line, where the line '4-middle' is mixed with '2-start'.

      +

      --ungroup is fast, but can cause half a line from one job to be mixed with half a line of another job. That has happend in the second line, where the line '4-middle' is mixed with '2-start'.

      -

      To avoid this use --linebuffer:

      +

      To avoid this use --linebuffer:

        parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
      @@ -952,7 +1034,7 @@ 4-middle 4-end -

      To force the output in the same order as the arguments use --keep-order/-k:

      +

      To force the output in the same order as the arguments use --keep-order/-k:

        parallel -j2 -k 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
      @@ -970,21 +1052,21 @@

      Saving output into files

      -

      GNU Parallel can save the output of each job into files:

      +

      GNU parallel can save the output of each job into files:

        parallel --files echo ::: A B C
      -

      Output will be similar to:

      +

      Output will be similar to this:

        /tmp/pAh6uWuQCg.par
   /tmp/opjhZCzAX4.par
   /tmp/W0AT_Rph2o.par
      -

      By default GNU Parallel will cache the output in files in /tmp. This can be changed by setting $TMPDIR or --tmpdir:

      +

      By default GNU parallel will cache the output in files in /tmp. This can be changed by setting $TMPDIR or --tmpdir:

        parallel --tmpdir /var/tmp --files echo ::: A B C
      -

      Output will be similar to:

      +

      Output will be similar to this:

        /var/tmp/N_vk7phQRc.par
   /var/tmp/7zA4Ccf3wZ.par
@@ -996,7 +1078,7 @@
 
 
      Output: Same as above.
      
 
-
      The output files can be saved in a structured way using --results:
      
+
      The output files can be saved in a structured way using --results:
      
 
 
        parallel --results outdir echo ::: A B C
      
 
@@ -1006,43 +1088,50 @@
   B
   C
      -

      but also these files were generated containing the standard output (stdout) and standard error (stderr):

      +

      These files were also generated containing the standard output (stdout), standard error (stderr), and the sequence number (seq):

      -
        outdir/1/A/stderr
+
        outdir/1/A/seq
+  outdir/1/A/stderr
   outdir/1/A/stdout
+  outdir/1/B/seq
   outdir/1/B/stderr
   outdir/1/B/stdout
+  outdir/1/C/seq
   outdir/1/C/stderr
   outdir/1/C/stdout
      
 
-
      This is useful if you are running multiple variables:
      
+
      --header : will take the first value as name and use that in the directory structure. This is useful if you are using multiple input sources:
      
 
 
        parallel --header : --results outdir echo ::: f1 A B ::: f2 C D
      
 
 
      Generated files:
      
 
-
        outdir/f1/A/f2/C/stderr
+
        outdir/f1/A/f2/C/seq
+  outdir/f1/A/f2/C/stderr
   outdir/f1/A/f2/C/stdout
+  outdir/f1/A/f2/D/seq
   outdir/f1/A/f2/D/stderr
   outdir/f1/A/f2/D/stdout
+  outdir/f1/B/f2/C/seq
   outdir/f1/B/f2/C/stderr
   outdir/f1/B/f2/C/stdout
+  outdir/f1/B/f2/D/seq
   outdir/f1/B/f2/D/stderr
   outdir/f1/B/f2/D/stdout
      
 
 
      The directories are named after the variables and their values.
      
 
-
      Control the execution
      
+
      Controlling the execution
      
 
 
      Number of simultaneous jobs
      
 
-
      The number of concurrent jobs is given with --jobs/-j:
      
+
      The number of concurrent jobs is given with --jobs/-j:
      
 
 
        /usr/bin/time parallel -N0 -j64 sleep 1 :::: num128
      
 
-
      With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run - depending on how fast your machine is.
      
+
      With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run - depending on how fast your machine is.
      
 
-
      By default --jobs is the same as the number of CPU cores. So this:
      
+
      By default --jobs is the same as the number of CPU cores. So this:
      
 
 
        /usr/bin/time parallel -N0 sleep 1 :::: num128
      
 
@@ -1050,13 +1139,13 @@
 
 
        /usr/bin/time parallel -N0 --jobs 200% sleep 1 :::: num128
      
 
-
      --jobs 0 will run as many jobs in parallel as possible:
      
+
      --jobs 0 will run as many jobs in parallel as possible:
      
 
 
        /usr/bin/time parallel -N0 --jobs 0 sleep 1 :::: num128
      
 
 
      which should take 1-7 seconds depending on how fast your machine is.
      
 
-
      --jobs can read from a file which is re-read when a job finishes:
      
+
      --jobs can read from a file which is re-read when a job finishes:
      
 
 
        echo 50% > my_jobs
   /usr/bin/time parallel -N0 --jobs my_jobs sleep 1 :::: num128 &
@@ -1064,15 +1153,15 @@
   echo 0 > my_jobs
   wait
      
 
-
      The first second only 50% of the CPU cores will run a job. The '0' is put into my_jobs and then the rest of the jobs will be started in parallel.
      
+
      The first second only 50% of the CPU cores will run a job. Then 0 is put into my_jobs and then the rest of the jobs will be started in parallel.
      
 
-
      Instead of basing the percentage on the number of CPU cores GNU Parallel can base it on the number of CPUs:
      
+
      Instead of basing the percentage on the number of CPU cores GNU parallel can base it on the number of CPUs:
      
 
 
        parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
      
 
 
      Shuffle job order
      
 
-
      If you have many jobs (e.g. by multiple combinations of input sources), it can be handy to shuffle the jobs, so you get different values run.
      
+
      If you have many jobs (e.g. by multiple combinations of input sources), it can be handy to shuffle the jobs, so you get different values run. Use --shuf for that:
      
 
 
        parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C
      
 
@@ -1082,7 +1171,7 @@
 
 
      Interactivity
      
 
-
      GNU Parallel can ask the user if a command should be run using --interactive:
      
+
      GNU parallel can ask the user if a command should be run using --interactive:
      
 
 
        parallel --interactive echo ::: 1 2 3
      
 
@@ -1094,7 +1183,7 @@
   echo 3 ?...y
   3
      
 
-
      GNU Parallel can be used to put arguments on the command line for an interactive command such as emacs to edit one file at a time:
      
+
      GNU parallel can be used to put arguments on the command line for an interactive command such as emacs to edit one file at a time:
      
 
 
        parallel --tty emacs ::: 1 2 3
      
 
@@ -1104,7 +1193,7 @@
 
 
      A terminal for every job
      
 
-
      Using tmux GNU Parallel can start a terminal for every job run:
      
+
      Using --tmux GNU parallel can start a terminal for every job run:
      
 
 
        seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}'
      
 
@@ -1112,11 +1201,11 @@
 
 
        tmux -S /tmp/tmsrPrO0 attach
      
 
-
      Using normal tmux keystrokes (CTRL-b n or CTRL-b p) you can cycle between windows of the running jobs. When a job is finished it will pause for 10 seconds before closing the window.
      
+
      Using normal tmux keystrokes (CTRL-b n or CTRL-b p) you can cycle between windows of the running jobs. When a job is finished it will pause for 10 seconds before closing the window.
      
 
 
      Timing
      
 
-
      Some jobs do heavy I/O when they start. To avoid a thundering herd GNU Parallel can delay starting new jobs. --delay X will make sure there is at least X seconds between each start:
      
+
      Some jobs do heavy I/O when they start. To avoid a thundering herd GNU parallel can delay starting new jobs. --delay X will make sure there is at least X seconds between each start:
      
 
 
        parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3
      
 
@@ -1129,7 +1218,7 @@
   Starting 3
   Thu Aug 15 16:24:38 CEST 2013
      -

      If jobs taking more than a certain amount of time are known to fail, they can be stopped with --timeout. The accuracy of --timeout is 2 seconds:

      +

      If jobs taking more than a certain amount of time are known to fail, they can be stopped with --timeout. The accuracy of --timeout is 2 seconds:

        parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8
      @@ -1138,7 +1227,7 @@ 
        2
   4
      -

      GNU Parallel can compute the median runtime for jobs and kill those that take more than 200% of the median runtime:

      +

      GNU parallel can compute the median runtime for jobs and kill those that take more than 200% of the median runtime:

        parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3
      @@ -1149,7 +1238,9 @@ 3 2.3 -

      Based on the runtime of completed jobs GNU Parallel can estimate the total runtime:

      +

      Progress information

      + +

      Based on the runtime of completed jobs GNU parallel can estimate the total runtime:

        parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1
      @@ -1161,9 +1252,7 @@ Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete ETA: 2s 0left 1.11avg local:0/9/100%/1.1s -

      Progress

      - -

      GNU Parallel can give progress information with --progress:

      +

      GNU parallel can give progress information with --progress:

        parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
      @@ -1175,15 +1264,15 @@ Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete local:0/9/100%/1.1s -

      A progress bar can be shown with --bar:

      +

      A progress bar can be shown with --bar:

        parallel --bar sleep ::: 1 3 2 2 1 3 3 2 1
      -

      And a graphic bar can be shown with --bar and zenity:

      +

      And a graphic bar can be shown with --bar and zenity:

        seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill)
      -

      A logfile of the jobs completed so far can be generated with --joblog:

      +

      A logfile of the jobs completed so far can be generated with --joblog:

        parallel --joblog /tmp/log exit  ::: 1 2 3 0 
   cat /tmp/log
      @@ -1196,9 +1285,9 @@ 3 : 1376577364.990 0.013 0 0 3 0 exit 3 4 : 1376577365.003 0.003 0 0 0 0 exit 0 -

      The log contains the job sequence, which host the job was run on, the start time and run time, how much data was transferred if the job was run on a remote host, the exit value, the signal that killed the job, and finally the command being run.

      +

      The log contains the job sequence, which host the job was run on, the start time and run time, how much data was transferred, the exit value, the signal that killed the job, and finally the command being run.

      -

      With a joblog GNU Parallel can be stopped and later pickup where it left off. It it important that the input of the completed jobs is unchanged.

      +

      With a joblog GNU parallel can be stopped and later pickup where it left off. It it important that the input of the completed jobs is unchanged.

        parallel --joblog /tmp/log exit  ::: 1 2 3 0 
   cat /tmp/log
@@ -1221,9 +1310,9 @@
   5       :       1376580070.028  0.009   0       0       0       0       exit 0
   6       :       1376580070.038  0.007   0       0       0       0       exit 0
      -

      Note how the start time of the last 2 jobs is clearly from the second run.

      +

      Note how the start time of the last 2 jobs is clearly different from the second run.

      -

      With --resume-failed GNU Parallel will re-run the jobs that failed:

      +

      With --resume-failed GNU parallel will re-run the jobs that failed:

        parallel --resume-failed --joblog /tmp/log exit  ::: 1 2 3 0 0 0
   cat /tmp/log
      @@ -1241,11 +1330,32 @@ 2 : 1376580154.444 0.022 0 0 2 0 exit 2 3 : 1376580154.466 0.005 0 0 3 0 exit 3 -

      Note how seq 1 2 3 have been repeated because they had exit value != 0.

      +

      Note how seq 1 2 3 have been repeated because they had exit value different from 0.

      + +

      --retry-failed does almost the same as --resume-failed. Where --resume-failed reads the commands from the command line (and ignores the commands in the joblog), --retry-failed ignores the command line and reruns the commands mentioned in the joblog.

      + +
        parallel --resume-failed --joblog /tmp/log
+  cat /tmp/log
      + +

      Output:

      + +
        Seq     Host    Starttime       Runtime Send    Receive Exitval Signal  Command
+  1       :       1376580069.544  0.008   0       0       1       0       exit 1
+  2       :       1376580069.552  0.009   0       0       2       0       exit 2
+  3       :       1376580069.560  0.012   0       0       3       0       exit 3
+  4       :       1376580069.571  0.005   0       0       0       0       exit 0
+  5       :       1376580070.028  0.009   0       0       0       0       exit 0
+  6       :       1376580070.038  0.007   0       0       0       0       exit 0
+  1       :       1376580154.433  0.010   0       0       1       0       exit 1
+  2       :       1376580154.444  0.022   0       0       2       0       exit 2
+  3       :       1376580154.466  0.005   0       0       3       0       exit 3
+  1       :       1376580164.633  0.010   0       0       1       0       exit 1
+  2       :       1376580164.644  0.022   0       0       2       0       exit 2
+  3       :       1376580164.666  0.005   0       0       3       0       exit 3

      Termination

      -

      For certain jobs there is no need to continue if one of the jobs fails and has an exit code != 0. GNU Parallel will stop spawning new jobs with --halt soon,fail=1:

      +

      For certain jobs there is no need to continue if one of the jobs fails and has an exit code different from 0. GNU parallel will stop spawning new jobs with --halt soon,fail=1:

        parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
      @@ -1260,7 +1370,7 @@ parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: echo 2; exit 2 -

      With --halt now,fail=1 the running jobs will be killed immediately:

      +

      With --halt now,fail=1 the running jobs will be killed immediately:

        parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
      @@ -1272,7 +1382,7 @@ parallel: This job failed: echo 1; exit 1 -

      If --halt is given a percentage this percentage of the jobs must fail before GNU Parallel stops spawning more jobs:

      +

      If --halt is given a percentage this percentage of the jobs must fail before GNU parallel stops spawning more jobs:

        parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9
      @@ -1290,9 +1400,9 @@ parallel: This job failed: echo 3; exit 3 -

      If you are looking for success instead of failures, you can use success:

      +

      If you are looking for success instead of failures, you can use success. This will finish as soon as the first job succeeds:

      -
        parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
      +
        parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6

      Output:

      @@ -1301,11 +1411,9 @@ 3 0 parallel: This job succeeded: - echo 0; exit 0 - parallel: Starting no more jobs. Waiting for 1 jobs to finish. - 4 + echo 0; exit 0 -

      GNU Parallel can retry the command with --retries. This is useful if a command fails for unknown reasons now and then.

      +

      GNU parallel can retry the command with --retries. This is useful if a command fails for unknown reasons now and then.

        parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
   cat /tmp/runs
      @@ -1326,9 +1434,42 @@

      Note how job 1 and 2 were tried 3 times, but 0 was not retried because it had exit code 0.

      +

      Termination signals (advanced)

      + +

      Using --termseq you can control which signals are sent when killing children. Normally children will be killed by sending them SIGTERM, waiting 200 ms, then another SIGTERM, waiting 100 ms, then another SIGTERM, waiting 50 ms, then a SIGKILL, finally waiting 25 ms before giving up. It looks like this:

      + +
        show_signals() {
+    perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' 
+  }
+  export -f show_signals
+  echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals
      + +

      Output:

      + +
        Got TERM
+  Got TERM
+  Got TERM
      + +

      Or just:

      + +
        echo | parallel -u --timeout 1 show_signals
      + +

      Output: Same as above.

      + +

      You can change this to SIGINT, SIGTERM, SIGKILL:

      + +
        echo | parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals
      + +

      Output:

      + +
        Got INT
+  Got TERM
      + +

      The SIGKILL does not show because it cannot be caught, and thus the child dies.

      +

      Limiting the resources

      -

      To avoid overloading systems GNU Parallel can look at the system load before starting another job:

      +

      To avoid overloading systems GNU parallel can look at the system load before starting another job:

        parallel --load 100% echo load is less than {} job per cpu ::: 1
      @@ -1337,7 +1478,7 @@ 
        [when then load is less than the number of cpu cores]
   load is less than 1 job per cpu
      -

      GNU Parallel can also check if the system is swapping.

      +

      GNU parallel can also check if the system is swapping.

        parallel --noswap echo the system is not swapping ::: now
      @@ -1346,7 +1487,11 @@ 
        [when then system is not swapping]
   the system is not swapping now
      -

      GNU Parallel can run the jobs with a nice value. This will work both locally and remotely.

      +

      Some jobs need a lot of memory, and should only be started when there is enough memory free. Using --memfree GNU parallel can check if there is enough memory free. Additionally, GNU parallel will kill off the youngest job if the memory free falls below 50% of the size. The killed job will put back on the queue and retried later.

      + +
        parallel --memfree 1G echo will run if more than 1 GB is ::: free
      + +

      GNU parallel can run the jobs with a nice value. This will work both locally and remotely.

        parallel --nice 17 echo this is being run with nice -n ::: 17
      @@ -1356,11 +1501,11 @@

      Remote execution

      -

      GNU Parallel can run jobs on remote servers. It uses ssh to communicate with the remote machines.

      +

      GNU parallel can run jobs on remote servers. It uses ssh to communicate with the remote machines.

      Sshlogin

      -

      The most basic sshlogin is -S host:

      +

      The most basic sshlogin is -S host:

        parallel -S $SERVER1 echo running on ::: $SERVER1
      @@ -1368,7 +1513,7 @@ 
        running on [$SERVER1]
      -

      To use a different username prepend the server with username@

      +

      To use a different username prepend the server with username@:

        parallel -S username@$SERVER1 echo running on ::: username@$SERVER1
      @@ -1376,7 +1521,7 @@ 
        running on [username@$SERVER1]
      -

      The special sshlogin ':' is the local machine:

      +

      The special sshlogin : is the local machine:

        parallel -S : echo running on ::: the_local_machine
      @@ -1384,7 +1529,7 @@ 
        running on the_local_machine
      -

      If ssh is not in $PATH it can be prepended to $SERVER1:

      +

      If ssh is not in $PATH it can be prepended to $SERVER1:

        parallel -S '/usr/bin/ssh '$SERVER1 echo custom ::: ssh
      @@ -1392,7 +1537,16 @@ 
        custom ssh
      -

      Several servers can be given using multiple -S:

      +

      The ssh command can also be given using --ssh:

      + +
        parallel --ssh /usr/bin/ssh -S $SERVER1 echo custom ::: ssh
      + +

      or by setting $PARALLEL_SSH:

      + +
        export PARALLEL_SSH=/usr/bin/ssh
+  parallel -S $SERVER1 echo custom ::: ssh
      + +

      Several servers can be given using multiple -S:

        parallel -S $SERVER1 -S $SERVER2 echo ::: running on more hosts
      @@ -1403,7 +1557,7 @@ more hosts -

      Or they can be separated by ,:

      +

      Or they can be separated by ,:

        parallel -S $SERVER1,$SERVER2 echo ::: running on more hosts
      @@ -1415,7 +1569,7 @@ SERVERS="`echo $SERVER1; echo $SERVER2`" parallel -S "$SERVERS" echo ::: running on more hosts -

      The can also be read from a file (replace user@ with the user on $SERVER2):

      +

      They can also be read from a file (replace user@ with the user on $SERVER2):

        echo $SERVER1 > nodefile
   # Force 4 cores, special ssh-command, username
@@ -1424,9 +1578,11 @@
 
 
      Output: Same as above.
      
 
-
      The special --sshloginfile '..' reads from ~/.parallel/sshloginfile.
      
+
      Every time a job finished, the --sshloginfile will be re-read, so it is possible to both add and remove hosts while running.
      
 
-
      To force GNU Parallel to treat a server having a given number of CPU cores prepend #/ to the sshlogin:
      
+
      The special --sshloginfile .. reads from ~/.parallel/sshloginfile.
      
+
+
      To force GNU parallel to treat a server having a given number of CPU cores prepend the number of core followed by / to the sshlogin:
      
 
 
        parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4
      
 
@@ -1434,18 +1590,20 @@
 
 
        force 4 cpus on server
      
 
-
      Servers can be put into groups by prepending '@groupname' to the server and the group can then be selected by appending '@groupname' to the argument if using '--hostgroup'.
      
+
      Servers can be put into groups by prepending @groupname to the server and the group can then be selected by appending @groupname to the argument if using --hostgroup:
      
 
-
        parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
      
+
        parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: \
+    run_on_grp1@grp1 run_on_grp2@grp2
      
 
 
      Output:
      
 
 
        run_on_grp1
   run_on_grp2
      
 
-
      A host can be in multiple groups by separating groups with '+', and you can force GNU parallel to limit the groups on which the command can be run with '-S @groupname':
      
+
      A host can be in multiple groups by separating the groups with +, and you can force GNU parallel to limit the groups on which the command can be run with -S @groupname:
      
 
-
        parallel -S @grp1 -S @grp1+grp2/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1 also_grp1
      
+
        parallel -S @grp1 -S @grp1+grp2/$SERVER1 -S @grp2/SERVER2 echo {} ::: \
+    run_on_grp1 also_grp1
      
 
 
      Output:
      
 
@@ -1454,32 +1612,32 @@
 
 
      Transferring files
      
 
-
      GNU Parallel can transfer the files to be processed to the remote host. It does that using rsync.
      
+
      GNU parallel can transfer the files to be processed to the remote host. It does that using rsync.
      
 
 
        echo This is input_file > input_file
-  parallel -S $SERVER1 --transfer cat ::: input_file 
      
+  parallel -S $SERVER1 --transferfile {} cat ::: input_file

      Output:

        This is input_file
      -

      If the files is processed into another file, the resulting file can be transferred back:

      +

      If the files are processed into another file, the resulting file can be transferred back:

        echo This is input_file > input_file
-  parallel -S $SERVER1 --transfer --return {}.out cat {} ">"{}.out ::: input_file 
+  parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file 
   cat input_file.out

      Output: Same as above.

      -

      To remove the input and output file on the remote server use --cleanup:

      +

      To remove the input and output file on the remote server use --cleanup:

        echo This is input_file > input_file
-  parallel -S $SERVER1 --transfer --return {}.out --cleanup cat {} ">"{}.out ::: input_file 
+  parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file 
   cat input_file.out

      Output: Same as above.

      -

      There is a short hand for --transfer --return --cleanup called --trc:

      +

      There is a shorthand for --transferfile {} --return --cleanup called --trc:

        echo This is input_file > input_file
   parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file 
@@ -1487,7 +1645,7 @@
 
 
      Output: Same as above.
      
 
-
      Some jobs need a common database for all jobs. GNU Parallel can transfer that using --basefile which will transfer the file before the first job:
      
+
      Some jobs need a common database for all jobs. GNU parallel can transfer that using --basefile which will transfer the file before the first job:
      
 
 
        echo common data > common_file
   parallel --basefile common_file -S $SERVER1 cat common_file\; echo {} ::: foo
      
@@ -1497,17 +1655,17 @@
 
        common data
   foo
      
 
-
      To remove it from the remote host after the last job use --cleanup.
      
+
      To remove it from the remote host after the last job use --cleanup.
      
 
 
      Working dir
      
 
-
      The default working dir on the remote machines is the login dir. This can be changed with --workdir mydir.
      
+
      The default working dir on the remote machines is the login dir. This can be changed with --workdir mydir.
      
 
-
      Files transferred using --transfer and --return will be relative to mydir on remote computers, and the command will be executed in the dir mydir.
      
+
      Files transferred using --transferfile and --return will be relative to mydir on remote computers, and the command will be executed in the dir mydir.
      
 
-
      The special mydir value ... will create working dirs under ~/.parallel/tmp/ on the remote computers. If --cleanup is given these dirs will be removed.
      
+
      The special mydir value ... will create working dirs under ~/.parallel/tmp on the remote computers. If --cleanup is given these dirs will be removed.
      
 
-
      The special mydir value . uses the current working dir. If the current working dir is beneath your home dir, the value . is treated as the relative path to your home dir. This means that if your home dir is different on remote computers (e.g. if your login is different) the relative path will still be relative to your home dir.
      
+
      The special mydir value . uses the current working dir. If the current working dir is beneath your home dir, the value . is treated as the relative path to your home dir. This means that if your home dir is different on remote computers (e.g. if your login is different) the relative path will still be relative to your home dir.
      
 
 
        parallel -S $SERVER1 pwd ::: ""
   parallel --workdir . -S $SERVER1 pwd ::: ""
@@ -1521,7 +1679,7 @@
 
 
      Avoid overloading sshd
      
 
-
      If many jobs are started on the same server, sshd can be overloaded. GNU Parallel can insert a delay between each job run on the same server:
      
+
      If many jobs are started on the same server, sshd can be overloaded. GNU parallel can insert a delay between each job run on the same server:
      
 
 
        parallel -S $SERVER1 --sshdelay 0.2 echo ::: 1 2 3
      
 
@@ -1531,7 +1689,7 @@
   2
   3
      
 
-
      Sshd will be less overloaded if using --controlmaster, which will multiplex ssh connections:
      
+
      sshd will be less overloaded if using --controlmaster, which will multiplex ssh connections:
      
 
 
        parallel --controlmaster -S $SERVER1 echo ::: 1 2 3
      
 
@@ -1539,7 +1697,7 @@
 
 
      Ignore hosts that are down
      
 
-
      In clusters with many hosts a few of the are often down. GNU Parallel can ignore those hosts. In this case the host 173.194.32.46 is down:
      
+
      In clusters with many hosts a few of them are often down. GNU parallel can ignore those hosts. In this case the host 173.194.32.46 is down:
      
 
 
        parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar 
      
 
@@ -1549,7 +1707,7 @@
 
 
      Running the same commands on all hosts
      
 
-
      GNU Parallel can run the same command on all the hosts:
      
+
      GNU parallel can run the same command on all the hosts:
      
 
 
        parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar
      
 
@@ -1560,7 +1718,7 @@
   foo
   bar
      -

      Often you will just want to run a single command on all hosts with out arguments. --nonall is a no argument --onall:

      +

      Often you will just want to run a single command on all hosts with out arguments. --nonall is a no argument --onall:

        parallel --nonall -S $SERVER1,$SERVER2 echo foo bar
      @@ -1569,7 +1727,7 @@ 
        foo bar
   foo bar
      -

      When --tag is used with --nonall and --onall the --tagstring is the host:

      +

      When --tag is used with --nonall and --onall the --tagstring is the host:

        parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
      @@ -1578,11 +1736,11 @@ 
        $SERVER1 foo bar
   $SERVER2 foo bar
      -

      --jobs sets the number of servers to log in to in parallel.

      +

      --jobs sets the number of servers to log in to in parallel.

      -

      Transfer environment variables and functions

      +

      Transferring environment variables and functions

      -

      Using --env GNU Parallel can transfer an environment variable to the remote system.

      +

      Using --env GNU parallel can transfer an environment variable to the remote system.

        MYVAR='foo bar'
   export MYVAR
@@ -1592,7 +1750,7 @@
 
 
        foo bar baz
      
 
-
      This works for functions too if your shell is Bash:
      
+
      This works for functions, too, if your shell is Bash:
      
 
 
        # This only works in Bash
   my_func() {
@@ -1605,7 +1763,7 @@
 
 
        in my_func baz
      
 
-
      GNU Parallel can copy all defined variables and functions to the remote system. It just needs to record which ones to ignore in ~/.parallel/ignored_vars. Do that by running this once:
      
+
      GNU parallel can copy all defined variables and functions to the remote system. It just needs to record which ones to ignore in ~/.parallel/ignored_vars. Do that by running this once:
      
 
 
        parallel --record-env
   cat ~/.parallel/ignored_vars
      
@@ -1614,7 +1772,7 @@
 
 
        [list of variables to ignore - including $PATH and $HOME]
      
 
-
      Now all new variables and functions defined will be copied when using --env _:
      
+
      Now all new variables and functions defined will be copied when using --env _:
      
 
 
        # The function is only copied if using Bash
   my_func2() {
@@ -1633,21 +1791,25 @@
 
 
      Showing what is actually run
      
 
-
      --verbose will show the command that would be run on the local machine. When a job is run on a remote machine this is wrapped with ssh and possibly transferring files and environment variables, setting the workdir, and setting --nice value. -vv shows all of this.
      
+
      --verbose will show the command that would be run on the local machine. When a job is run on a remote machine, this is wrapped with ssh and possibly transferring files and environment variables, setting the workdir, and setting --nice value. -vv shows all of this.
      
 
 
        parallel -vv -S $SERVER1 echo ::: bar
      
 
 
      Output:
      
 
-
        ssh lo exec perl\ -e\ \\\$ENV\\\{\\\"PARALLEL_PID\\\"\\\}=\\\"2554030\\\"\\\;\
-  \\$ENV\\\{\\\"PARALLEL_SEQ\\\"\\\}=\\\"1\\\"\\\;\\\$bashfunc\\\ =\\\ \\\"\\\"\
-  \\;@ARGV=\\\"echo\\\ bar\\\"\\\;\\\$SIG\\\{CHLD\\\}=sub\\\{\\\$done=1\\\;\\\}\
-  \\;\\\$pid=fork\\\;unless\\\(\\\$pid\\\)\\\{setpgrp\\\;exec\\\$ENV\\\{SHELL\\\
-  },\\\"-c\\\",\\\(\\\$bashfunc.\\\"@ARGV\\\"\\\)\\\;die\\\"exec:\\\$\\\!\\\\n\\
-  \"\\\;\\\}do\\\{\\\$s=\\\$s\\\<1\\\?0.001+\\\$s\\\*1.03:\\\$s\\\;select\\\(
-  undef,undef,undef,\\\$s\\\)\\\;\\\}until\\\(\\\$done\\\|\\\|getppid==1\\\)\\\;
-  kill\\\(SIGHUP,-\\\$\\\{pid\\\}\\\)unless\\\$done\\\;wait\\\;exit\\\(\\\$\\\?\
-  \\&127\\\?128+\\\(\\\$\\\?\\\&127\\\):1+\\\$\\\?\\\>\\\>8\\\);
+
        ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
+  eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' 
+  JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
+  IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
+  TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
+  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
+  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
+  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
+  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
+  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
+  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
+  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
   bar
      
 
 
      When the command gets more complex, the output is so hard to read, that it is only useful for debugging:
      
@@ -1660,37 +1822,104 @@
 
 
      Output will be similar to:
      
 
-
        ( ssh lo mkdir -p ./.parallel/tmp/aspire-2554425-1;rsync --protocol 30 -rlDzR 
-  -essh ./abc-file lo:./.parallel/tmp/aspire-2554425-1 );ssh lo exec perl -e \''
-  @GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";
-  $SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my(
-  $in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){
-  close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(
-  join"",@ARGV));close$in;exit;}wait;eval$eval;'\' QlpoOTFBWSZTWayP388AAbdfgAAQd
-  X/+3//l/wS/7//vQAIq3U6bauIpk1NPUwnqGgND1NGI9TTQ0A0ADIDQNITU9NGqfonpPJRvU0ZQZAA
-  GhpoBoABpqaFNBMh+kGoybUaHogwCYBAADCSgRNqek1TxTeknqemhGyBqek8pk2jRPSeo002mQRo5f
-  oSZYrgzQFDd3HNWaskbx+MxNR89BdDzESFbADOJkI+QhIlnojHCWRVuGc2j2lzMzE41wC7auAMQ06c
-  S3AlqQfKcdo0gd506U0HzAAxMkGJBHjDCZULOMpVbowhIVxxaQz7yansTsBgurEZaGO/6K0Nc4iodr
-  BW4m9SXErqRbLNy5eANDvZ+TIt2c2GBcWSlmYuloxY5u2bGUdU/dGsO5EhyrvKCpZMhIgmQFAQhcwR
-  mD+jMKRawkRFJSGyTNC3PqWnE51ucPyx29Yxjnkyub98lytpyk+v8BUc4eA3xz98dMYjxvb0pgWksh
-  oHZ7HwGQRq1vuDyzKgkwPL9lwGIdL+WPNJFSljlVAahIhQpDCAOJpTqDhgmfoRQcy54PC9T0T3iMnV
-  JeTUdL8P0/s18NqDSUavMNV3qD0CtYi6entl0neNsOQN2VDSLHj0xOMls65LNPo+Wh28rJtVoh2JgE
-  7Q9Qo/XBr6krGIsYpQR6nRDuJCD/5aaQBBFFQGtv2VoFTwkXiUTxFP1CC4AGBznAaMklgWQvVtKguJ
-  zQnPqr9ABtHwbB5GTzPOQ4iWAmrUxvl4j5wqrVchOZcs3NYUQmGO2+VYBimFVxhGcaxDALMZ6bWEUo
-  yt8eC8W5o1ObFtTnHAvjOQgYEL/nHTcxU0G57QMKCzJcASQWFNpe2CpQcgYlBxIN4kwtfxdyRThQkK
-  yP388;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ 
-  ./.parallel/tmp/aspire-2554425-1/./.\;\ rsync -rlDzR -essh lo:./abc-file.out 
-  ./.;ssh lo \(rm\ -f\ ./.parallel/tmp/aspire-2554425-1/abc-file\;\ sh\ -c\ \'
-  rmdir\ ./.parallel/tmp/aspire-2554425-1/\ ./.parallel/tmp/\ ./.parallel/\ 2\>
-  /dev/null\'\;rm\ -rf\ ./.parallel/tmp/aspire-2554425-1\;\);ssh lo \(rm\ -f\ 
-  ./.parallel/tmp/aspire-2554425-1/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.parallel
-  /tmp/aspire-2554425-1/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ 
-  -rf\ ./.parallel/tmp/aspire-2554425-1\;\);ssh lo rm -rf .parallel/tmp/
-  aspire-2554425-1; exit $_EXIT_status;
      
+
        ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
+  -rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
+  exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
+  eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\'
+  c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
+  OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
+  YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
+  KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
+  U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
+  VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
+  cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
+  c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
+  RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
+  PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
+  RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
+  RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
+  fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
+  IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
+  eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
+  fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
+  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
+  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
+  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
+  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
+  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
+  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
+  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
+  mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
+  ./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
+  lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
+  ./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \'rmdir\
+  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
+  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
+  \(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \'rmdir\
+  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
+  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
+  -rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;
      
+
+
      Saving to an SQL base (advanced)
      
+
+
      GNU parallel can save into an SQL base. Point GNU parallel to a table and it will put the joblog there together with the variables and the outout each in their own column.
      
+
+
      GNU parallel uses a DBURL to address the table. A DBURL has this format:
      
+
+
        vendor://[[user][:password]@][host][:port]/[database[/table]
      
+
+
      Example:
      
+
+
        mysql://scott:tiger@my.example.com/mydatabase/mytable
+  postgresql://scott:tiger@pg.example.com/mydatabase/mytable
+  sqlite3:///%2Ftmp%2Fmydatabase/mytable
      
+
+
      To refer to /tmp/mydatabase with sqlite you need to encode the / as %2F.
      
+
+
      Run a job using sqlite on mytable in /tmp/mydatabase:
      
+
+
        DBURL=sqlite3:///%2Ftmp%2Fmydatabase
+  DBURLTABLE=$DBURL/mytable
+  parallel --sqlandworker $DBURLTABLE echo ::: foo bar ::: baz quuz
      
+
+
      To see the result:
      
+
+
        sql $DBURL 'SELECT * FROM mytable ORDER BY Seq;'
      
+
+
      Output will be similar to:
      
+
+
        Seq|Host|Starttime|JobRuntime|Send|Receive|Exitval|_Signal|Command|V1|V2|Stdout|Stderr
+  1|:|1451619638.903|0.806||8|0|0|echo foo baz|foo|baz|foo baz
+  |
+  2|:|1451619639.265|1.54||9|0|0|echo foo quuz|foo|quuz|foo quuz
+  |
+  3|:|1451619640.378|1.43||8|0|0|echo bar baz|bar|baz|bar baz
+  |
+  4|:|1451619641.473|0.958||9|0|0|echo bar quuz|bar|quuz|bar quuz
+  |
      
+
+
      The first columns are well known from --joblog. V1 and V2 are data from the input sources. Stdout and Stderr are standard output and standard error, respectively.
      
+
+
      Using multiple workers
      
+
+
      Using an SQL base as storage costs a lot of performance.
      
+
+
      One of the situations where it makes sense is if you have multiple workers.
      
+
+
      You can then have a single master machine that submits jobs to the SQL base (but does not do any of the work):
      
+
+
        parallel --sql $DBURLTABLE echo ::: foo bar ::: baz quuz
      
+
+
      On the worker machines you run exactly the same command except you replace --sql with --sqlworker.
      
+
+
        parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz
      
+
+
      To run a master and a worker on the same machine use --sqlandworker as shown earlier.
      
 
 
      --pipe
      
 
-
      The --pipe functionality puts GNU Parallel in a different mode: Instead of treating the data on stdin (standard input) as arguments for a command to run, the data will be sent to stdin (standard input) of the command.
      
+
      The --pipe functionality puts GNU parallel in a different mode: Instead of treating the data on stdin (standard input) as arguments for a command to run, the data will be sent to stdin (standard input) of the command.
      
 
 
      The typical situation is:
      
 
@@ -1700,7 +1929,7 @@
 
 
      Chunk size
      
 
-
      By default GNU Parallel will start an instance of command_B, read a chunk of 1 MB, and pass that to the instance. Then start another instance, read another chunk, and pass that to the second instance.
      
+
      By default GNU parallel will start an instance of command_B, read a chunk of 1 MB, and pass that to the instance. Then start another instance, read another chunk, and pass that to the second instance.
      
 
 
        cat num1000000 | parallel --pipe wc
      
 
@@ -1714,7 +1943,7 @@
   149796  149796 1048572
    85349   85349  597444
      
 
-
      The size of the chunk is not exactly 1 MB because GNU Parallel only passes full lines - never half a line, thus the blocksize is only average 1 MB. You can change the block size to 2 MB with --block:
      
+
      The size of the chunk is not exactly 1 MB because GNU parallel only passes full lines - never half a line, thus the blocksize is only average 1 MB. You can change the block size to 2 MB with --block:
      
 
 
        cat num1000000 | parallel --pipe --block 2M wc
      
 
@@ -1725,7 +1954,7 @@
   299593  299593 2097151
    85349   85349  597444
      
 
-
      GNU Parallel treats each line as a record. If the order of record is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use --round-robin. Without --round-robin GNU Parallel will start a command per block; with --round-robin only the requested number of jobs will be started (--jobs). The records will then be distributed between the running jobs:
      
+
      GNU parallel treats each line as a record. If the order of record is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use --round-robin. Without --round-robin GNU parallel will start a command per block; with --round-robin only the requested number of jobs will be started (--jobs). The records will then be distributed between the running jobs:
      
 
 
        cat num1000000 | parallel --pipe -j4 --round-robin wc
      
 
@@ -1740,9 +1969,9 @@
 
 
      Records
      
 
-
      GNU Parallel sees the input as records. The default record is a single line.
      
+
      GNU parallel sees the input as records. The default record is a single line.
      
 
-
      Using -N140000 GNU Parallel will read 140000 records at a time:
      
+
      Using -N140000 GNU parallel will read 140000 records at a time:
      
 
 
        cat num1000000 | parallel --pipe -N140000 wc
      
 
@@ -1759,7 +1988,7 @@
 
 
      Notice that the last job could not get the full 140000 lines, but only 20000 lines.
      
 
-
      If a record is 75 lines -L can be used:
      
+
      If a record is 75 lines -L can be used:
      
 
 
        cat num1000000 | parallel --pipe -L75 wc
      
 
@@ -1774,17 +2003,17 @@
    85350   85350  597450
       25      25     176
      
 
-
      Notice GNU Parallel still reads a block of around 1 MB; but instead of passing full lines to 'wc' it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).
      
+
      Notice GNU parallel still reads a block of around 1 MB; but instead of passing full lines to wc it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).
      
 
 
      Record separators
      
 
-
      GNU Parallel uses separators to determine where two records split.
      
+
      GNU parallel uses separators to determine where two records split.
      
 
-
      --recstart gives the string that starts a record; --recend gives the string that ends a record. The default is --recend '\n' (newline).
      
+
      --recstart gives the string that starts a record; --recend gives the string that ends a record. The default is --recend '\n' (newline).
      
 
-
      If both --recend and --recstart are given, then the record will only split if the recend string is immediately followed by the recstart string.
      
+
      If both --recend and --recstart are given, then the record will only split if the recend string is immediately followed by the recstart string.
      
 
-
      Here the --recend is set to ', ':
      
+
      Here the --recend is set to ', ':
      
 
 
        echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
      
 
@@ -1800,9 +2029,9 @@
   qux/,
   END
      -

      Here the --recstart is set to '/':

      +

      Here the --recstart is set to /:

      -
        echo /foo, bar/, /baz, qux/, | parallel -kN1 --recstart '/' --pipe echo JOB{#}\;cat\;echo END
      +
        echo /foo, bar/, /baz, qux/, | parallel -kN1 --recstart / --pipe echo JOB{#}\;cat\;echo END

      Output:

      @@ -1816,9 +2045,9 @@ /, END -

      Here both --recend and --recstart are set:

      +

      Here both --recend and --recstart are set:

      -
        echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --recstart '/' --pipe echo JOB{#}\;cat\;echo END
      +
        echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --recstart / --pipe echo JOB{#}\;cat\;echo END

      Output:

      @@ -1830,9 +2059,9 @@

      Note the difference between setting one string and setting both strings.

      -

      With --regexp the --recend and --recstart will be treated as a regular expression:

      +

      With --regexp the --recend and --recstart will be treated as a regular expression:

      -
        echo foo,bar,_baz,__qux, | parallel -kN1 --regexp --recend ',_+' --pipe echo JOB{#}\;cat\;echo END
      +
        echo foo,bar,_baz,__qux, | parallel -kN1 --regexp --recend ,_+ --pipe echo JOB{#}\;cat\;echo END

      Output:

      @@ -1844,9 +2073,9 @@ qux, END -

      GNU Parallel can remove the record separators with --remove-rec-sep/--rrs:

      +

      GNU parallel can remove the record separators with --remove-rec-sep/--rrs:

      -
        echo foo,bar,_baz,__qux, | parallel -kN1 --rrs --regexp --recend ',_+' --pipe echo JOB{#}\;cat\;echo END
      +
        echo foo,bar,_baz,__qux, | parallel -kN1 --rrs --regexp --recend ,_+ --pipe echo JOB{#}\;cat\;echo END

      Output:

      @@ -1860,7 +2089,7 @@ -

      If the input data has a header, the header can be repeated for each job by matching the header with --header. If headers start with %:

      +

      If the input data has a header, the header can be repeated for each job by matching the header with --header. If headers start with % you can do this:

        cat num_%header | parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat
      @@ -1889,7 +2118,7 @@ %head2 10 -

      If the header is 2 lines, --header 2 will work:

      +

      If the header is 2 lines, --header 2 will work:

        cat num_%header | parallel --header 2 --pipe -N3 echo JOB{#}\;cat
      @@ -1897,7 +2126,7 @@

      --pipepart

      -

      --pipe is not very efficient. It maxes out at around 500 MB/s. --pipepart can easily deliver 5 GB/s. But there are a few limitations. The input has to be a normal file (not a pipe) given by -a or :::: and -L/-l/-N do not work.

      +

      --pipe is not very efficient. It maxes out at around 500 MB/s. --pipepart can easily deliver 5 GB/s. But there are a few limitations. The input has to be a normal file (not a pipe) given by -a or :::: and -L/-l/-N do not work.

        parallel --pipepart -a num1000000 --block 3m wc
      @@ -1911,17 +2140,17 @@

      Input data and parallel command in the same file

      -

      GNU Parallel is often called as:

      +

      GNU parallel is often called as this:

        cat input_file | parallel command
      -

      With --shebang the input_file and parallel can be combined into the same script.

      +

      With --shebang the input_file and parallel can be combined into the same script.

      -

      UNIX-scripts start with a shebang line like:

      +

      UNIX-scripts start with a shebang line like this:

        #!/bin/bash
      -

      GNU Parallel can do that, too. With --shebang the arguments can be listed in the file. The parallel command is the first line of the script:

      +

      GNU parallel can do that, too. With --shebang the arguments can be listed in the file. The parallel command is the first line of the script:

        #!/usr/bin/parallel --shebang -r echo
 
@@ -1937,19 +2166,19 @@
 
 
      Parallelizing existing scripts
      
 
-
      GNU Parallel is often called as:
      
+
      GNU parallel is often called as:
      
 
 
        cat input_file | parallel command
   parallel command ::: foo bar
      
 
-
      If command is a script parallel can be combined into a single file so:
      
+
      If command is a script parallel can be combined into a single file so:
      
 
 
        cat input_file | command
   command foo bar
      
 
-
      will run the script in parallel.
      
+
      will run the script in parallel.
      
 
-
      This perl script perl_echo works like echo:
      
+
      This perl script perl_echo works like echo:
      
 
 
        #!/usr/bin/perl
 
@@ -1959,7 +2188,7 @@
 
 
        parallel perl_echo ::: foo bar
      
 
-
      By changing the #!-line it can be run in parallel
      
+
      By changing the #!-line it can be run in parallel:
      
 
 
        #!/usr/bin/parallel --shebang-wrap /usr/bin/perl
 
@@ -1983,56 +2212,92 @@
 
 
      #!/usr/bin/parallel --shebang-wrap /usr/bin/perl
      
 
+
      print "Arguments @ARGV\n";
      
+
 
 
      Python:
      
 
      
 
 
      #!/usr/bin/parallel --shebang-wrap /usr/bin/python
      
 
+
      import sys
      
+
+
      print 'Arguments', str(sys.argv)
      
+
 
      
 
      Bash:
      
 
      
 
 
      #!/usr/bin/parallel --shebang-wrap /bin/bash
      
 
+
      echo Arguments "$@"
      
+
 
      
 
      R:
      
 
      
 
 
      #!/usr/bin/parallel --shebang-wrap /usr/bin/Rscript --vanilla --slave
      
 
+
      args <- commandArgs(trailingOnly = TRUE) print(paste("Arguments ",args))
      
+
 
      
 
      GNUplot:
      
 
      
 
 
      #!/usr/bin/parallel --shebang-wrap ARG={} /usr/bin/gnuplot
      
 
+
      print "Arguments ", system('echo $ARG')
      
+
 
      
 
      Ruby:
      
 
      
 
 
      #!/usr/bin/parallel --shebang-wrap /usr/bin/ruby
      
 
+
      print "Arguments " puts ARGV
      
+
+
      
+
      Octave:
      
+
      
+
+
      #!/usr/bin/parallel --shebang-wrap /usr/bin/octave
      
+
+
      printf ("Arguments"); arg_list = argv (); for i = 1:nargin printf (" %s", arg_list{i}); endfor printf ("\n");
      
+
+
      
+
      Common LISP:
      
+
      
+
+
      #!/usr/bin/parallel --shebang-wrap /usr/bin/clisp
      
+
+
      (format t "~&~S~&" 'Arguments) (format t "~&~S~&" *args*)
      
+
+
      LUA PHP Javascript nodejs Tcl C#?
      
+
 
      
 
 
 
      Semaphore
      
 
-
      GNU Parallel can work as a counting semaphore. This is slower and less efficient than its normal mode.
      
+
      GNU parallel can work as a counting semaphore. This is slower and less efficient than its normal mode.
      
 
 
      A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to be available.
      
 
-
      An alias for 'parallel --semaphore' is 'sem'.
      
+
      An alias for parallel --semaphore is sem.
      
 
-
      'sem' will follow a person to the toilets, wait until a toilet is available, leave the person in the toilet and exit.
      
+
      sem will follow a person to the toilets, wait until a toilet is available, leave the person in the toilet and exit.
      
 
-
      'sem --fg' will follow a person to the toilets, wait until a toilet is available, stay with the person in the toilet and exit when the person exits.
      
+
      sem --fg will follow a person to the toilets, wait until a toilet is available, stay with the person in the toilet and exit when the person exits.
      
 
-
      'sem --wait' will wait for all persons to leave the toilets.
      
+
      sem --wait will wait for all persons to leave the toilets.
      
 
-
      'sem' does not have a queue discipline, so the next person is chosen randomly.
      
+
      sem does not have a queue discipline, so the next person is chosen randomly.
      
 
-
      -j sets the number of toilets. The default is to have only one toilet (technically this is called a mutex). The program is started in the background and 'sem' exits immediately. Use --wait to wait for all 'sem's to finish:
      
+
      -j sets the number of toilets.
      
+
+
      Mutex
      
+
+
      The default is to have only one toilet (this is called a mutex). The program is started in the background and sem exits immediately. Use --wait to wait for all sems to finish:
      
 
 
        sem 'sleep 1; echo The first finished' &&
     echo The first is now running in the background &&
@@ -2047,7 +2312,7 @@
   The second is now running in the background
   The second finished
      
 
-
      The command can be run in the foreground with --fg, which will only exit when the command completes:
      
+
      The command can be run in the foreground with --fg, which will only exit when the command completes:
      
 
 
        sem --fg 'sleep 1; echo The first finished' &&
     echo The first finished running in the foreground &&
@@ -2055,9 +2320,9 @@
     echo The second finished running in the foreground
   sem --wait
      
 
-
      The difference between this and just running the command, is that a mutex is set, so if other sems were running in the background only one would run at the same time.
      
+
      The difference between this and just running the command, is that a mutex is set, so if other sems were running in the background only one would run at a time.
      
 
-
      To tell the difference between which semaphore is used, use --semaphorename/--id. Run this in one terminal:
      
+
      To tell the difference between which semaphore is used, use --semaphorename/--id. Run this in one terminal:
      
 
 
        sem --id my_id -u 'echo First started; sleep 10; echo The first finished'
      
 
@@ -2071,7 +2336,7 @@
 
 
      A mutex is like having a single toilet: When it is in use everyone else will have to wait. A counting semaphore is like having multiple toilets: Several people can use the toilets, but when they all are in use, everyone else will have to wait.
      
 
-
      sem can emulate a counting semaphore. Use --jobs to set the number of toilets:
      
+
      sem can emulate a counting semaphore. Use --jobs to set the number of toilets like this:
      
 
 
        sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' &&
   sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' &&
@@ -2092,7 +2357,7 @@
 
 
      Timeout
      
 
-
      With --semaphoretimeout you can force running the command anyway after a period (postive number) or give up (negative number):
      
+
      With --semaphoretimeout you can force running the command anyway after a period (postive number) or give up (negative number):
      
 
 
        sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' &&
   sem --id foo --semaphoretimeout 1 'echo Force this running after 1 sec' &&
@@ -2111,9 +2376,9 @@
 
 
      Informational
      
 
-
      GNU Parallel has some options to give short information about the configuration.
      
+
      GNU parallel has some options to give short information about the configuration.
      
 
-
      --help will print a summary of the most important options:
      
+
      --help will print a summary of the most important options:
      
 
 
        parallel --help
      
 
@@ -2146,11 +2411,11 @@
   When using GNU Parallel for a publication please cite:
   
   O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
      
+  ;login: The USENIX Magazine, February 2011:42-47.
 
-
      When asking for help, always report the full output of:
      
+    When asking for help, always report the full output of this:
 
-
        parallel --version
      
+  parallel --version
      
 
 
      Output:
      
 
@@ -2167,7 +2432,7 @@
   O. Tange (2011): GNU Parallel - The Command-Line Power Tool, 
   ;login: The USENIX Magazine, February 2011:42-47.
      
 
-
      In scripts --minversion can be used to ensure the user has at least this version:
      
+
      In scripts --minversion can be used to ensure the user has at least this version:
      
 
 
        parallel --minversion 20130722 && echo Your version is at least 20130722.
      
 
@@ -2176,7 +2441,7 @@
 
        20130722
   Your version is at least 20130722.
      
 
-
      If using GNU Parallel for research the BibTeX citation can be generated using --bibtex.
      
+
      If using GNU parallel for research the BibTeX citation can be generated using --bibtex:
      
 
 
        parallel --bibtex
      
 
@@ -2195,7 +2460,7 @@
    pages = {42-47}
   }
      
 
-
      With --max-line-length-allowed GNU Parallel will report the maximal size of the command line:
      
+
      With --max-line-length-allowed GNU parallel will report the maximal size of the command line:
      
 
 
        parallel --max-line-length-allowed
      
 
@@ -2203,7 +2468,7 @@
 
 
        131071
      
 
-
      --number-of-cpus and --number-of-cores run system specific code to determine the number of CPUs and CPU cores on the system. On unsupported platforms they will return 1:
      
+
      --number-of-cpus and --number-of-cores run system specific code to determine the number of CPUs and CPU cores on the system. On unsupported platforms they will return 1:
      
 
 
        parallel --number-of-cpus 
   parallel --number-of-cores
      
@@ -2215,9 +2480,9 @@
 
 
      Profiles
      
 
-
      The defaults for GNU Parallel can be changed systemwise by putting the command line options in /etc/parallel/config. They can be changed for a user by putting them in ~/.parallel/config.
      
+
      The defaults for GNU parallel can be changed systemwide by putting the command line options in /etc/parallel/config. They can be changed for a user by putting them in ~/.parallel/config.
      
 
-
      Profiles work the same way, but have to be referred to with --profile:
      
+
      Profiles work the same way, but have to be referred to with --profile:
      
 
 
        echo '--nice 17' > ~/.parallel/nicetimeout
   echo '--timeout 300%' >> ~/.parallel/nicetimeout
@@ -2244,7 +2509,7 @@
 
 
      I hope you have learned something from this tutorial.
      
 
-
      If you like GNU Parallel:
      
+
      If you like GNU parallel:
      
 
 
        
 
@@ -2257,7 +2522,7 @@
 
      • Post the intro videos and the tutorial on Reddit, Diaspora*, forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, mailing lists
        
 
 
        • 
-
      • Request or write a review for your favourite blog or magazine
        
+
      • Request or write a review for your favourite blog or magazine (especially if you do something cool with GNU parallel)
        
 
 
        • 
 
      • Invite me for your next conference
        
@@ -2265,16 +2530,16 @@
 
        • 
 
      
 
-
      If you use GNU Parallel for research:
      
+
      If you use GNU parallel for research:
      
 
 
        
 
-
      • Please cite GNU Parallel in you publications (use --bibtex)
        
+
      • Please cite GNU parallel in you publications (use --bibtex)
        
 
 
        • 
 
      
 
-
      If GNU Parallel saves you money:
      
+
      If GNU parallel saves you money:
      
 
 
        
 
@@ -2283,7 +2548,7 @@
 
 
      
 
-
      (C) 2013,2014,2015 Ole Tange, GPLv3
      
+
      (C) 2013,2014,2015,2016 Ole Tange, GPLv3
      
 
 
 
diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod
index a0af2232..e2894562 100644
--- a/src/parallel_tutorial.pod
+++ b/src/parallel_tutorial.pod
@@ -2,8 +2,8 @@
 
 =head1 GNU Parallel Tutorial
 
-This tutorial shows off much of GNU Parallel's functionality. The
-tutorial is meant to learn the options in GNU Parallel.  The tutorial
+This tutorial shows off much of GNU B's functionality. The
+tutorial is meant to learn the options in GNU B.  The tutorial
 is not to show realistic examples from the real world.
 
 Spend an hour walking through the tutorial. Your command line will
@@ -107,7 +107,7 @@ and using an empty pass phrase.
 
 =head1 Input sources
 
-GNU Parallel reads input from input sources. These can be files, the
+GNU B reads input from input sources. These can be files, the
 command line, and stdin (standard input or a pipe).
 
 =head2 A single input source
@@ -138,8 +138,8 @@ Output: Same as above.
 
 =head2 Multiple input sources
 
-GNU Parallel can take multiple input sources given on the command
-line. GNU Parallel then generates all combinations of the input
+GNU B can take multiple input sources given on the command
+line. GNU B then generates all combinations of the input
 sources:
 
   parallel echo ::: A B C ::: D E F
@@ -162,13 +162,13 @@ The input sources can be files:
 
 Output: Same as above.
 
-STDIN (standard input) can be one of the input sources using '-':
+STDIN (standard input) can be one of the input sources using B<->:
 
   cat abc-file | parallel -a - -a def-file echo 
 
 Output: Same as above.
 
-Instead of -a files can be given after '::::':
+Instead of B<-a> files can be given after B<::::>:
 
   cat abc-file | parallel echo :::: - def-file
 
@@ -182,7 +182,7 @@ Output: Same as above.
 
 =head3 Matching arguments from all input sources
 
-With --xapply you can get one argument from each input source:
+With B<--xapply> you can get one argument from each input source:
 
   parallel --xapply echo ::: A B C ::: D E F
 
@@ -206,8 +206,8 @@ Output (the order may be different):
 
 =head2 Changing the argument separator.
 
-GNU Parallel can use other separators than ::: or ::::. This is
-typically useful if ::: or :::: is used in the command to run:
+GNU B can use other separators than B<:::> or B<::::>. This is
+typically useful if B<:::> or B<::::> is used in the command to run:
 
   parallel --arg-sep ,, echo ,, A B C :::: def-file
 
@@ -232,8 +232,8 @@ Output: Same as above.
 
 =head2 Changing the argument delimiter
 
-GNU Parallel will normally treat a full line as a single argument: It
-uses \n as argument delimiter. This can be changed with -d:
+GNU B will normally treat a full line as a single argument: It
+uses B<\n> as argument delimiter. This can be changed with B<-d>:
 
   parallel -d _ echo :::: abc_-file
 
@@ -243,14 +243,14 @@ Output (the order may be different):
   B
   C
 
-NULL can be given as \0:
+NULL can be given as B<\0>:
 
   parallel -d '\0' echo :::: abc0-file
 
 Output: Same as above.
 
-A shorthand for -d '\0' is -0 (this will often be used to read files
-from find ... -print0):
+A shorthand for B<-d '\0'> is B<-0> (this will often be used to read files
+from B):
 
   parallel -0 echo :::: abc0-file
 
@@ -258,7 +258,7 @@ Output: Same as above.
 
 =head2 End-of-file value for input source
 
-GNU Parallel can stop reading when it encounters a certain value:
+GNU B can stop reading when it encounters a certain value:
 
   parallel -E stop echo ::: A B stop C D
 
@@ -269,7 +269,7 @@ Output:
 
 =head2 Skipping empty lines
 
-Using --no-run-if-empty GNU Parallel will skip empty lines.
+Using B<--no-run-if-empty> GNU B will skip empty lines.
 
   (echo 1; echo; echo 2) | parallel --no-run-if-empty echo
 
@@ -295,7 +295,7 @@ Output (the order may be different):
   [/path/to/current/working/dir]
 
 The command can be a script, a binary or a Bash function if the function is
-exported using 'export -f':
+exported using B:
 
   # Only works in Bash
   my_func() {
@@ -314,8 +314,8 @@ Output (the order may be different):
 
 =head3 The 7 predefined replacement strings
 
-GNU Parallel has several replacement strings. If no replacement
-strings are used the default is to append {}:
+GNU B has several replacement strings. If no replacement
+strings are used the default is to append B<{}>:
 
   parallel echo ::: A/B.C
 
@@ -323,7 +323,7 @@ Output:
 
   A/B.C
 
-The default replacement string is {}:
+The default replacement string is B<{}>:
 
   parallel echo {} ::: A/B.C
 
@@ -331,7 +331,7 @@ Output:
 
   A/B.C
 
-The replacement string {.} removes the extension:
+The replacement string B<{.}> removes the extension:
 
   parallel echo {.} ::: A/B.C
 
@@ -339,7 +339,7 @@ Output:
 
   A/B
 
-The replacement string {/} removes the path:
+The replacement string B<{/}> removes the path:
 
   parallel echo {/} ::: A/B.C
 
@@ -347,7 +347,7 @@ Output:
 
   B.C
 
-The replacement string {//} keeps only the path:
+The replacement string B<{//}> keeps only the path:
 
   parallel echo {//} ::: A/B.C
 
@@ -355,7 +355,7 @@ Output:
 
   A
 
-The replacement string {/.} removes the path and the extension:
+The replacement string B<{/.}> removes the path and the extension:
 
   parallel echo {/.} ::: A/B.C
 
@@ -363,7 +363,7 @@ Output:
 
   B
 
-The replacement string {#} gives the job number:
+The replacement string B<{#}> gives the job number:
 
   parallel echo {#} ::: A B C
 
@@ -373,7 +373,7 @@ Output (the order may be different):
   2
   3
 
-The replacement string {%} gives the job slot number (between 1 and
+The replacement string B<{%}> gives the job slot number (between 1 and
 number of jobs to run in parallel):
 
   parallel -j 2 echo {%} ::: A B C
@@ -386,7 +386,7 @@ Output (the order may be different and 1 and 2 may be swapped):
 
 =head3 Changing the replacement strings
 
-The replacement string {} can be changed with -I:
+The replacement string B<{}> can be changed with B<-I>:
 
   parallel -I ,, echo ,, ::: A/B.C
 
@@ -394,7 +394,7 @@ Output:
 
   A/B.C
 
-The replacement string {.} can be changed with --extensionreplace:
+The replacement string B<{.}> can be changed with B<--extensionreplace>:
 
   parallel --extensionreplace ,, echo ,, ::: A/B.C
 
@@ -402,7 +402,7 @@ Output:
 
   A/B
 
-The replacement string {/} can be replaced with --basenamereplace:
+The replacement string B<{/}> can be replaced with B<--basenamereplace>:
 
   parallel --basenamereplace ,, echo ,, ::: A/B.C
 
@@ -410,7 +410,7 @@ Output:
 
   B.C
 
-The replacement string {//} can be changed with --dirnamereplace:
+The replacement string B<{//}> can be changed with B<--dirnamereplace>:
 
   parallel --dirnamereplace ,, echo ,, ::: A/B.C
 
@@ -418,7 +418,7 @@ Output:
 
   A
 
-The replacement string {/.} can be changed with --basenameextensionreplace:
+The replacement string B<{/.}> can be changed with B<--basenameextensionreplace>:
 
   parallel --basenameextensionreplace ,, echo ,, ::: A/B.C
 
@@ -426,7 +426,7 @@ Output:
 
   B
 
-The replacement string {#} can be changed with --seqreplace:
+The replacement string B<{#}> can be changed with B<--seqreplace>:
 
   parallel --seqreplace ,, echo ,, ::: A B C
 
@@ -436,7 +436,7 @@ Output (the order may be different):
   2
   3
 
-The replacement string {%} can be changed with --slotreplace:
+The replacement string B<{%}> can be changed with B<--slotreplace>:
 
   parallel -j2 --slotreplace ,, echo ,, ::: A B C
 
@@ -450,7 +450,7 @@ Output (the order may be different and 1 and 2 may be swapped):
 
 When predefined replacement strings are not flexible enough a perl
 expression can be used instead. One example is to remove two
-extensions: foo.tar.gz -> foo
+extensions: foo.tar.gz becomes foo
 
   parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
 
@@ -458,26 +458,69 @@ Output:
 
   foo
 
-If the strings B<{=> and B<=}> cause problems they can be replaced with --parens:
+In B<{= =}> you can access all of GNU B's internal functions
+and variables. A few are worth mentioning.
+
+B returns the total number of jobs:
+
+  parallel echo Job {#} of {= '$_=total_jobs()' =} ::: {1..5}
+
+Output:
+
+  Job 1 of 5
+  Job 2 of 5
+  Job 3 of 5
+  Job 4 of 5
+  Job 5 of 5
+
+B shell quotes the string:
+
+  parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$'
+
+B<$job->>B skips the job:
+
+  parallel echo {= 'if($_==3) { $job->skip() }' =} ::: {1..5}
+
+Output:
+
+  1
+  2
+  4
+  5
+
+B<@arg> contains the input source variables:
+
+  parallel echo {= 'if($arg[1]==$arg[2]) { $job->skip() }' =} ::: {1..3} ::: {1..3}
+
+Output:
+
+  1 2
+  1 3
+  2 1
+  2 3
+  3 1
+  3 2
+
+If the strings B<{=> and B<=}> cause problems they can be replaced with B<--parens>:
 
   parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz
 
 Output: Same as above.
 
-To define a short hand replacement string use B<--rpl>:
+To define a shorthand replacement string use B<--rpl>:
 
   parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz
 
 Output: Same as above.
 
-If the short hand starts with '{' it can be used as a positional
+If the shorthand starts with B<{> it can be used as a positional
 replacement string, too:
 
   parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz
 
 Output: Same as above.
 
-GNU B's 7 replacement strings are implemented as:
+GNU B's 7 replacement strings are implemented as this:
 
   --rpl '{} '
   --rpl '{#} $_=$job->seq()'
@@ -490,7 +533,7 @@ GNU B's 7 replacement strings are implemented as:
 =head3 Positional replacement strings
 
 With multiple input sources the argument from the individual input
-sources can be access with {number}:
+sources can be accessed with B<{>numberB<}>:
 
   parallel echo {1} and {2} ::: A B ::: C D
 
@@ -501,7 +544,7 @@ Output (the order may be different):
   B and C
   B and D
 
-The positional replacement strings can also be modified using / // /. and  .:
+The positional replacement strings can also be modified using B, B, B, and  B<.>:
 
   parallel echo /={1/} //={1//} /.={1/.} .={1.} ::: A/B.C D/E.F
 
@@ -538,8 +581,8 @@ Output:
 
   foo bar
 
-If a defined short hand starts with '{' it can be used as a positional
-replacement string, too:
+If shorthand defined using B<--rpl> starts with B<{> it can be used as
+a positional replacement string, too:
 
   parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz
 
@@ -549,7 +592,7 @@ Output: Same as above.
 =head3 Input from columns
 
 The columns in a file can be bound to positional replacement strings
-using --colsep. Here the columns are separated with TAB (\t):
+using B<--colsep>. Here the columns are separated by TAB (\t):
 
   parallel --colsep '\t' echo 1={1} 2={2} :::: tsv-file.tsv
 
@@ -561,9 +604,9 @@ Output (the order may be different):
 
 =head3 Header defined replacement strings
 
-With --header GNU Parallel will use the first value of the input
+With B<--header> GNU B will use the first value of the input
 source as the name of the replacement string. Only the non-modified
-version {} is supported:
+version B<{}> is supported:
 
   parallel --header : echo f1={f1} f2={f2} ::: f1 A B ::: f2 C D
 
@@ -574,7 +617,7 @@ Output (the order may be different):
   f1=B f2=C
   f1=B f2=D
 
-It is useful with --colsep for processing files with TAB separated values:
+It is useful with B<--colsep> for processing files with TAB separated values:
 
   parallel --header : --colsep '\t' echo f1={f1} f2={f2} :::: tsv-file.tsv
 
@@ -583,9 +626,41 @@ Output (the order may be different):
   f1=A f2=B
   f1=C f2=D
 
+=head3 More pre-defined replacement strings
+
+B<--plus> adds the replacement strings B<{+/} {+.} {+..} {+...} {..}  {...}
+{/..} {/...} {##}>. The idea being that B<{+foo}> matches the opposite of B<{foo}>
+and B<{}> = B<{+/}>/B<{/}> = B<{.}>.B<{+.}> = B<{+/}>/B<{/.}>.B<{+.}> = B<{..}>.B<{+..}> =
+B<{+/}>/B<{/..}>.B<{+..}> = B<{...}>.B<{+...}> = B<{+/}>/B<{/...}>.B<{+...}>.
+
+  parallel --plus echo {} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {.}.{+.} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/.}.{+.} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {..}.{+..} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/..}.{+..} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {...}.{+...} ::: dir/sub/file.ext1.ext2.ext3
+  parallel --plus echo {+/}/{/...}.{+...} ::: dir/sub/file.ext1.ext2.ext3
+
+Output:
+
+  dir/sub/file.ext1.ext2.ext3
+
+B<{##}> is simply the number of jobs:
+
+  parallel --plus echo Job {#} of {##} ::: {1..5}
+
+Output:
+
+  Job 1 of 5
+  Job 2 of 5
+  Job 3 of 5
+  Job 4 of 5
+  Job 5 of 5
+
 =head2 More than one argument
 
-With --xargs will GNU Parallel fit as many arguments as possible on a
+With B<--xargs> GNU B will fit as many arguments as possible on a
 single line:
 
   cat num30000 | parallel --xargs echo | wc -l
@@ -596,7 +671,7 @@ Output (if you run this under Bash on GNU/Linux):
 
 The 30000 arguments fitted on 2 lines.
 
-The maximal length of a single line can be set with -s. With a maximal
+The maximal length of a single line can be set with B<-s>. With a maximal
 line length of 10000 chars 17 commands will be run:
 
   cat num30000 | parallel --xargs -s 10000 echo | wc -l
@@ -605,15 +680,15 @@ Output:
 
   17
 
-For better parallelism GNU Parallel can distribute the arguments
+For better parallelism GNU B can distribute the arguments
 between all the parallel jobs when end of file is met.
 
-Below GNU Parallel reads the last argument when generating the second
-job. When GNU Parallel reads the last argument, it spreads all the
+Below GNU B reads the last argument when generating the second
+job. When GNU B reads the last argument, it spreads all the
 arguments for the second job over 4 jobs instead, as 4 parallel jobs
 are requested.
 
-The first job will be the same as the --xargs example above, but the
+The first job will be the same as the B<--xargs> example above, but the
 second job will be split into 4 evenly sized jobs, resulting in a
 total of 5 jobs:
 
@@ -635,7 +710,7 @@ Output:
   7 8 9
   10
 
-A replacement string can be part of a word. -m will not repeat the context:
+A replacement string can be part of a word. B<-m> will not repeat the context:
 
   parallel --jobs 4 -m echo pre-{}-post ::: A B C D E F G
 
@@ -646,7 +721,7 @@ Output (the order may be different):
   pre-E F-post
   pre-G-post
 
-To repeat the context use -X which otherwise works like -m:
+To repeat the context use B<-X> which otherwise works like B<-m>:
 
   parallel --jobs 4 -X echo pre-{}-post ::: A B C D E F G
 
@@ -657,7 +732,7 @@ Output (the order may be different):
   pre-E-post pre-F-post
   pre-G-post
 
-To limit the number of arguments use -N:
+To limit the number of arguments use B<-N>:
 
   parallel -N3 echo ::: A B C D E F G H
 
@@ -667,7 +742,7 @@ Output (the order may be different):
   D E F
   G H
 
--N also sets the positional replacement strings:
+B<-N> also sets the positional replacement strings:
 
   parallel -N3 echo 1={1} 2={2} 3={3} ::: A B C D E F G H
 
@@ -677,7 +752,7 @@ Output (the order may be different):
   1=D 2=E 3=F
   1=G 2=H 3=
 
--N0 reads 1 argument but inserts none:
+B<-N0> reads 1 argument but inserts none:
 
   parallel -N0 echo foo ::: 1 2 3
 
@@ -691,7 +766,7 @@ Output:
 
 Command lines that contain special characters may need to be protected from the shell.
 
-The perl program 'print "@ARGV\n"' basically works like echo.
+The B program B basically works like B.
 
   perl -e 'print "@ARGV\n"' A
 
@@ -707,7 +782,7 @@ Output:
 
   [Nothing]
 
-To quote the command use -q:
+To quote the command use B<-q>:
 
   parallel -q perl -e 'print "@ARGV\n"' ::: This works
 
@@ -716,7 +791,7 @@ Output (the order may be different):
   This
   works
 
-Or you can quote the critical part using \':
+Or you can quote the critical part using B<\'>:
 
   parallel perl -e \''print "@ARGV\n"'\' ::: This works, too
 
@@ -726,7 +801,7 @@ Output (the order may be different):
   works,
   too
 
-GNU Parallel can also \-quote full lines. Simply run:
+GNU B can also \-quote full lines. Simply run this:
 
   parallel --shellquote
   parallel: Warning: Input is read from the terminal. Only experts do this on purpose. Press CTRL-D to exit.
@@ -750,7 +825,7 @@ Output (the order may be different):
 
 =head2 Trimming space
 
-Space can be trimmed on the arguments using --trim:
+Space can be trimmed on the arguments using B<--trim>:
 
   parallel --trim r echo pre-{}-post ::: ' A '
 
@@ -787,7 +862,7 @@ Output (the order may be different):
   B       foo-B
   C       foo-C
 
-To prefix it with another string use --tagstring:
+To prefix it with another string use B<--tagstring>:
 
   parallel --tagstring {}-bar echo foo-{} ::: A B C
 
@@ -797,7 +872,7 @@ Output (the order may be different):
   B-bar   foo-B
   C-bar   foo-C
 
-To see what commands will be run without running them:
+To see what commands will be run without running them use B<--dryrun>:
 
   parallel --dryrun echo {} ::: A B C
 
@@ -807,7 +882,7 @@ Output (the order may be different):
   echo B
   echo C
 
-To print the command before running them use --verbose:
+To print the command before running them use B<--verbose>:
 
   parallel --verbose echo {} ::: A B C
 
@@ -820,7 +895,7 @@ Output (the order may be different):
   B
   C
 
-GNU Parallel will postpone the output until the command completes:
+GNU B will postpone the output until the command completes:
 
   parallel -j2 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
 
@@ -836,7 +911,7 @@ Output:
   4-middle
   4-end
 
-To get the output immediately use --ungroup:
+To get the output immediately use B<--ungroup>:
 
   parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
 
@@ -852,11 +927,11 @@ Output:
   -middle
   4-end
 
---ungroup is fast, but can cause half a line from one job to be mixed
+B<--ungroup> is fast, but can cause half a line from one job to be mixed
 with half a line of another job. That has happend in the second line,
 where the line '4-middle' is mixed with '2-start'.
 
-To avoid this use --linebuffer:
+To avoid this use B<--linebuffer>:
 
   parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
 
@@ -872,7 +947,7 @@ Output:
   4-middle
   4-end
 
-To force the output in the same order as the arguments use --keep-order/-k:
+To force the output in the same order as the arguments use B<--keep-order>/B<-k>:
 
   parallel -j2 -k 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
 
@@ -888,24 +963,25 @@ Output:
   1-middle
   1-end
 
+
 =head2 Saving output into files
 
-GNU Parallel can save the output of each job into files:
+GNU B can save the output of each job into files:
 
   parallel --files echo ::: A B C
 
-Output will be similar to:
+Output will be similar to this:
 
   /tmp/pAh6uWuQCg.par
   /tmp/opjhZCzAX4.par
   /tmp/W0AT_Rph2o.par
 
-By default GNU Parallel will cache the output in files in /tmp. This
-can be changed by setting $TMPDIR or --tmpdir:
+By default GNU B will cache the output in files in B. This
+can be changed by setting B<$TMPDIR> or B<--tmpdir>:
 
   parallel --tmpdir /var/tmp --files echo ::: A B C
 
-Output will be similar to:
+Output will be similar to this:
 
   /var/tmp/N_vk7phQRc.par
   /var/tmp/7zA4Ccf3wZ.par
@@ -917,7 +993,7 @@ Or:
 
 Output: Same as above.
 
-The output files can be saved in a structured way using --results:
+The output files can be saved in a structured way using B<--results>:
 
   parallel --results outdir echo ::: A B C
 
@@ -927,45 +1003,54 @@ Output:
   B
   C
 
-but also these files were generated containing the standard output
-(stdout) and standard error (stderr):
+These files were also generated containing the standard output
+(stdout), standard error (stderr), and the sequence number (seq):
 
+  outdir/1/A/seq
   outdir/1/A/stderr
   outdir/1/A/stdout
+  outdir/1/B/seq
   outdir/1/B/stderr
   outdir/1/B/stdout
+  outdir/1/C/seq
   outdir/1/C/stderr
   outdir/1/C/stdout
 
-This is useful if you are running multiple variables:
+B<--header :> will take the first value as name and use that in the
+directory structure. This is useful if you are using multiple input
+sources:
 
   parallel --header : --results outdir echo ::: f1 A B ::: f2 C D
 
 Generated files:
 
+  outdir/f1/A/f2/C/seq
   outdir/f1/A/f2/C/stderr
   outdir/f1/A/f2/C/stdout
+  outdir/f1/A/f2/D/seq
   outdir/f1/A/f2/D/stderr
   outdir/f1/A/f2/D/stdout
+  outdir/f1/B/f2/C/seq
   outdir/f1/B/f2/C/stderr
   outdir/f1/B/f2/C/stdout
+  outdir/f1/B/f2/D/seq
   outdir/f1/B/f2/D/stderr
   outdir/f1/B/f2/D/stdout
 
 The directories are named after the variables and their values.
 
-=head1 Control the execution
+=head1 Controlling the execution
 
 =head2 Number of simultaneous jobs 
 
-The number of concurrent jobs is given with --jobs/-j:
+The number of concurrent jobs is given with B<--jobs>/B<-j>:
 
   /usr/bin/time parallel -N0 -j64 sleep 1 :::: num128
 
-With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run -
+With 64 jobs in parallel the 128 Bs will take 2-8 seconds to run -
 depending on how fast your machine is.
 
-By default --jobs is the same as the number of CPU cores. So this:
+By default B<--jobs> is the same as the number of CPU cores. So this:
 
   /usr/bin/time parallel -N0 sleep 1 :::: num128
 
@@ -973,13 +1058,13 @@ should take twice the time of running 2 jobs per CPU core:
 
   /usr/bin/time parallel -N0 --jobs 200% sleep 1 :::: num128
 
---jobs 0 will run as many jobs in parallel as possible:
+B<--jobs 0> will run as many jobs in parallel as possible:
 
   /usr/bin/time parallel -N0 --jobs 0 sleep 1 :::: num128
 
 which should take 1-7 seconds depending on how fast your machine is.
 
---jobs can read from a file which is re-read when a job finishes:
+B<--jobs> can read from a file which is re-read when a job finishes:
 
   echo 50% > my_jobs
   /usr/bin/time parallel -N0 --jobs my_jobs sleep 1 :::: num128 &
@@ -987,12 +1072,12 @@ which should take 1-7 seconds depending on how fast your machine is.
   echo 0 > my_jobs
   wait
 
-The first second only 50% of the CPU cores will run a job. The '0' is
-put into my_jobs and then the rest of the jobs will be started in
+The first second only 50% of the CPU cores will run a job. Then B<0> is
+put into B and then the rest of the jobs will be started in
 parallel.
 
 Instead of basing the percentage on the number of CPU cores 
-GNU Parallel can base it on the number of CPUs:
+GNU B can base it on the number of CPUs:
 
   parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
 
@@ -1000,7 +1085,7 @@ GNU Parallel can base it on the number of CPUs:
 
 If you have many jobs (e.g. by multiple combinations of input
 sources), it can be handy to shuffle the jobs, so you get different
-values run.
+values run. Use B<--shuf> for that:
 
   parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C
 
@@ -1010,7 +1095,7 @@ Output:
 
 =head2 Interactivity
 
-GNU Parallel can ask the user if a command should be run using --interactive:
+GNU B can ask the user if a command should be run using B<--interactive>:
 
   parallel --interactive echo ::: 1 2 3
 
@@ -1022,8 +1107,8 @@ Output:
   echo 3 ?...y
   3
 
-GNU Parallel can be used to put arguments on the command line for an
-interactive command such as emacs to edit one file at a time:
+GNU B can be used to put arguments on the command line for an
+interactive command such as B to edit one file at a time:
 
   parallel --tty emacs ::: 1 2 3
 
@@ -1033,7 +1118,7 @@ Or give multiple argument in one go to open multiple files:
 
 =head2 A terminal for every job
 
-Using tmux GNU Parallel can start a terminal for every job run:
+Using B<--tmux> GNU B can start a terminal for every job run:
 
   seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}'
 
@@ -1041,15 +1126,15 @@ This will tell you to run something similar to:
 
   tmux -S /tmp/tmsrPrO0 attach
 
-Using normal tmux keystrokes (CTRL-b n or CTRL-b p) you can cycle
+Using normal B keystrokes (CTRL-b n or CTRL-b p) you can cycle
 between windows of the running jobs. When a job is finished it will
 pause for 10 seconds before closing the window.
 
 =head2 Timing
 
 Some jobs do heavy I/O when they start. To avoid a thundering herd GNU
-Parallel can delay starting new jobs. --delay X will make sure there is
-at least X seconds between each start:
+B can delay starting new jobs. B<--delay> I will make
+sure there is at least I seconds between each start:
 
   parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3
 
@@ -1064,8 +1149,8 @@ Output:
 
 
 If jobs taking more than a certain amount of time are known to fail,
-they can be stopped with --timeout. The accuracy of --timeout is 2
-seconds:
+they can be stopped with B<--timeout>. The accuracy of B<--timeout> is
+2 seconds:
 
   parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8
 
@@ -1074,7 +1159,7 @@ Output:
   2
   4
 
-GNU Parallel can compute the median runtime for jobs and kill those
+GNU B can compute the median runtime for jobs and kill those
 that take more than 200% of the median runtime:
 
   parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3
@@ -1086,7 +1171,9 @@ Output:
   3
   2.3
 
-Based on the runtime of completed jobs GNU Parallel can estimate the
+=head2 Progress information
+
+Based on the runtime of completed jobs GNU B can estimate the
 total runtime:
 
   parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1
@@ -1099,9 +1186,7 @@ Output:
   Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
   ETA: 2s 0left 1.11avg  local:0/9/100%/1.1s 
 
-=head2 Progress
-
-GNU Parallel can give progress information with --progress:
+GNU B can give progress information with B<--progress>:
 
   parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
 
@@ -1113,15 +1198,15 @@ Output:
   Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
   local:0/9/100%/1.1s
 
-A progress bar can be shown with --bar:
+A progress bar can be shown with B<--bar>:
 
   parallel --bar sleep ::: 1 3 2 2 1 3 3 2 1
 
-And a graphic bar can be shown with --bar and zenity:
+And a graphic bar can be shown with B<--bar> and B:
 
   seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill)
 
-A logfile of the jobs completed so far can be generated with --joblog:
+A logfile of the jobs completed so far can be generated with B<--joblog>:
 
   parallel --joblog /tmp/log exit  ::: 1 2 3 0 
   cat /tmp/log
@@ -1135,11 +1220,11 @@ Output:
   4       :       1376577365.003  0.003   0       0       0       0       exit 0
 
 The log contains the job sequence, which host the job was run on, the
-start time and run time, how much data was transferred if the job was
-run on a remote host, the exit value, the signal that killed the job,
-and finally the command being run.
+start time and run time, how much data was transferred, the exit
+value, the signal that killed the job, and finally the command being
+run.
 
-With a joblog GNU Parallel can be stopped and later pickup where it
+With a joblog GNU B can be stopped and later pickup where it
 left off. It it important that the input of the completed jobs is
 unchanged.
 
@@ -1164,9 +1249,9 @@ Output:
   5       :       1376580070.028  0.009   0       0       0       0       exit 0
   6       :       1376580070.038  0.007   0       0       0       0       exit 0
 
-Note how the start time of the last 2 jobs is clearly from the second run.
+Note how the start time of the last 2 jobs is clearly different from the second run.
 
-With --resume-failed GNU Parallel will re-run the jobs that failed:
+With B<--resume-failed> GNU B will re-run the jobs that failed:
 
   parallel --resume-failed --joblog /tmp/log exit  ::: 1 2 3 0 0 0
   cat /tmp/log
@@ -1184,13 +1269,39 @@ Output:
   2       :       1376580154.444  0.022   0       0       2       0       exit 2
   3       :       1376580154.466  0.005   0       0       3       0       exit 3
 
-Note how seq 1 2 3 have been repeated because they had exit value != 0.
+Note how seq 1 2 3 have been repeated because they had exit value
+different from 0.
+
+B<--retry-failed> does almost the same as B<--resume-failed>. Where
+B<--resume-failed> reads the commands from the command line (and
+ignores the commands in the joblog), B<--retry-failed> ignores the
+command line and reruns the commands mentioned in the joblog.
+
+  parallel --resume-failed --joblog /tmp/log
+  cat /tmp/log
+
+Output:
+
+  Seq     Host    Starttime       Runtime Send    Receive Exitval Signal  Command
+  1       :       1376580069.544  0.008   0       0       1       0       exit 1
+  2       :       1376580069.552  0.009   0       0       2       0       exit 2
+  3       :       1376580069.560  0.012   0       0       3       0       exit 3
+  4       :       1376580069.571  0.005   0       0       0       0       exit 0
+  5       :       1376580070.028  0.009   0       0       0       0       exit 0
+  6       :       1376580070.038  0.007   0       0       0       0       exit 0
+  1       :       1376580154.433  0.010   0       0       1       0       exit 1
+  2       :       1376580154.444  0.022   0       0       2       0       exit 2
+  3       :       1376580154.466  0.005   0       0       3       0       exit 3
+  1       :       1376580164.633  0.010   0       0       1       0       exit 1
+  2       :       1376580164.644  0.022   0       0       2       0       exit 2
+  3       :       1376580164.666  0.005   0       0       3       0       exit 3
+
 
 =head2 Termination
 
 For certain jobs there is no need to continue if one of the jobs fails
-and has an exit code != 0. GNU Parallel will stop spawning new jobs
-with --halt soon,fail=1:
+and has an exit code different from 0. GNU B will stop spawning new jobs
+with B<--halt soon,fail=1>:
 
   parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
 
@@ -1205,7 +1316,7 @@ Output:
   parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
   echo 2; exit 2
 
-With --halt now,fail=1 the running jobs will be killed immediately:
+With B<--halt now,fail=1> the running jobs will be killed immediately:
 
   parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
 
@@ -1217,8 +1328,8 @@ Output:
   parallel: This job failed:
   echo 1; exit 1
 
-If --halt is given a percentage this percentage of the jobs must fail
-before GNU Parallel stops spawning more jobs:
+If B<--halt> is given a percentage this percentage of the jobs must fail
+before GNU B stops spawning more jobs:
 
   parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9
 
@@ -1236,22 +1347,21 @@ Output:
   parallel: This job failed:
   echo 3; exit 3
 
-If you are looking for success instead of failures, you can use success:
+If you are looking for success instead of failures, you can use
+B. This will finish as soon as the first job succeeds:
 
-  parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
+  parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
 
 Output:
-                
+
   1
   2
   3
   0
   parallel: This job succeeded:
   echo 0; exit 0
-  parallel: Starting no more jobs. Waiting for 1 jobs to finish.
-  4
 
-GNU Parallel can retry the command with --retries. This is useful if a
+GNU B can retry the command with B<--retries>. This is useful if a
 command fails for unknown reasons now and then.
 
   parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
@@ -1273,9 +1383,47 @@ Output:
 
 Note how job 1 and 2 were tried 3 times, but 0 was not retried because it had exit code 0.
 
+=head3 Termination signals (advanced)
+
+Using B<--termseq> you can control which signals are sent when killing
+children. Normally children will be killed by sending them B,
+waiting 200 ms, then another B, waiting 100 ms, then another
+B, waiting 50 ms, then a B, finally waiting 25 ms
+before giving up. It looks like this:
+
+  show_signals() {
+    perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' 
+  }
+  export -f show_signals
+  echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals
+
+Output:
+
+  Got TERM
+  Got TERM
+  Got TERM
+
+Or just:
+
+  echo | parallel -u --timeout 1 show_signals
+
+Output: Same as above.
+
+You can change this to B, B, B:
+
+  echo | parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals
+
+Output:                               
+
+  Got INT
+  Got TERM
+
+The B does not show because it cannot be caught, and thus the child dies.
+
+
 =head2 Limiting the resources
 
-To avoid overloading systems GNU Parallel can look at the system load
+To avoid overloading systems GNU B can look at the system load
 before starting another job:
 
   parallel --load 100% echo load is less than {} job per cpu ::: 1 
@@ -1285,7 +1433,7 @@ Output:
   [when then load is less than the number of cpu cores]
   load is less than 1 job per cpu
 
-GNU Parallel can also check if the system is swapping. 
+GNU B can also check if the system is swapping. 
 
   parallel --noswap echo the system is not swapping ::: now
 
@@ -1294,7 +1442,15 @@ Output:
   [when then system is not swapping]
   the system is not swapping now
 
-GNU Parallel can run the jobs with a nice value. This will work both
+Some jobs need a lot of memory, and should only be started when there
+is enough memory free. Using B<--memfree> GNU B can check if
+there is enough memory free. Additionally, GNU B will kill
+off the youngest job if the memory free falls below 50% of the
+size. The killed job will put back on the queue and retried later.
+
+  parallel --memfree 1G echo will run if more than 1 GB is ::: free
+
+GNU B can run the jobs with a nice value. This will work both
 locally and remotely.
 
   parallel --nice 17 echo this is being run with nice -n ::: 17
@@ -1305,12 +1461,12 @@ Output:
 
 =head1 Remote execution
 
-GNU Parallel can run jobs on remote servers. It uses ssh to
+GNU B can run jobs on remote servers. It uses B to
 communicate with the remote machines. 
 
 =head2 Sshlogin
 
-The most basic sshlogin is -S host:
+The most basic sshlogin is B<-S> I:
 
   parallel -S $SERVER1 echo running on ::: $SERVER1
 
@@ -1318,7 +1474,7 @@ Output:
 
   running on [$SERVER1]
 
-To use a different username prepend the server with username@
+To use a different username prepend the server with I:
 
   parallel -S username@$SERVER1 echo running on ::: username@$SERVER1
 
@@ -1326,7 +1482,7 @@ Output:
 
   running on [username@$SERVER1]
 
-The special sshlogin ':' is the local machine:
+The special sshlogin B<:> is the local machine:
 
   parallel -S : echo running on ::: the_local_machine
 
@@ -1334,7 +1490,7 @@ Output:
 
   running on the_local_machine
 
-If ssh is not in $PATH it can be prepended to $SERVER1:
+If B is not in $PATH it can be prepended to $SERVER1:
 
   parallel -S '/usr/bin/ssh '$SERVER1 echo custom ::: ssh
 
@@ -1342,7 +1498,16 @@ Output:
 
   custom ssh
 
-Several servers can be given using multiple -S:
+The B command can also be given using B<--ssh>:
+
+  parallel --ssh /usr/bin/ssh -S $SERVER1 echo custom ::: ssh
+
+or by setting B<$PARALLEL_SSH>:
+
+  export PARALLEL_SSH=/usr/bin/ssh
+  parallel -S $SERVER1 echo custom ::: ssh
+
+Several servers can be given using multiple B<-S>:
 
   parallel -S $SERVER1 -S $SERVER2 echo ::: running on more hosts
 
@@ -1353,7 +1518,7 @@ Output (the order may be different):
   more
   hosts
 
-Or they can be separated by ,:
+Or they can be separated by B<,>:
 
   parallel -S $SERVER1,$SERVER2 echo ::: running on more hosts
 
@@ -1365,7 +1530,7 @@ Or newline:
   SERVERS="`echo $SERVER1; echo $SERVER2`"
   parallel -S "$SERVERS" echo ::: running on more hosts
 
-The can also be read from a file (replace user@ with the user on $SERVER2):
+They can also be read from a file (replace I with the user on B<$SERVER2>):
 
   echo $SERVER1 > nodefile
   # Force 4 cores, special ssh-command, username
@@ -1374,10 +1539,13 @@ The can also be read from a file (replace user@ with the user on $SERVER2):
 
 Output: Same as above.
 
-The special --sshloginfile '..' reads from ~/.parallel/sshloginfile.
+Every time a job finished, the B<--sshloginfile> will be re-read, so
+it is possible to both add and remove hosts while running.
 
-To force GNU Parallel to treat a server having a given number of CPU
-cores prepend #/ to the sshlogin:
+The special B<--sshloginfile ..> reads from B<~/.parallel/sshloginfile>.
+
+To force GNU B to treat a server having a given number of CPU
+cores prepend the number of core followed by B to the sshlogin:
 
   parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4
 
@@ -1385,22 +1553,24 @@ Output:
 
   force 4 cpus on server
 
-Servers can be put into groups by prepending '@groupname' to the
-server and the group can then be selected by appending '@groupname' to
-the argument if using '--hostgroup'.
+Servers can be put into groups by prepending I<@groupname> to the
+server and the group can then be selected by appending I<@groupname> to
+the argument if using B<--hostgroup>:
 
-  parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
+  parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: \
+    run_on_grp1@grp1 run_on_grp2@grp2
 
 Output:
 
   run_on_grp1
   run_on_grp2
 
-A host can be in multiple groups by separating groups with '+', and
+A host can be in multiple groups by separating the groups with B<+>, and
 you can force GNU B to limit the groups on which the command
-can be run with '-S @groupname':
+can be run with B<-S> I<@groupname>:
 
-  parallel -S @grp1 -S @grp1+grp2/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1 also_grp1
+  parallel -S @grp1 -S @grp1+grp2/$SERVER1 -S @grp2/SERVER2 echo {} ::: \
+    run_on_grp1 also_grp1
 
 Output:
 
@@ -1409,34 +1579,34 @@ Output:
 
 =head2 Transferring files
 
-GNU Parallel can transfer the files to be processed to the remote
+GNU B can transfer the files to be processed to the remote
 host. It does that using rsync.
 
   echo This is input_file > input_file
-  parallel -S $SERVER1 --transfer cat ::: input_file 
+  parallel -S $SERVER1 --transferfile {} cat ::: input_file 
 
 Output:
 
   This is input_file
 
-If the files is processed into another file, the resulting file can be
+If the files are processed into another file, the resulting file can be
 transferred back:
 
   echo This is input_file > input_file
-  parallel -S $SERVER1 --transfer --return {}.out cat {} ">"{}.out ::: input_file 
+  parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file 
   cat input_file.out
 
 Output: Same as above.
 
-To remove the input and output file on the remote server use --cleanup:
+To remove the input and output file on the remote server use B<--cleanup>:
 
   echo This is input_file > input_file
-  parallel -S $SERVER1 --transfer --return {}.out --cleanup cat {} ">"{}.out ::: input_file 
+  parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file 
   cat input_file.out
 
 Output: Same as above.
 
-There is a short hand for --transfer --return --cleanup called --trc:
+There is a shorthand for B<--transferfile {} --return --cleanup> called B<--trc>:
 
   echo This is input_file > input_file
   parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file 
@@ -1444,8 +1614,8 @@ There is a short hand for --transfer --return --cleanup called --trc:
 
 Output: Same as above.
 
-Some jobs need a common database for all jobs. GNU Parallel can
-transfer that using --basefile which will transfer the file before the
+Some jobs need a common database for all jobs. GNU B can
+transfer that using B<--basefile> which will transfer the file before the
 first job:
 
   echo common data > common_file
@@ -1456,24 +1626,24 @@ Output:
   common data
   foo
 
-To remove it from the remote host after the last job use --cleanup.
+To remove it from the remote host after the last job use B<--cleanup>.
 
 
 =head2 Working dir
 
 The default working dir on the remote machines is the login dir. This
-can be changed with --workdir I.
+can be changed with B<--workdir> I.
 
-Files transferred using --transfer and --return will be relative
+Files transferred using B<--transferfile> and B<--return> will be relative
 to I on remote computers, and the command will be executed in
 the dir I.
 
-The special I value ... will create working dirs under
-~/.parallel/tmp/ on the remote computers. If --cleanup is given
+The special I value B<...> will create working dirs under
+B<~/.parallel/tmp> on the remote computers. If B<--cleanup> is given
 these dirs will be removed.
 
-The special I value . uses the current working dir.  If the
-current working dir is beneath your home dir, the value . is
+The special I value B<.> uses the current working dir.  If the
+current working dir is beneath your home dir, the value B<.> is
 treated as the relative path to your home dir. This means that if your
 home dir is different on remote computers (e.g. if your login is
 different) the relative path will still be relative to your home dir.
@@ -1491,8 +1661,8 @@ Output:
 
 =head2 Avoid overloading sshd
 
-If many jobs are started on the same server, sshd can be
-overloaded. GNU Parallel can insert a delay between each job run on
+If many jobs are started on the same server, B can be
+overloaded. GNU B can insert a delay between each job run on
 the same server:
 
   parallel -S $SERVER1 --sshdelay 0.2 echo ::: 1 2 3
@@ -1503,7 +1673,7 @@ Output (the order may be different):
   2
   3
 
-Sshd will be less overloaded if using --controlmaster, which will
+B will be less overloaded if using B<--controlmaster>, which will
 multiplex ssh connections:
 
   parallel --controlmaster -S $SERVER1 echo ::: 1 2 3
@@ -1512,7 +1682,7 @@ Output: Same as above.
 
 =head2 Ignore hosts that are down
 
-In clusters with many hosts a few of the are often down. GNU Parallel
+In clusters with many hosts a few of them are often down. GNU B
 can ignore those hosts. In this case the host 173.194.32.46 is down:
 
   parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar 
@@ -1523,7 +1693,7 @@ Output:
 
 =head2 Running the same commands on all hosts
 
-GNU Parallel can run the same command on all the hosts:
+GNU B can run the same command on all the hosts:
 
   parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar
 
@@ -1535,7 +1705,7 @@ Output (the order may be different):
   bar
 
 Often you will just want to run a single command on all hosts with out
-arguments. --nonall is a no argument --onall:
+arguments. B<--nonall> is a no argument B<--onall>:
 
   parallel --nonall -S $SERVER1,$SERVER2 echo foo bar
 
@@ -1544,7 +1714,7 @@ Output:
   foo bar
   foo bar
 
-When --tag is used with --nonall and --onall the --tagstring is the host:
+When B<--tag> is used with B<--nonall> and B<--onall> the B<--tagstring> is the host:
 
   parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
 
@@ -1553,11 +1723,11 @@ Output (the order may be different):
   $SERVER1 foo bar
   $SERVER2 foo bar
 
---jobs sets the number of servers to log in to in parallel.
+B<--jobs> sets the number of servers to log in to in parallel.
 
-=head2 Transfer environment variables and functions
+=head2 Transferring environment variables and functions
 
-Using --env GNU Parallel can transfer an environment variable to the
+Using B<--env> GNU B can transfer an environment variable to the
 remote system.
 
   MYVAR='foo bar'
@@ -1568,7 +1738,7 @@ Output:
 
   foo bar baz
 
-This works for functions too if your shell is Bash:
+This works for functions, too, if your shell is Bash:
 
   # This only works in Bash
   my_func() {
@@ -1581,9 +1751,9 @@ Output:
 
   in my_func baz
 
-GNU Parallel can copy all defined variables and functions to the
+GNU B can copy all defined variables and functions to the
 remote system. It just needs to record which ones to ignore in
-~/.parallel/ignored_vars. Do that by running this once:
+B<~/.parallel/ignored_vars>. Do that by running this once:
 
   parallel --record-env
   cat ~/.parallel/ignored_vars
@@ -1593,7 +1763,7 @@ Output:
   [list of variables to ignore - including $PATH and $HOME]
 
 Now all new variables and functions defined will be copied when using
---env _:
+B<--env _>:
 
   # The function is only copied if using Bash
   my_func2() {
@@ -1612,24 +1782,28 @@ Output:
 
 =head2 Showing what is actually run
 
---verbose will show the command that would be run on the local
-machine. When a job is run on a remote machine this is wrapped with
-ssh and possibly transferring files and environment variables, setting
-the workdir, and setting --nice value. -vv shows all of this.
+B<--verbose> will show the command that would be run on the local
+machine. When a job is run on a remote machine, this is wrapped with
+B and possibly transferring files and environment variables, setting
+the workdir, and setting B<--nice> value. B<-vv> shows all of this.
 
   parallel -vv -S $SERVER1 echo ::: bar
 
 Output:
-
-  ssh lo exec perl\ -e\ \\\$ENV\\\{\\\"PARALLEL_PID\\\"\\\}=\\\"2554030\\\"\\\;\
-  \\$ENV\\\{\\\"PARALLEL_SEQ\\\"\\\}=\\\"1\\\"\\\;\\\$bashfunc\\\ =\\\ \\\"\\\"\
-  \\;@ARGV=\\\"echo\\\ bar\\\"\\\;\\\$SIG\\\{CHLD\\\}=sub\\\{\\\$done=1\\\;\\\}\
-  \\;\\\$pid=fork\\\;unless\\\(\\\$pid\\\)\\\{setpgrp\\\;exec\\\$ENV\\\{SHELL\\\
-  },\\\"-c\\\",\\\(\\\$bashfunc.\\\"@ARGV\\\"\\\)\\\;die\\\"exec:\\\$\\\!\\\\n\\
-  \"\\\;\\\}do\\\{\\\$s=\\\$s\\\<1\\\?0.001+\\\$s\\\*1.03:\\\$s\\\;select\\\(
-  undef,undef,undef,\\\$s\\\)\\\;\\\}until\\\(\\\$done\\\|\\\|getppid==1\\\)\\\;
-  kill\\\(SIGHUP,-\\\$\\\{pid\\\}\\\)unless\\\$done\\\;wait\\\;exit\\\(\\\$\\\?\
-  \\&127\\\?128+\\\(\\\$\\\?\\\&127\\\):1+\\\$\\\?\\\>\\\>8\\\);
+                        
+  ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
+  eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' 
+  JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
+  IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
+  TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
+  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
+  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
+  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
+  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
+  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
+  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
+  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
   bar
 
 When the command gets more complex, the output is so hard to read, that it is only useful for debugging:
@@ -1642,37 +1816,113 @@ When the command gets more complex, the output is so hard to read, that it is on
 
 Output will be similar to:
 
-  ( ssh lo mkdir -p ./.parallel/tmp/aspire-2554425-1;rsync --protocol 30 -rlDzR 
-  -essh ./abc-file lo:./.parallel/tmp/aspire-2554425-1 );ssh lo exec perl -e \''
-  @GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";
-  $SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my(
-  $in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){
-  close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(
-  join"",@ARGV));close$in;exit;}wait;eval$eval;'\' QlpoOTFBWSZTWayP388AAbdfgAAQd
-  X/+3//l/wS/7//vQAIq3U6bauIpk1NPUwnqGgND1NGI9TTQ0A0ADIDQNITU9NGqfonpPJRvU0ZQZAA
-  GhpoBoABpqaFNBMh+kGoybUaHogwCYBAADCSgRNqek1TxTeknqemhGyBqek8pk2jRPSeo002mQRo5f
-  oSZYrgzQFDd3HNWaskbx+MxNR89BdDzESFbADOJkI+QhIlnojHCWRVuGc2j2lzMzE41wC7auAMQ06c
-  S3AlqQfKcdo0gd506U0HzAAxMkGJBHjDCZULOMpVbowhIVxxaQz7yansTsBgurEZaGO/6K0Nc4iodr
-  BW4m9SXErqRbLNy5eANDvZ+TIt2c2GBcWSlmYuloxY5u2bGUdU/dGsO5EhyrvKCpZMhIgmQFAQhcwR
-  mD+jMKRawkRFJSGyTNC3PqWnE51ucPyx29Yxjnkyub98lytpyk+v8BUc4eA3xz98dMYjxvb0pgWksh
-  oHZ7HwGQRq1vuDyzKgkwPL9lwGIdL+WPNJFSljlVAahIhQpDCAOJpTqDhgmfoRQcy54PC9T0T3iMnV
-  JeTUdL8P0/s18NqDSUavMNV3qD0CtYi6entl0neNsOQN2VDSLHj0xOMls65LNPo+Wh28rJtVoh2JgE
-  7Q9Qo/XBr6krGIsYpQR6nRDuJCD/5aaQBBFFQGtv2VoFTwkXiUTxFP1CC4AGBznAaMklgWQvVtKguJ
-  zQnPqr9ABtHwbB5GTzPOQ4iWAmrUxvl4j5wqrVchOZcs3NYUQmGO2+VYBimFVxhGcaxDALMZ6bWEUo
-  yt8eC8W5o1ObFtTnHAvjOQgYEL/nHTcxU0G57QMKCzJcASQWFNpe2CpQcgYlBxIN4kwtfxdyRThQkK
-  yP388;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ 
-  ./.parallel/tmp/aspire-2554425-1/./.\;\ rsync -rlDzR -essh lo:./abc-file.out 
-  ./.;ssh lo \(rm\ -f\ ./.parallel/tmp/aspire-2554425-1/abc-file\;\ sh\ -c\ \'
-  rmdir\ ./.parallel/tmp/aspire-2554425-1/\ ./.parallel/tmp/\ ./.parallel/\ 2\>
-  /dev/null\'\;rm\ -rf\ ./.parallel/tmp/aspire-2554425-1\;\);ssh lo \(rm\ -f\ 
-  ./.parallel/tmp/aspire-2554425-1/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.parallel
-  /tmp/aspire-2554425-1/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ 
-  -rf\ ./.parallel/tmp/aspire-2554425-1\;\);ssh lo rm -rf .parallel/tmp/
-  aspire-2554425-1; exit $_EXIT_status;
+  ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
+  -rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
+  exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
+  eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\'
+  c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
+  OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
+  YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
+  KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
+  U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
+  VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
+  cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
+  c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
+  RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
+  PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
+  RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
+  RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
+  fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
+  IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
+  eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
+  fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
+  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
+  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
+  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
+  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
+  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
+  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
+  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
+  mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
+  ./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
+  lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
+  ./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \'rmdir\
+  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
+  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
+  \(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \'rmdir\
+  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
+  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
+  -rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;
+
+=head1 Saving to an SQL base (advanced)
+
+GNU B can save into an SQL base. Point GNU B to a
+table and it will put the joblog there together with the variables and
+the outout each in their own column.
+
+GNU B uses a DBURL to address the table. A DBURL has this format:
+
+  vendor://[[user][:password]@][host][:port]/[database[/table]
+
+Example:
+
+  mysql://scott:tiger@my.example.com/mydatabase/mytable
+  postgresql://scott:tiger@pg.example.com/mydatabase/mytable
+  sqlite3:///%2Ftmp%2Fmydatabase/mytable
+
+To refer to B with B you need to encode the B as B<%2F>.
+
+Run a job using B on B in B:
+
+  DBURL=sqlite3:///%2Ftmp%2Fmydatabase
+  DBURLTABLE=$DBURL/mytable
+  parallel --sqlandworker $DBURLTABLE echo ::: foo bar ::: baz quuz
+
+To see the result:
+
+  sql $DBURL 'SELECT * FROM mytable ORDER BY Seq;'
+
+Output will be similar to:
+
+  Seq|Host|Starttime|JobRuntime|Send|Receive|Exitval|_Signal|Command|V1|V2|Stdout|Stderr
+  1|:|1451619638.903|0.806||8|0|0|echo foo baz|foo|baz|foo baz
+  |
+  2|:|1451619639.265|1.54||9|0|0|echo foo quuz|foo|quuz|foo quuz
+  |
+  3|:|1451619640.378|1.43||8|0|0|echo bar baz|bar|baz|bar baz
+  |
+  4|:|1451619641.473|0.958||9|0|0|echo bar quuz|bar|quuz|bar quuz
+  |
+
+The first columns are well known from B<--joblog>. B and B are
+data from the input sources. B and B are standard
+output and standard error, respectively.
+
+=head2 Using multiple workers
+
+Using an SQL base as storage costs a lot of performance.
+
+One of the situations where it makes sense is if you have multiple
+workers.
+
+You can then have a single master machine that submits jobs to the SQL
+base (but does not do any of the work):
+
+  parallel --sql $DBURLTABLE echo ::: foo bar ::: baz quuz
+
+On the worker machines you run exactly the same command except you
+replace B<--sql> with B<--sqlworker>.
+
+  parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz
+
+To run a master and a worker on the same machine use B<--sqlandworker>
+as shown earlier.
+
 
 =head1 --pipe
 
-The --pipe functionality puts GNU Parallel in a different mode:
+The B<--pipe> functionality puts GNU B in a different mode:
 Instead of treating the data on stdin (standard input) as arguments
 for a command to run, the data will be sent to stdin (standard input)
 of the command.
@@ -1685,7 +1935,7 @@ where command_B is slow, and you want to speed up command_B.
 
 =head2 Chunk size
 
-By default GNU Parallel will start an instance of command_B, read a
+By default GNU B will start an instance of command_B, read a
 chunk of 1 MB, and pass that to the instance. Then start another
 instance, read another chunk, and pass that to the second instance.
 
@@ -1701,9 +1951,9 @@ Output (the order may be different):
   149796  149796 1048572
    85349   85349  597444
 
-The size of the chunk is not exactly 1 MB because GNU Parallel only
+The size of the chunk is not exactly 1 MB because GNU B only
 passes full lines - never half a line, thus the blocksize is only
-average 1 MB. You can change the block size to 2 MB with --block:
+average 1 MB. You can change the block size to 2 MB with B<--block>:
 
   cat num1000000 | parallel --pipe --block 2M wc
 
@@ -1714,12 +1964,12 @@ Output (the order may be different):
   299593  299593 2097151
    85349   85349  597444
 
-GNU Parallel treats each line as a record. If the order of record is
+GNU B treats each line as a record. If the order of record is
 unimportant (e.g. you need all lines processed, but you do not care
-which is processed first), then you can use --round-robin. Without
---round-robin GNU Parallel will start a command per block; with
---round-robin only the requested number of jobs will be started
-(--jobs). The records will then be distributed between the running
+which is processed first), then you can use B<--round-robin>. Without
+B<--round-robin> GNU B will start a command per block; with
+B<--round-robin> only the requested number of jobs will be started
+(B<--jobs>). The records will then be distributed between the running
 jobs:
 
   cat num1000000 | parallel --pipe -j4 --round-robin wc
@@ -1736,10 +1986,10 @@ records each, and one instance got 1 full and 1 partial record.
 
 =head2 Records
 
-GNU Parallel sees the input as records. The default record is a single
+GNU B sees the input as records. The default record is a single
 line.
 
-Using -N140000 GNU Parallel will read 140000 records at a time:
+Using B<-N140000> GNU B will read 140000 records at a time:
 
   cat num1000000 | parallel --pipe -N140000 wc
 
@@ -1757,7 +2007,7 @@ Output (the order may be different):
 Notice that the last job could not get the full 140000 lines, but only
 20000 lines.
 
-If a record is 75 lines -L can be used:
+If a record is 75 lines B<-L> can be used:
 
   cat num1000000 | parallel --pipe -L75 wc
 
@@ -1772,23 +2022,23 @@ Output (the order may be different):
    85350   85350  597450
       25      25     176
 
-Notice GNU Parallel still reads a block of around 1 MB; but instead of
-passing full lines to 'wc' it passes full 75 lines at a time. This
+Notice GNU B still reads a block of around 1 MB; but instead of
+passing full lines to B it passes full 75 lines at a time. This
 of course does not hold for the last job (which in this case got 25
 lines).
 
 =head2 Record separators
 
-GNU Parallel uses separators to determine where two records split.
+GNU B uses separators to determine where two records split.
 
---recstart gives the string that starts a record; --recend gives the
-string that ends a record. The default is --recend '\n' (newline).
+B<--recstart> gives the string that starts a record; B<--recend> gives the
+string that ends a record. The default is B<--recend '\n'> (newline).
 
-If both --recend and --recstart are given, then the record will only
+If both B<--recend> and B<--recstart> are given, then the record will only
 split if the recend string is immediately followed by the recstart
 string.
 
-Here the --recend is set to ', ':
+Here the B<--recend> is set to B<', '>:
 
   echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
 
@@ -1804,9 +2054,9 @@ Output:
   qux/,
   END
 
-Here the --recstart is set to '/':
+Here the B<--recstart> is set to B:
 
-  echo /foo, bar/, /baz, qux/, | parallel -kN1 --recstart '/' --pipe echo JOB{#}\;cat\;echo END
+  echo /foo, bar/, /baz, qux/, | parallel -kN1 --recstart / --pipe echo JOB{#}\;cat\;echo END
 
 Output:
 
@@ -1820,9 +2070,9 @@ Output:
   /,
   END
 
-Here both --recend and --recstart are set:
+Here both B<--recend> and B<--recstart> are set:
 
-  echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --recstart '/' --pipe echo JOB{#}\;cat\;echo END
+  echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --recstart / --pipe echo JOB{#}\;cat\;echo END
 
 Output:
 
@@ -1834,9 +2084,9 @@ Output:
 
 Note the difference between setting one string and setting both strings.
 
-With --regexp the --recend and --recstart will be treated as a regular expression:
+With B<--regexp> the B<--recend> and B<--recstart> will be treated as a regular expression:
 
-  echo foo,bar,_baz,__qux, | parallel -kN1 --regexp --recend ',_+' --pipe echo JOB{#}\;cat\;echo END
+  echo foo,bar,_baz,__qux, | parallel -kN1 --regexp --recend ,_+ --pipe echo JOB{#}\;cat\;echo END
 
 Output:
 
@@ -1848,9 +2098,9 @@ Output:
   qux,
   END
 
-GNU Parallel can remove the record separators with --remove-rec-sep/--rrs:
+GNU B can remove the record separators with B<--remove-rec-sep>/B<--rrs>:
 
-  echo foo,bar,_baz,__qux, | parallel -kN1 --rrs --regexp --recend ',_+' --pipe echo JOB{#}\;cat\;echo END
+  echo foo,bar,_baz,__qux, | parallel -kN1 --rrs --regexp --recend ,_+ --pipe echo JOB{#}\;cat\;echo END
 
 Output:
 
@@ -1865,7 +2115,8 @@ Output:
 =head2 Header
 
 If the input data has a header, the header can be repeated for each
-job by matching the header with --header. If headers start with %:
+job by matching the header with B<--header>. If headers start with
+B<%> you can do this:
 
   cat num_%header | parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat
 
@@ -1894,7 +2145,7 @@ Output (the order may be different):
   %head2
   10
 
-If the header is 2 lines, --header 2 will work:
+If the header is 2 lines, B<--header> 2 will work:
 
   cat num_%header | parallel --header 2 --pipe -N3 echo JOB{#}\;cat
 
@@ -1902,10 +2153,10 @@ Output: Same as above.
 
 =head2 --pipepart
 
---pipe is not very efficient. It maxes out at around 500
-MB/s. --pipepart can easily deliver 5 GB/s. But there are a few
+B<--pipe> is not very efficient. It maxes out at around 500
+MB/s. B<--pipepart> can easily deliver 5 GB/s. But there are a few
 limitations. The input has to be a normal file (not a pipe) given by
--a or :::: and -L/-l/-N do not work.
+B<-a> or B<::::> and B<-L>/B<-l>/B<-N> do not work.
 
   parallel --pipepart -a num1000000 --block 3m wc
 
@@ -1920,18 +2171,18 @@ Output (the order may be different):
 
 =head2 Input data and parallel command in the same file
 
-GNU Parallel is often called as:
+GNU B is often called as this:
 
   cat input_file | parallel command
 
-With --shebang the input_file and parallel can be combined into the same script.
+With B<--shebang> the I and B can be combined into the same script.
 
-UNIX-scripts start with a shebang line like:
+UNIX-scripts start with a shebang line like this:
 
   #!/bin/bash
 
-GNU Parallel can do that, too. With --shebang the arguments can be
-listed in the file. The parallel command is the first line of the
+GNU B can do that, too. With B<--shebang> the arguments can be
+listed in the file. The B command is the first line of the
 script:
 
   #!/usr/bin/parallel --shebang -r echo
@@ -1948,19 +2199,19 @@ Output (the order may be different):
 
 =head2 Parallelizing existing scripts
 
-GNU Parallel is often called as:
+GNU B is often called as:
 
   cat input_file | parallel command
   parallel command ::: foo bar
 
-If command is a script parallel can be combined into a single file so:
+If command is a script B can be combined into a single file so:
 
   cat input_file | command
   command foo bar
 
-will run the script in parallel.
+will run the script in B.
 
-This perl script perl_echo works like echo:
+This B script B works like B:
 
   #!/usr/bin/perl
 
@@ -1970,7 +2221,7 @@ It can be called as:
 
   parallel perl_echo ::: foo bar
 
-By changing the #!-line it can be run in parallel
+By changing the B<#!>-line it can be run in parallel:
 
   #!/usr/bin/parallel --shebang-wrap /usr/bin/perl
 
@@ -1993,55 +2244,106 @@ This technique can be used for:
 
 #!/usr/bin/parallel --shebang-wrap /usr/bin/perl
 
+print "Arguments @ARGV\n";
+
+
 =item Python:
 
 #!/usr/bin/parallel --shebang-wrap /usr/bin/python
 
+import sys
+
+print 'Arguments', str(sys.argv)
+
+
 =item Bash:
 
 #!/usr/bin/parallel --shebang-wrap /bin/bash
 
+echo Arguments "$@"
+
+
 =item R:
 
 #!/usr/bin/parallel --shebang-wrap /usr/bin/Rscript --vanilla --slave
 
+args <- commandArgs(trailingOnly = TRUE)
+print(paste("Arguments ",args))
+
+
 =item GNUplot:
 
 #!/usr/bin/parallel --shebang-wrap ARG={} /usr/bin/gnuplot
 
+print "Arguments ", system('echo $ARG')
+
+
 =item Ruby:
 
 #!/usr/bin/parallel --shebang-wrap /usr/bin/ruby
 
+print "Arguments "
+puts ARGV
+
+
+=item Octave:
+
+#!/usr/bin/parallel --shebang-wrap /usr/bin/octave
+
+printf ("Arguments");
+arg_list = argv ();
+for i = 1:nargin
+  printf (" %s", arg_list{i});
+endfor
+printf ("\n");
+
+=item Common LISP:
+
+#!/usr/bin/parallel --shebang-wrap /usr/bin/clisp
+
+(format t "~&~S~&" 'Arguments)
+(format t "~&~S~&" *args*)
+
+LUA
+PHP
+Javascript
+nodejs
+Tcl
+C#?
+
+
 =back
 
 =head1 Semaphore
 
-GNU Parallel can work as a counting semaphore. This is slower and less
+GNU B can work as a counting semaphore. This is slower and less
 efficient than its normal mode.
 
 A counting semaphore is like a row of toilets. People needing a toilet
 can use any toilet, but if there are more people than toilets, they
 will have to wait for one of the toilets to be available.
 
-An alias for 'parallel --semaphore' is 'sem'.
+An alias for B is B.
 
-'sem' will follow a person to the toilets, wait until a toilet is
+B will follow a person to the toilets, wait until a toilet is
 available, leave the person in the toilet and exit.
 
-'sem --fg' will follow a person to the toilets, wait until a toilet is
+B will follow a person to the toilets, wait until a toilet is
 available, stay with the person in the toilet and exit when the person
 exits.
 
-'sem --wait' will wait for all persons to leave the toilets.
+B will wait for all persons to leave the toilets.
 
-'sem' does not have a queue discipline, so the next person is chosen
+B does not have a queue discipline, so the next person is chosen
 randomly.
 
--j sets the number of toilets. The default is to have only one toilet
-(technically this is called a mutex). The program is started in the
-background and 'sem' exits immediately. Use --wait to wait for all
-'sem's to finish:
+B<-j> sets the number of toilets.
+
+=head2 Mutex
+
+The default is to have only one toilet (this is called a mutex). The
+program is started in the background and B exits immediately. Use
+B<--wait> to wait for all Bs to finish:
 
   sem 'sleep 1; echo The first finished' &&
     echo The first is now running in the background &&
@@ -2056,7 +2358,7 @@ Output:
   The second is now running in the background
   The second finished
 
-The command can be run in the foreground with --fg, which will only
+The command can be run in the foreground with B<--fg>, which will only
 exit when the command completes:
 
   sem --fg 'sleep 1; echo The first finished' &&
@@ -2066,11 +2368,11 @@ exit when the command completes:
   sem --wait
 
 The difference between this and just running the command, is that a
-mutex is set, so if other sems were running in the background only one
-would run at the same time.
+mutex is set, so if other Bs were running in the background only one
+would run at a time.
 
 To tell the difference between which semaphore is used, use
---semaphorename/--id. Run this in one terminal:
+B<--semaphorename>/B<--id>. Run this in one terminal:
 
   sem --id my_id -u 'echo First started; sleep 10; echo The first finished'
 
@@ -2087,8 +2389,8 @@ else will have to wait. A counting semaphore is like having multiple
 toilets: Several people can use the toilets, but when they all are in
 use, everyone else will have to wait.
 
-sem can emulate a counting semaphore. Use --jobs to set the number of
-toilets:
+B can emulate a counting semaphore. Use B<--jobs> to set the number of
+toilets like this:
 
   sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' &&
   sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' &&
@@ -2109,7 +2411,7 @@ Output:
 
 =head2 Timeout
 
-With --semaphoretimeout you can force running the command anyway after
+With B<--semaphoretimeout> you can force running the command anyway after
 a period (postive number) or give up (negative number):
 
   sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' &&
@@ -2129,10 +2431,10 @@ Note how the 'Give up' was not run.
 
 =head1 Informational
 
-GNU Parallel has some options to give short information about the
+GNU B has some options to give short information about the
 configuration.
 
---help will print a summary of the most important options:
+B<--help> will print a summary of the most important options:
 
   parallel --help
 
@@ -2167,7 +2469,7 @@ Output:
   O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
   ;login: The USENIX Magazine, February 2011:42-47.
 
-When asking for help, always report the full output of:
+    When asking for help, always report the full output of this:
 
   parallel --version
 
@@ -2186,7 +2488,7 @@ Output:
   O. Tange (2011): GNU Parallel - The Command-Line Power Tool, 
   ;login: The USENIX Magazine, February 2011:42-47.
 
-In scripts --minversion can be used to ensure the user has at least
+In scripts B<--minversion> can be used to ensure the user has at least
 this version:
 
   parallel --minversion 20130722 && echo Your version is at least 20130722.
@@ -2196,8 +2498,8 @@ Output:
   20130722
   Your version is at least 20130722.
 
-If using GNU Parallel for research the BibTeX citation can be
-generated using --bibtex.
+If using GNU B for research the BibTeX citation can be
+generated using B<--bibtex>:
 
   parallel --bibtex
 
@@ -2216,7 +2518,7 @@ Output:
    pages = {42-47}
   }
 
-With --max-line-length-allowed GNU Parallel will report the maximal
+With B<--max-line-length-allowed> GNU B will report the maximal
 size of the command line:
 
   parallel --max-line-length-allowed
@@ -2225,7 +2527,7 @@ Output (may vary on different systems):
 
   131071
 
---number-of-cpus and --number-of-cores run system specific code to
+B<--number-of-cpus> and B<--number-of-cores> run system specific code to
 determine the number of CPUs and CPU cores on the system. On
 unsupported platforms they will return 1:
 
@@ -2239,11 +2541,11 @@ Output (may vary on different systems):
 
 =head1 Profiles
 
-The defaults for GNU Parallel can be changed systemwise by putting the
-command line options in /etc/parallel/config. They can be changed for
-a user by putting them in ~/.parallel/config.
+The defaults for GNU B can be changed systemwide by putting the
+command line options in B. They can be changed for
+a user by putting them in B<~/.parallel/config>.
 
-Profiles work the same way, but have to be referred to with --profile:
+Profiles work the same way, but have to be referred to with B<--profile>:
 
   echo '--nice 17' > ~/.parallel/nicetimeout
   echo '--timeout 300%' >> ~/.parallel/nicetimeout
@@ -2271,7 +2573,7 @@ Output:
 
 I hope you have learned something from this tutorial.
 
-If you like GNU Parallel:
+If you like GNU B:
 
 =over 2
 
@@ -2293,6 +2595,7 @@ mailing lists
 =item *
 
 Request or write a review for your favourite blog or magazine
+(especially if you do something cool with GNU B)
 
 =item *
 
@@ -2300,17 +2603,17 @@ Invite me for your next conference
 
 =back
 
-If you use GNU Parallel for research:
+If you use GNU B for research:
 
 =over 2
 
 =item *
 
-Please cite GNU Parallel in you publications (use --bibtex)
+Please cite GNU B in you publications (use B<--bibtex>)
 
 =back
 
-If GNU Parallel saves you money:
+If GNU B saves you money:
 
 =over 2
 
@@ -2321,7 +2624,7 @@ https://my.fsf.org/donate/
 
 =back
 
-(C) 2013,2014,2015 Ole Tange, GPLv3
+(C) 2013,2014,2015,2016 Ole Tange, GPLv3
 
 
 =cut
diff --git a/src/sql b/src/sql
index 46f1f53e..acbd2db4 100755
--- a/src/sql
+++ b/src/sql
@@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20151222;
+    $Global::version = 20160101;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself
diff --git a/testsuite/tests-to-run/parallel-local-sql.sh b/testsuite/tests-to-run/parallel-local-sql.sh
new file mode 100644
index 00000000..9f4a9a78
--- /dev/null
+++ b/testsuite/tests-to-run/parallel-local-sql.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+export SQLITE=sqlite3:///%2Frun%2Fshm%2Fparallel.db
+export SQLITETBL=$SQLITE/parsql
+export PG=pg://tange:tange@lo/tange
+export PGTBL=$PG/parsql
+export MYSQL=mysql://tange:tange@lo/tange
+export MYSQLTBL=$MYSQL/parsql
+export PGTBL2=${PGTBL}2
+export PGTBL3=${PGTBL}3
+export PGTBL4=${PGTBL}4
+export PGTBL5=${PGTBL}5
+export T1=$(tempfile)
+export T2=$(tempfile)
+export T3=$(tempfile)
+export T4=$(tempfile)
+export T5=$(tempfile)
+export T6=$(tempfile)
+export T7=$(tempfile)
+export T8=$(tempfile)
+export T9=$(tempfile)
+export T10=$(tempfile)
+export T11=$(tempfile)
+export T12=$(tempfile)
+export T13=$(tempfile)
+export T14=$(tempfile)
+
+#sql mysql://tange:tange@lo/ 'create database tange;'; 
+cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 | perl -pe 's/\s*\d+\.?\d+\s*/999/g;s/999e+999.\s+.\s+/999e+999|999/g;'
+echo '### --sqlandworker mysql'
+  (sleep 2; parallel --sqlworker $MYSQLTBL sleep .3\;echo >$T1) &
+  parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; 
+  true sort -u $T1 $T2; 
+  sql $MYSQL 'select * from parsql order by seq;'
+
+echo '### --sqlandworker postgresql'
+  (sleep 2; parallel --sqlworker $PGTBL sleep .3\;echo >$T3) &
+  parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4; 
+  true sort -u $T3 $T4; 
+  sql $PG 'select * from parsql order by seq;'
+
+echo '### --sqlandworker sqlite'
+  (sleep 2; parallel --sqlworker $SQLITETBL sleep .3\;echo >$T5) &
+  parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6; 
+  true sort -u $T5 $T6; 
+  sql $SQLITE 'select * from parsql order by seq;'
+
+echo '### --sqlandworker postgresql -S lo'
+  (sleep 2; parallel -S lo --sqlworker $PGTBL2 sleep .3\;echo >$T7) &
+  parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8; 
+  true sort -u $T7 $T8; 
+  sql $PG 'select * from parsql2 order by seq;'
+
+echo '### --sqlandworker postgresql --results'
+  mkdir -p /tmp/out--sql
+  (sleep 2; parallel --results /tmp/out--sql --sqlworker $PGTBL3 sleep .3\;echo >$T9) &
+  parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T10; 
+  true sort -u $T9 $T10; 
+  sql $PG 'select * from parsql3 order by seq;'
+
+echo '### --sqlandworker postgresql --linebuffer'
+  (sleep 2; parallel --linebuffer --sqlworker $PGTBL4 sleep .3\;echo >$T11) &
+  parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T12; 
+  true sort -u $T11 $T12; 
+  sql $PG 'select * from parsql4 order by seq;'
+
+echo '### --sqlandworker postgresql -u'
+  (sleep 2; parallel -u --sqlworker $PGTBL5 sleep .3\;echo >$T13) &
+  parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T14; 
+  true sort -u $T13 $T14; 
+  sql $PG 'select * from parsql5 order by seq;'
+
+EOF
+
+eval rm '$T'{1..14}
\ No newline at end of file