src/parallel: --transfer files with /./ as relative to --workdir.

NEWS

@@ -1,3 +1,67 @@
+20131022
+
+* --transfer files with /./ in the path will copy the files relative
+  to the --workdir.
+
+* The maximal command length is now cached in a file halfing the
+  startup time.
+
+* GNU Parallel was cited in: Investigating speaker gender using rhythm
+  metrics in Arabic dialects
+  http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=6602389&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D6602389
+
+* GNU Parallel was cited in: Tiedon erist ̈minen ja visualisointi a
+  lastensuojelun asiakaskertomuksista
+  https://tampub.uta.fi/handle/10024/84978
+
+* Some Additional Useful Bioinformatics Software
+  http://nix-bio.blogspot.dk/2013/10/additional-software-for-bioinformatics.html
+
+* GNU Parallel Example Screen Cast (bil to Geotiff conversion)
+  http://www.youtube.com/watch?v=PpF27eC7Mys
+
+* Using GNU Parallel to roll-your-own Map Reduce!
+  http://www.rankfocus.com/hello-world/
+
+* Use multiple CPU Cores with your Linux commands
+  http://www.rankfocus.com/use-cpu-cores-linux-commands/
+
+* Using GNU Parallel at HPC @ Uni.lu
+  https://hpc.uni.lu/users/use_cases/
+
+* Scaling up with parallelization
+  https://www.msi.umn.edu/sites/default/files/AdvPython_1.pdf
+
+* How to teach yourself to be a technical tester
+  http://infiniteundo.com/post/62421363463/qa-toc
+
+* Faster Rasters For All
+  http://2013.foss4g.org/conf/programme/presentations/52/
+
+* Optimizing translated file downloads
+  http://www.smartling.com/blog/2013/05/20/optimizing-translated-file-downloads/
+
+* EMC NetWorker parallel saveset cloning with nsrclone and GNU parallel
+  http://www.beebotech.com.au/2013/10/emc-networker-parallel-saveset-cloning-with-nsrclone-and-gnu-parallel/
+
+* Best Practices for Amazon EMR
+  http://media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf
+
+* Computing checksums for backup
+  https://identi.ca/evan/note/6yf1GzAARtyBhj__xzMvAg
+
+* Un exemplu de utilizare a programului GNU parallel
+  http://vundicind.blogspot.dk/2013/10/exemplu-utilizare-gnu-parallel.html
+
+* 平行化你的工作
+  http://www.slideshare.net/drakeguan/part1-23705978
+
+* なんか気持ちよかです Gnu Parallel
+  http://a3-lab.jpn.org/note/%E3%81%AA%E3%82%93%E3%81%8B%E6%B0%97%E6%8C%81%E3%81%A1%E3%82%88%E3%81%8B%E3%81%A7%E3%81%99-gnu-parallel
+
+* Bug fixes and man page updates.
+
+
 20130922
 
 * PDF-files of documentation is now included.

README

@@ -40,9 +40,9 @@ document.
 
 Full installation of GNU Parallel is as simple as:
 
-  wget http://ftpmirror.gnu.org/parallel/parallel-20130922.tar.bz2
+  wget http://ftpmirror.gnu.org/parallel/parallel-20131022.tar.bz2
-  bzip2 -dc parallel-20130922.tar.bz2 | tar xvf -
+  bzip2 -dc parallel-20131022.tar.bz2 | tar xvf -
-  cd parallel-20130922
+  cd parallel-20131022
   ./configure && make && make install
 
 
@@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as:
 If you are not root you can add ~/bin to your path and install in
 ~/bin and ~/share:
 
-  wget http://ftpmirror.gnu.org/parallel/parallel-20130922.tar.bz2
+  wget http://ftpmirror.gnu.org/parallel/parallel-20131022.tar.bz2
-  bzip2 -dc parallel-20130922.tar.bz2 | tar xvf -
+  bzip2 -dc parallel-20131022.tar.bz2 | tar xvf -
-  cd parallel-20130922
+  cd parallel-20131022
   ./configure --prefix=$HOME && make && make install
 
 Or if your system lacks 'make' you can simply copy src/parallel

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20130922], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20131022], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/release_new_version

@@ -208,14 +208,31 @@ available for download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
+* --transfer files with /./ in the path will copy the files relative
+  to the --workdir.
+
+* The maximal command length is now cached in a file halfing the
+  startup time.
+
+* GNU Parallel was cited in: Investigating speaker gender using rhythm
+  metrics in Arabic dialects
+  http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=6602389&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D6602389
+
+* GNU Parallel was cited in: Tiedon erist ̈minen ja visualisointi a
+  lastensuojelun asiakaskertomuksista
+  https://tampub.uta.fi/handle/10024/84978
+
+* Some Additional Useful Bioinformatics Software
+  http://nix-bio.blogspot.dk/2013/10/additional-software-for-bioinformatics.html
+
+* GNU Parallel Example Screen Cast (bil to Geotiff conversion)
+  http://www.youtube.com/watch?v=PpF27eC7Mys
+
 * Using GNU Parallel to roll-your-own Map Reduce!
   http://www.rankfocus.com/hello-world/
 
-* 平行化你的工作
+* Use multiple CPU Cores with your Linux commands
-  http://www.slideshare.net/drakeguan/part1-23705978
+  http://www.rankfocus.com/use-cpu-cores-linux-commands/
-
-* Best Practices for Amazon EMR
-  http://media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf
 
 * Using GNU Parallel at HPC @ Uni.lu
   https://hpc.uni.lu/users/use_cases/
@@ -223,14 +240,34 @@ New in this release:
 * Scaling up with parallelization
   https://www.msi.umn.edu/sites/default/files/AdvPython_1.pdf
 
-* Optimizing translated file downloads
+* How to teach yourself to be a technical tester
-  http://www.smartling.com/blog/2013/05/20/optimizing-translated-file-downloads/
+  http://infiniteundo.com/post/62421363463/qa-toc
-
-https://identi.ca/evan/note/6yf1GzAARtyBhj__xzMvAg
 
 * Faster Rasters For All
   http://2013.foss4g.org/conf/programme/presentations/52/
 
+* Optimizing translated file downloads
+  http://www.smartling.com/blog/2013/05/20/optimizing-translated-file-downloads/
+
+* EMC NetWorker parallel saveset cloning with nsrclone and GNU parallel
+  http://www.beebotech.com.au/2013/10/emc-networker-parallel-saveset-cloning-with-nsrclone-and-gnu-parallel/
+
+* Best Practices for Amazon EMR
+  http://media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf
+
+* Computing checksums for backup
+  https://identi.ca/evan/note/6yf1GzAARtyBhj__xzMvAg
+
+* Un exemplu de utilizare a programului GNU parallel
+  http://vundicind.blogspot.dk/2013/10/exemplu-utilizare-gnu-parallel.html
+
+* 平行化你的工作
+  http://www.slideshare.net/drakeguan/part1-23705978
+
+* なんか気持ちよかです Gnu Parallel
+  http://a3-lab.jpn.org/note/%E3%81%AA%E3%82%93%E3%81%8B%E6%B0%97%E6%8C%81%E3%81%A1%E3%82%88%E3%81%8B%E3%81%A7%E3%81%99-gnu-parallel
+
+
 * Bug fixes and man page updates.
 
 

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20130922;
+$Global::version = 20131022;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -32,6 +32,7 @@ use File::Path;
 use Getopt::Long;
 # Used to ensure code quality
 use strict;
+use File::Basename;
 
 if(not $ENV{SHELL}) {
     # $ENV{SHELL} is sometimes not set on Mac OS X and Windows
@@ -184,7 +185,7 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
 		::die_bug("host check too many col0: $_");
 	    }
 	} else {
-	    ::die_bug("host check unmatched short jobline: $_");
+	    ::die_bug("host check unmatched short jobline ($col[0]): $_");
 	}
     }
     close $host_fh;
@@ -207,6 +208,10 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
 }
 
 if($opt::nonall or $opt::onall) {
+
+# TODO --transfer foo/./bar --cleanup
+# multiple --transfer and --basefile with different /./
+
   sub tmp_joblog {
     my $joblog = shift;
     if(not defined $joblog) {
@@ -790,7 +795,7 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20130927;
+    $Global::version = 20131022;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -1357,6 +1362,40 @@ sub shell_quote_scalar {
     return $a;
 }
 
+sub shell_quote_file {
+    # Quote the string so shell will not expand any special chars and prepend ./ if needed
+    # Returns:
+    #   string quoted with \ as needed by the shell and ./ if needed
+    my $a = shell_quote_scalar(shift);
+    if(defined $a) {
+	if($a =~ m:^/: or $a =~ m:^\./:) {
+	    # /abs/path or ./rel/path => skip
+	} else {
+	    # rel/path => ./rel/path
+	    $a = "./".$a;
+	}
+    }
+    return $a;
+}
+
+sub shell_quote_files {
+    # Quote the strings so shell will not expand any special chars and prepend ./ if needed
+    # Returns:
+    #   strings quoted with \ as needed by the shell and ./ if needed
+    my @strings = shell_quote(@_);
+    for my $a (@strings) {
+	if(defined $a) {
+	    if($a =~ m:^/: or m:\./:) {
+		# /abs/path or ./rel/path => skip
+	    } else {
+		# rel/path => ./rel/path
+		$a = "./".$a;
+	    }
+	}
+    }
+    return @strings;;
+}
+
 sub maybe_quote {
     # If $Global::quoting then quote the string so shell will not expand any special chars
     # Else do not quote
@@ -1990,20 +2029,18 @@ sub setup_basefile {
     # This needs to be done before first jobs on $sshlogin is run
     # Returns: N/A
     my $cmd = "";
+    my $rsync_destdir;
+    my $workdir;
     for my $sshlogin (values %Global::host) {
-        if($sshlogin->string() eq ":") { next }
+      if($sshlogin->string() eq ":") { next }
-        my $sshcmd = $sshlogin->sshcommand();
+      for my $file (@opt::basefile) {
-        my $serverlogin = $sshlogin->serverlogin();
+	if($file !~ m:^/: and $opt::workdir eq "...") {
-        my $rsync_opt = "-rlDzR -e".shell_quote_scalar($sshcmd);
+	  ::error("Work dir '...' will not work with relative basefiles\n");
-        for my $file (@opt::basefile) {
+	  ::wait_and_exit(255);
-            my $f = $file;
+	}
-            my $relpath = ($f !~ m:^/:); # Is the path relative?
+	$workdir ||= Job->new("")->workdir();
-            # Use different subdirs depending on abs or rel path
+	$cmd .= $sshlogin->rsync_transfer_cmd($file,$workdir) . "&";
-            my $rsync_destdir = ($relpath ? "./" : "/");
+      }
-            $f =~ s:/\./:/:g; # Rsync treats /./ special. We dont want that
-            $f = shell_quote_scalar($f);
-            $cmd .= "rsync $rsync_opt $f $serverlogin:$rsync_destdir &";
-        }
     }
     $cmd .= "wait;";
     debug("basesetup: $cmd\n");
@@ -2014,12 +2051,11 @@ sub cleanup_basefile {
     # Remove the basefiles transferred
     # Returns: N/A
     my $cmd="";
+    my $workdir = Job->new("")->workdir();
     for my $sshlogin (values %Global::host) {
         if($sshlogin->string() eq ":") { next }
-        my $sshcmd = $sshlogin->sshcommand();
-        my $serverlogin = $sshlogin->serverlogin();
         for my $file (@opt::basefile) {
-            $cmd .= "$sshcmd $serverlogin rm -f ".shell_quote_scalar(shell_quote_scalar($file))."&";
+	  $cmd .= $sshlogin->cleanup_cmd($file,$workdir)."&";
         }
     }
     $cmd .= "wait;";
@@ -3627,6 +3663,73 @@ sub control_path_dir {
 }
 
 
+sub rsync_transfer_cmd {
+  # Command to run to transfer a file
+  # Input:
+  #   $file = filename of file to transfer
+  #   $workdir = destination dir
+  # Returns:
+  #   $cmd = rsync command to run to transfer $file ("" if unreadable)
+  my $self = shift;
+  my $file = shift;
+  my $workdir = shift;
+  if(not -r $file) {
+    ::warning($file, " is not readable and will not be transferred.\n");
+    return "true";
+  }
+  my $rsync_destdir;
+  if($file =~ m:^/:) {
+    # rsync /foo/bar /
+    $rsync_destdir = "/";
+  } else {
+    $rsync_destdir = ::shell_quote_file($workdir);
+  }
+  $file = ::shell_quote_file($file);
+  my $sshcmd = $self->sshcommand();
+  my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd);
+  my $serverlogin = $self->serverlogin();
+  # Make dir if it does not exist
+  return "( $sshcmd $serverlogin mkdir -p $rsync_destdir;" .
+    "rsync $rsync_opt $file $serverlogin:$rsync_destdir )";
+}
+
+sub cleanup_cmd {
+  # Command to run to remove the remote file
+  # Input:
+  #   $file = filename to remove
+  #   $workdir = destination dir
+  # Returns:
+  #   $cmd = ssh command to run to remove $file and empty parent dirs
+  my $self = shift;
+  my $file = shift;
+  my $workdir = shift;
+  my $f = $file;
+  if($f =~ m:/\./:) {
+      # foo/bar/./baz/quux => workdir/baz/quux
+      # /foo/bar/./baz/quux => workdir/baz/quux
+      $f =~ s:.*/\./:$workdir/:;
+  } elsif($f =~ m:^[^/]:) {
+      # foo/bar => workdir/foo/bar
+      $f = $workdir."/".$f;
+  }
+  my @subdirs = split m:/:, ::dirname($f);
+  my @rmdir;
+  my $dir = "";
+  for(@subdirs) {
+    $dir .= $_."/";
+    unshift @rmdir, ::shell_quote_file($dir);
+  }
+  my $rmdir = @rmdir ? "rmdir @rmdir 2>/dev/null;" : "";
+  if(defined $opt::workdir and $opt::workdir eq "...") {
+    $rmdir .= "rm -rf " . ::shell_quote_file($workdir).';';
+  }
+
+  $f = ::shell_quote_file($f);
+  my $sshcmd = $self->sshcommand();
+  my $serverlogin = $self->serverlogin();
+  return "$sshcmd $serverlogin ".::shell_quote_scalar("(rm -f $f; $rmdir)");
+}
+
 package JobQueue;
 
 sub new {
@@ -4125,9 +4228,9 @@ sub sshlogin_wrap {
 		$remote_pre .= ::shell_quote_scalar('tty >/dev/null && stty isig -onlcr -echo;');
 	    }
 	    if($opt::workdir) {
-		$remote_pre .= 
+		my $wd = ::shell_quote_file($self->workdir());
-		    ::shell_quote_scalar("mkdir -p ".$self->workdir()."; "
+		$remote_pre .= ::shell_quote_scalar("mkdir -p ") . $wd .
-					 . "cd ".$self->workdir()." && ");
+		    ::shell_quote_scalar("; cd ") . $wd . ::shell_quote_scalar(" && ");
 	    }
 	    $self->{'sshlogin_wrap'} =
 		($pre
@@ -4142,6 +4245,8 @@ sub sshlogin_wrap {
 
 sub transfer {
     # Files to transfer
+    # Returns:
+    #   @transfer - File names of files to transfer
     my $self = shift;
     my @transfer = ();
     $self->{'transfersize'} = 0;
@@ -4166,36 +4271,16 @@ sub transfersize {
 }
 
 sub sshtransfer {
+  # Returns for each transfer file:
+  #   rsync $file remote:$workdir
     my $self = shift;
+    my @pre;
     my $sshlogin = $self->sshlogin();
-    my $sshcmd = $sshlogin->sshcommand();
+    my $workdir = $self->workdir();
-    my $serverlogin = $sshlogin->serverlogin();
-    my $rsync_opt = "-rlDzR -e".::shell_quote_scalar($sshcmd);
-    my $pre = "";
     for my $file ($self->transfer()) {
-	$file =~ s:/\./:/:g; # Rsync treats /./ special. We dont want that
+      push @pre, $sshlogin->rsync_transfer_cmd($file,$workdir).";";
-	$file =~ s:^\./::g; # Remove ./ if any
-	my $relpath = ($file !~ m:^/:); # Is the path relative?
-	# Use different subdirs depending on abs or rel path
-	# Abs path: rsync -rlDzR /home/tange/dir/subdir/file.gz server:/
-	# Rel path: rsync -rlDzR ./subdir/file.gz server:.parallel/tmp/tempid/
-	# Rel path: rsync -rlDzR ./subdir/file.gz server:$workdir/
-	my $remote_workdir = $self->workdir($file);
-	my $rsync_destdir = ($relpath ? $remote_workdir : "/");
-	if($relpath) {
-	    $file = "./".$file;
-	}
-	if(-r $file) {
-	    my $mkremote_workdir =
-		$remote_workdir eq "." ? "true" :
-		"$sshcmd $serverlogin mkdir -p $rsync_destdir";
-	    $pre .= "$mkremote_workdir; rsync $rsync_opt "
-		. ::shell_quote_scalar($file)." $serverlogin:$rsync_destdir;";
-	} else {
-	    ::warning($file, " is not readable and will not be transferred.\n");
-	}
     }
-    return $pre;
+    return join("",@pre);
 }
 
 sub return {
@@ -4226,6 +4311,8 @@ sub returnsize {
 }
 
 sub sshreturn {
+  # Returns for each return-file:
+  #   rsync remote:$workdir/$file .
     my $self = shift;
     my $sshlogin = $self->sshlogin();
     my $sshcmd = $sshlogin->sshcommand();
@@ -4242,13 +4329,13 @@ sub sshreturn {
 	    #   rsync -avR /foo/./bar/baz.c remote:/tmp/
 	    # == (on old systems)
 	    #   rsync -avR --rsync-path="cd /foo; rsync" remote:bar/baz.c /tmp/
-	    $wd = ::shell_quote_scalar($self->workdir())."/";
+	    $wd = ::shell_quote_file($self->workdir()."/");
 	}
 	# Only load File::Basename if actually needed
 	$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;";
-	$cd = ::shell_quote_scalar(dirname($file));
+	$cd = ::shell_quote_file(dirname($file));
 	my $rsync_cd = '--rsync-path='.::shell_quote_scalar("cd $wd$cd; rsync");
-	my $basename = ::shell_quote_scalar(::shell_quote_scalar(basename($file)));
+	my $basename = ::shell_quote_scalar(::shell_quote_file(basename($file)));
 	# --return
 	#   mkdir -p /home/tange/dir/subdir/; 
         #   rsync -rlDzR --rsync-path="cd /home/tange/dir/subdir/; rsync" 
@@ -4268,28 +4355,14 @@ sub sshcleanup {
     my $sshcmd = $sshlogin->sshcommand();
     my $serverlogin = $sshlogin->serverlogin();
     my $workdir = $self->workdir();
-    my $removeworkdir = "";
     my $cleancmd = "";
 
     for my $file ($self->cleanup()) {
 	my @subworkdirs = parentdirs_of($file);
-	if(@subworkdirs) {
+	$cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";";
-	    $removeworkdir = "rmdir 2>/dev/null ".
-		join(" ",map { ::shell_quote_scalar($workdir."/".$_) }
-		     @subworkdirs).";";
-	}
-	my $relpath = ($file !~ m:^/:); # Is the path relative?
-	my $cleandir = ($relpath ? $workdir."/" : "");
-	$cleancmd .= 
-	    ::shell_quote_scalar("rm -f "
-				 . ::shell_quote_scalar($cleandir.$file).';'
-				 . $removeworkdir);
     }
     if(defined $opt::workdir and $opt::workdir eq "...") {
-	$cleancmd .= "rm -rf " . ::shell_quote_scalar($workdir).'\;';
+	$cleancmd .= "$sshcmd $serverlogin rm -rf " . ::shell_quote_scalar($workdir).';';
-    }
-    if($cleancmd) {
-	$cleancmd = "$sshcmd $serverlogin ".$cleancmd.";";
     }
     return $cleancmd;
 }

src/parallel.pod

@@ -301,9 +301,9 @@ string that is not in the command line.
 See also: B<:::>.
 
 
-=item B<--basefile> I<file>
+=item B<--basefile> I<file> (alpha testing)
 
-=item B<--bf> I<file>
+=item B<--bf> I<file> (alpha testing)
 
 I<file> will be transferred to each sshlogin before a jobs is
 started. It will be removed if B<--cleanup> is active. The file may be
@@ -361,7 +361,7 @@ I<size> defaults to 1M.
 See B<--pipe> for use of this.
 
 
-=item B<--cleanup>
+=item B<--cleanup> (alpha testing)
 
 Remove transferred files. B<--cleanup> will remove the transferred files
 on the remote computer after processing is done.
@@ -430,7 +430,7 @@ occurs as a line of input, the rest of the input is ignored.  If
 neither B<-E> nor B<-e> is used, no end of file string is used.
 
 
-=item B<--delay> I<secs> (beta testing)
+=item B<--delay> I<secs>
 
 Delay starting next job I<secs> seconds. GNU B<parallel> will pause
 I<secs> seconds after starting each job. I<secs> can be less than 1
@@ -456,7 +456,7 @@ If I<eof-str> is omitted, there is no end of file string.  If neither
 B<-E> nor B<-e> is used, no end of file string is used.
 
 
-=item B<--env> I<var> (beta testing)
+=item B<--env> I<var>
 
 Copy environment variable I<var>. This will copy I<var> to the
 environment that the command is run in. This is especially useful for
@@ -489,7 +489,7 @@ See also B<--bg>, B<man sem>.
 Implies B<--semaphore>.
 
 
-=item B<--filter-hosts> (beta testing)
+=item B<--filter-hosts>
 
 Remove down hosts. For each remote host: check that login through ssh
 works. If not: do not use this host.
@@ -551,7 +551,7 @@ status will be the exit status from the failing job.
 =back
 
 
-=item B<--header> I<regexp> (beta testing)
+=item B<--header> I<regexp>
 
 Use regexp as header. For normal usage the matched header (typically
 the first line: B<--header '.*\n'>) will be split using B<--colsep>
@@ -579,7 +579,7 @@ specified, and for B<-I>{} otherwise.  This option is deprecated;
 use B<-I> instead.
 
 
-=item B<--joblog> I<logfile> (alpha testing)
+=item B<--joblog> I<logfile> (beta testing)
 
 Logfile for executed jobs. Save a list of the executed jobs to
 I<logfile> in the following TAB separated format: sequence number,
@@ -705,7 +705,7 @@ B<-l 0> is an alias for B<-l 1>.
 Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
 
 
-=item B<--line-buffer> (beta testing)
+=item B<--line-buffer>
 
 Buffer output on line basis. B<--group> will keep the output together
 for a whole job. B<--ungroup> allows output to mixup with half a line
@@ -774,7 +774,7 @@ This is useful for scripts that depend on features only available from
 a certain version of GNU B<parallel>.
 
 
-=item B<--nonall> (alpha testing)
+=item B<--nonall> (beta testing)
 
 B<--onall> with no arguments. Run the command on all computers given
 with B<--sshlogin> but take no arguments. GNU B<parallel> will log
@@ -785,7 +785,7 @@ This is useful for running the same command (e.g. uptime) on a list of
 servers.
 
 
-=item B<--onall> (alpha testing)
+=item B<--onall> (beta testing)
 
 Run all the jobs on all computers given with B<--sshlogin>. GNU
 B<parallel> will log into B<--jobs> number of computers in parallel
@@ -797,19 +797,19 @@ When using B<--group> the output will be grouped by each server, so
 all the output from one server will be grouped together.
 
 
-=item B<--output-as-files> (alpha testing)
+=item B<--output-as-files> (beta testing)
 
-=item B<--outputasfiles> (alpha testing)
+=item B<--outputasfiles> (beta testing)
 
-=item B<--files> (alpha testing)
+=item B<--files> (beta testing)
 
 Instead of printing the output to stdout (standard output) the output
 of each job is saved in a file and the filename is then printed.
 
 
-=item B<--pipe> (beta testing)
+=item B<--pipe>
 
-=item B<--spreadstdin> (beta testing)
+=item B<--spreadstdin>
 
 Spread input to jobs on stdin (standard input). Read a block of data
 from stdin (standard input) and give one block of data as input to one
@@ -967,7 +967,7 @@ If the stdin (standard input) only contains whitespace, do not run the command.
 If used with B<--pipe> this is slow.
 
 
-=item B<--record-env> (beta testing)
+=item B<--record-env>
 
 Record current environment variables in ~/.parallel/ignored_vars. This
 is useful before using B<--env _>.
@@ -1015,9 +1015,9 @@ it to the command.
 Only used with B<--pipe>.
 
 
-=item B<--results> I<prefix> (beta testing)
+=item B<--results> I<prefix>
 
-=item B<--res> I<prefix> (beta testing)
+=item B<--res> I<prefix>
 
 Save the output into files. The files will be stored in a directory tree
 rooted at I<prefix>.  Within this directory tree, each command will result
@@ -1091,7 +1091,7 @@ re-use the computers. This is useful if some jobs fail for no apparent
 reason (such as network failure).
 
 
-=item B<--return> I<filename>
+=item B<--return> I<filename> (alpha testing)
 
 Transfer files from remote computers. B<--return> is used with
 B<--sshlogin> when the arguments are files on the remote computers. When
@@ -1126,9 +1126,9 @@ B<--return> is ignored when used with B<--sshlogin :> or when not used
 with B<--sshlogin>.
 
 
-=item B<--round-robin> (beta testing)
+=item B<--round-robin>
 
-=item B<--round> (beta testing)
+=item B<--round>
 
 Normally B<--pipe> will give a single block to each instance of the
 command. With B<--round-robin> all blocks will at random be written to
@@ -1293,7 +1293,7 @@ be needed.
 
 An I<sshlogin> is of the form:
 
-  [sshcommand [options]][username@]hostname
+  [sshcommand [options]] [username@]hostname
 
 The sshlogin must not require a password.
 
@@ -1318,9 +1318,9 @@ B<--sshlogin> is often used with B<--transfer>, B<--return>,
 B<--cleanup>, and B<--trc>.
 
 
-=item B<--sshloginfile> I<filename> (beta testing)
+=item B<--sshloginfile> I<filename>
 
-=item B<--slf> I<filename> (beta testing)
+=item B<--slf> I<filename>
 
 File with sshlogins. The file consists of sshlogins on separate
 lines. Empty lines and lines starting with '#' are ignored. Example:
@@ -1412,7 +1412,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
 setting $TMPDIR.
 
 
-=item B<--timeout> I<val> (beta testing)
+=item B<--timeout> I<val>
 
 Time out for command. If the command runs for longer than I<val>
 seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms
@@ -1445,7 +1445,7 @@ Print the job to be run on stderr (standard error).
 See also B<-v>, B<-p>.
 
 
-=item B<--transfer>
+=item B<--transfer> (alpha testing)
 
 Transfer files to remote computers. B<--transfer> is used with
 B<--sshlogin> when the arguments are files and should be transferred to
@@ -1557,9 +1557,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely.
 Print the version GNU B<parallel> and exit.
 
 
-=item B<--workdir> I<mydir> (beta testing)
+=item B<--workdir> I<mydir>
 
-=item B<--wd> I<mydir> (beta testing)
+=item B<--wd> I<mydir>
 
 Files transferred using B<--transfer> and B<--return> will be relative
 to I<mydir> on remote computers, and the command will be executed in
@@ -2163,6 +2163,23 @@ To run commands on a remote computer SSH needs to be set up and you
 must be able to login without entering a password (The commands
 B<ssh-copy-id> and B<ssh-agent> may help you do that).
 
+If you need to login to a whole cluster, you typically do not want to
+accept the host key for every host. You want to accept them the first
+time and be warned if they are ever changed. To do that:
+
+  # Add the servers to the sshloginfile
+  (echo servera; echo serverb) > .parallel/my_cluster
+  # Make sure .ssh/config exist
+  touch .ssh/config
+  cp .ssh/config .ssh/config.backup
+  # Disable StrictHostKeyChecking temporarily
+  (echo 'Host *'; echo StrictHostKeyChecking no) >> .ssh/config
+  parallel --slf my_cluster --nonall true
+  # Remove the disabling of StrictHostKeyChecking
+  mv .ssh/config.backup .ssh/config 
+
+The servers in B<.parallel/my_cluster> are now added in B<.ssh/known_hosts>.
+
 To run B<echo> on B<server.example.com>:
 
   seq 10 | parallel --sshlogin server.example.com echo

src/parallel.texi

@@ -1394,7 +1394,7 @@ be needed.
 An @emph{sshlogin} is of the form:
 
 @verbatim
-  [sshcommand [options]][username@]hostname
+  [sshcommand [options]] [username@]hostname
 @end verbatim
 
 The sshlogin must not require a password.
@@ -2317,6 +2317,25 @@ To run commands on a remote computer SSH needs to be set up and you
 must be able to login without entering a password (The commands
 @strong{ssh-copy-id} and @strong{ssh-agent} may help you do that).
 
+If you need to login to a whole cluster, you typically do not want to
+accept the host key for every host. You want to accept them the first
+time and be warned if they are ever changed. To do that:
+
+@verbatim
+  # Add the servers to the sshloginfile
+  (echo servera; echo serverb) > .parallel/my_cluster
+  # Make sure .ssh/config exist
+  touch .ssh/config
+  cp .ssh/config .ssh/config.backup
+  # Disable StrictHostKeyChecking temporarily
+  (echo 'Host *'; echo StrictHostKeyChecking no) >> .ssh/config
+  parallel --slf my_cluster --nonall true
+  # Remove the disabling of StrictHostKeyChecking
+  mv .ssh/config.backup .ssh/config 
+@end verbatim
+
+The servers in @strong{.parallel/my_cluster} are now added in @strong{.ssh/known_hosts}.
+
 To run @strong{echo} on @strong{server.example.com}:
 
 @verbatim

src/sql

@@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20130922;
+    $Global::version = 20131022;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/parallel-local-ssh1.sh

@@ -1,21 +1,22 @@
 #!/bin/bash
 
+rm -rf tmp
+mkdir tmp
+cd tmp
+
 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j10 -k -L1
 echo '### Test bug #34241: --pipe should not spawn unneeded processes'
   seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
 
 echo '### --env _'
-  fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DISPLAY' ::: test
+  fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
-echo '### --env _ with explicit mentioning of normally ignored var $DISPLAY'
+echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
-  fUbAr="OK FUBAR" parallel -S csh@lo --env DISPLAY,_ echo '$fUbAr $DISPLAY' ::: test
+  fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
 
 echo 'bug #40137: SHELL not bash: Warning when exporting funcs'
   . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning
   . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning
 
-echo '### --filter-hosts --slf <()'
-  parallel --nonall --filter-hosts --slf <(echo localhost) echo OK
-
 echo '### bug #40002: --files and --nonall seem not to work together:'
   parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l
 
@@ -25,4 +26,77 @@ echo '### bug #40001: --joblog and --nonall seem not to work together:'
 echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME'
   cd && parallel --workdir . -S lo pwd ::: ""
 
+echo '### test filename :'
+  echo content-of-: > :; 
+  echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; 
+  cat :.:; rm : :.:
+
+echo '### Test --wd ... --cleanup which should remove the filled tmp dir'
+  find ~/.parallel/tmp |grep uNiQuE_sTrInG.6 | parallel rm; 
+  stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S lo -v echo ">"{}.6 :::  uNiQuE_sTrInG; 
+  find ~/.parallel/tmp |grep uNiQuE_sTrInG.6
+
+echo '### Test --wd --'
+  stdout parallel --wd -- -S lo echo OK ">"{}.7 ::: uNiQuE_sTrInG; 
+  cat ~/--/uNiQuE_sTrInG.7; 
+  stdout rm ~/--/uNiQuE_sTrInG.7
+
+echo '### Test --wd " "'
+  stdout parallel --wd " " -S lo echo OK ">"{}.8 ::: uNiQuE_sTrInG; 
+  cat ~/" "/uNiQuE_sTrInG.8; 
+  stdout rm ~/" "/uNiQuE_sTrInG.8
+
+echo "### Test --wd \"'\""
+  stdout parallel --wd "'" -S lo echo OK ">"{}.9 ::: uNiQuE_sTrInG; 
+  cat ~/"'"/uNiQuE_sTrInG.9; 
+  stdout rm ~/"'"/uNiQuE_sTrInG.9
+
+echo '### Test --trc --/--foo1'
+  mkdir -p ./--; echo 'Content --/--foo1' > ./--/--foo1; 
+  stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./--/--foo1; cat ./--/--foo1.1; 
+  stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: --/--foo1; cat ./--/--foo1.2
+
+echo '### Test --trc :dir/:foo2'
+  mkdir -p ./:dir; echo 'Content :dir/:foo2' > ./:dir/:foo2; 
+  stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./:dir/:foo2; cat ./:dir/:foo2.1; 
+  stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: :dir/:foo2; cat ./:dir/:foo2.2
+
+echo '### Test --trc " "/" "foo3'
+  mkdir -p ./" "; echo 'Content _/_foo3' > ./" "/" "foo3; 
+  stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1; 
+  stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: " "/" "foo3; cat ./" "/" "foo3.2
+
+#echo '### Test --trc --/./--foo4'
+#  mkdir -p ./--; echo 'Content --/./--foo4' > ./--/./--foo4; 
+#  stdout parallel --trc {}.1 -S lo '(cat ./--foo4; echo remote{}) > --foo4.1' ::: --/./--foo4; cat ./--/./--foo4.1
+#
+#echo '### Test --trc :/:foo2'
+#  mkdir -p ./:; echo 'Content :/:foo2' > ./:/:foo2; 
+#  stdout parallel --trc {}.1 -S lo '(cat {}; echo remote) > {}.1' ::: ./:/:foo2; cat ./:/:foo2.1
+#
+#echo '### Test --trc " "/" "foo3'
+#  mkdir -p ./" "; echo 'Content _/_foo2' > ./" "/" "foo3; 
+#  stdout parallel --trc {}.1 -S lo '(cat {}; echo remote) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1
+#
+#
+#
+#
+#
+#
+#echo '### Test --trc --/./--foo4 :/./:foo5 " "/./" "foo6 ./foo10/./foo10'
+#  mkdir ./--; echo 'Content --/--foo4' > ./--/--foo4; 
+#  mkdir ./:; echo 'Content :/:foo5' > ./:/:foo5; 
+#  mkdir ./" "; echo 'Content _/_foo2' > ./" "/" "foo3; 
+
+### echo '### Test --trc --basefile --/./--foo7 :/./:foo8 " "/./" "foo9 ./foo11/./foo11'
+### echo missing
+### echo '### Test --trc "-- "'
+### echo missing
+### echo '### Test --trc " --"'
+### echo missing
+### 
 EOF
+
+rm -rf tmp
+mkdir tmp
+cd tmp

testsuite/tests-to-run/parallel-local-ssh2.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j8 -k -L1
+echo '### --filter-hosts --slf <()'
+  parallel --nonall --filter-hosts --slf <(echo localhost) echo OK
+
+EOF

testsuite/tests-to-run/test17.sh

@@ -16,9 +16,8 @@ rm -rf /tmp/parallel.file*
 stdout ssh $SSHLOGIN1 rm -rf 'tmp/parallel.file*'  '/tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*'
 # (seq 1 3;echo '>fire';seq 5 10; /bin/echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17
-(echo '>fire';/bin/echo ' : & ) \n*.jpg'; echo '/./sub dir') >/tmp/test17
+(echo '>fire';/bin/echo ' : & ) \n*.jpg') >/tmp/test17
 echo '# Create some weirdly files in /tmp'
-mkdir -p /tmp/parallel.file
 cat /tmp/test17 | parallel -k /bin/echo file{} '>'/tmp/parallel.file{}.file
 cat /tmp/test17 | parallel -k /bin/echo /tmp/parallel.file{}.file >/tmp/test17abs
 cat /tmp/test17 | parallel -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel
@@ -69,14 +68,14 @@ stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*'
 rm -rf /tmp/parallel.file*out
 cat /tmp/test17abs | parallel -k --return {.}.out --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p /tmp/parallel.file";"echo {} ">"{.}.out
-ls /tmp/parallel.file*out /tmp/parallel.file/*out
+stdout ls /tmp/parallel.file*out
 
 echo '### --return - relpath'
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
 rm -rf /tmp/parallel.file*out
 cat /tmp/test17rel | parallel -k --return {.}.out --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out
-ls tmp/parallel.file*out tmp/parallel.file/*out
+stdout ls tmp/parallel.file*out
 
 echo '### --return - multiple files'
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
@@ -84,7 +83,7 @@ stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
 rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
 cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
-ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+stdout ls tmp/parallel.file*out tmp/parallel.file*done
 
 echo '### --return --cleanup - abspath'
 stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*'
@@ -92,19 +91,19 @@ stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*'
 rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
 cat /tmp/test17abs | parallel -k --return {.}.out --return {}.done --cleanup \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p /tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK
 
 echo '### --return --cleanup - relpath'
-rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+rm -rf tmp/parallel.file*out tmp/parallel.file*done
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
 cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
-ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+stdout ls tmp/parallel.file*out tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
@@ -113,35 +112,33 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
 echo '### --return --cleanup - multiple returns'
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
-rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+rm -rf tmp/parallel.file*out tmp/parallel.file*done
 cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 echo good if no file
-stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
+stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*'
-# Should give: No such file or directory
+stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*'
-stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
 
 echo '### --transfer --return --cleanup - abspath'
 stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*'
-rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
 cat /tmp/test17abs | parallel -k --transfer --return {.}.out --return {}.done --cleanup \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 cat {} ">"{.}.out';'cat {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK
 
-
 echo '### --transfer --return --cleanup - relpath'
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
-rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+rm -rf tmp/parallel.file*out tmp/parallel.file*done
 cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
@@ -150,10 +147,10 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
 echo '### --transfer --return --cleanup - multiple files'
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
-rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+rm -rf tmp/parallel.file*out tmp/parallel.file*done
 cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
@@ -161,10 +158,10 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
 echo '### --trc - abspath'
 stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*'
-rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
 cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
@@ -173,10 +170,10 @@ stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK
 echo '### --trc - relpath'
 stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*'
-rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+rm -rf tmp/parallel.file*out tmp/parallel.file*done
 cat /tmp/test17rel | parallel -k --trc {.}.out --trc {}.done \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
-ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
+stdout ls tmp/parallel.file*out tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
@@ -185,10 +182,10 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK
 echo '### --trc - multiple files'
 stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*'
 stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*'
-rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
 cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \
   --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
-ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
+stdout ls /tmp/parallel.file*out /tmp/parallel.file*done
 echo good if no file
 stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK
 # Should give: No such file or directory

testsuite/tests-to-run/test35.sh

@@ -18,15 +18,15 @@ echo '### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs';
   find . -name '*.7' | sort
 
 echo '### Test --workdir ...'
-parallel -j9 -k --workdir ... --basefile 1-col.txt --trc {}.1 -S .. echo ">"{}.1 ::: 2-col.txt
+parallel -j9 -k --workdir ... --trc {}.1 -S .. echo ">"{}.1 ::: 2-col.txt
 find . -name '*.1' | sort
 
 echo '### Test --wd ...'
-parallel -k --wd ... --basefile 1-col.txt --trc {}.2 -S .. -v echo ">"{}.2 ::: 2-col.txt
+parallel -k --wd ... --trc {}.2 -S .. -v echo ">"{}.2 ::: 2-col.txt
 find . -name '*.2' | sort
 
 echo '### Test --wd ... with space dirs'
-stdout parallel -j9 -k --wd ... --basefile 1-col.txt --trc {}.3 -S .. -v echo ">"{}.3 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
+stdout parallel -j9 -k --wd ... --trc {}.3 -S .. -v echo ">"{}.3 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
 # A few rmdir errors are OK as we have multiple files in the same dirs
 find . -name '*.3' | sort
 
@@ -39,9 +39,5 @@ stdout parallel -k -j9 --wd /tmp/ --basefile 1-col.txt --trc {}.5 -S .. -v echo
 # A few rmdir errors are OK as we have multiple files in the same dirs
 find . -name '*.5' | sort
 
-echo '### Test --wd ... --cleanup'
-stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S lo -v echo ">"{}.6 :::  uNiQuE_sTrInG
-find ~/.parallel/tmp |grep uNiQuE_sTrInG
-
 cd ..
 rm -rf tmp

testsuite/wanted-results/parallel-local-ssh1

@@ -7,18 +7,42 @@ Block_end
 5
 Block_end
 ### --env _
-DISPLAY: Undefined variable.
+DEBEMAIL: Undefined variable.
-### --env _ with explicit mentioning of normally ignored var $DISPLAY
+### --env _ with explicit mentioning of normally ignored var $DEBEMAIL
-OK FUBAR :0.0 test
+OK FUBAR ole@tange.dk test
 bug #40137: SHELL not bash: Warning when exporting funcs
 no_warning
 warning
 parallel: Warning: Shell functions may not be supported in /bin/sh
-### --filter-hosts --slf <()
-OK
 ### bug #40002: --files and --nonall seem not to work together:
 1
 ### bug #40001: --joblog and --nonall seem not to work together:
 3
 ### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME
 /home/tange
+### test filename :
+remote-:.:
+content-of-:
+### Test --wd ... --cleanup which should remove the filled tmp dir
+echo >uNiQuE_sTrInG.6
+### Test --wd --
+OK
+### Test --wd " "
+OK
+### Test --wd "'"
+OK
+### Test --trc --/--foo1
+Content --/--foo1
+remote1
+Content --/--foo1
+remote2
+### Test --trc :dir/:foo2
+Content :dir/:foo2
+remote1
+Content :dir/:foo2
+remote2
+### Test --trc " "/" "foo3
+Content _/_foo3
+remote1
+Content _/_foo3
+remote2

testsuite/wanted-results/parallel-local-ssh2

@@ -0,0 +1,2 @@
+### --filter-hosts --slf <()
+OK

testsuite/wanted-results/test13

@@ -18,8 +18,8 @@ begin
 14
 15
 16
-17
 parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help.
+17
 18
 19
 20

testsuite/wanted-results/test17

@@ -3,25 +3,24 @@
 ### --transfer - abspath
 file>fire
 file : & ) \n*.jpg
-file/./sub dir
 good if no file
 ls: /tmp/parallel.file*: No such file or directory
+ls: /tmp/parallel.file*: No such file or directory
 ### --transfer - relpath
 file>fire
 file : & ) \n*.jpg
-file/./sub dir
 good if no file
 ls: tmp/parallel.file*: No such file or directory
+ls: tmp/parallel.file*: No such file or directory
 ### --transfer --cleanup - abspath
 file>fire
 file : & ) \n*.jpg
-file/./sub dir
 good if no file
 ls: /tmp/parallel.file*: No such file or directory
+ls: /tmp/parallel.file*: No such file or directory
 ### --transfer --cleanup - relpath
 file>fire
 file : & ) \n*.jpg
-file/./sub dir
 good if no file
 ls: tmp/parallel.file*: No such file or directory
 OK
@@ -29,62 +28,46 @@ ls: tmp/parallel.file*: No such file or directory
 OK
 ### --return - abspath
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.out
 ### --return - relpath
 tmp/parallel.file : & ) \n*.jpg.out
-tmp/parallel.file/sub dir.out
 tmp/parallel.file>fire.out
 ### --return - multiple files
 tmp/parallel.file : & ) \n*.jpg.file.done
 tmp/parallel.file : & ) \n*.jpg.out
-tmp/parallel.file/sub dir.file.done
-tmp/parallel.file/sub dir.out
 tmp/parallel.file>fire.file.done
 tmp/parallel.file>fire.out
 ### --return --cleanup - abspath
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 good if no file
 ### --return --cleanup - relpath
 tmp/parallel.file : & ) \n*.jpg.file.done
 tmp/parallel.file : & ) \n*.jpg.out
-tmp/parallel.file/sub dir.file.done
-tmp/parallel.file/sub dir.out
 tmp/parallel.file>fire.file.done
 tmp/parallel.file>fire.out
 good if no file
-ls: tmp/parallel.file*: No such file or directory
-OK
 ### --return --cleanup - multiple returns
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 good if no file
-ls: tmp/parallel.file*: No such file or directory
-OK
 ### --transfer --return --cleanup - abspath
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 good if no file
 ls: /tmp/parallel.file*: No such file or directory
 OK
+ls: /tmp/parallel.file*: No such file or directory
+OK
 ### --transfer --return --cleanup - relpath
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 good if no file
@@ -95,8 +78,6 @@ OK
 ### --transfer --return --cleanup - multiple files
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 ls: tmp/parallel.file*: No such file or directory
@@ -106,18 +87,16 @@ OK
 ### --trc - abspath
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 good if no file
 ls: /tmp/parallel.file*: No such file or directory
 OK
+ls: /tmp/parallel.file*: No such file or directory
+OK
 ### --trc - relpath
 tmp/parallel.file : & ) \n*.jpg.file.done
 tmp/parallel.file : & ) \n*.jpg.out
-tmp/parallel.file/sub dir.file.done
-tmp/parallel.file/sub dir.out
 tmp/parallel.file>fire.file.done
 tmp/parallel.file>fire.out
 good if no file
@@ -128,20 +107,19 @@ OK
 ### --trc - multiple files
 /tmp/parallel.file : & ) \n*.jpg.file.done
 /tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
 /tmp/parallel.file>fire.file.done
 /tmp/parallel.file>fire.out
 good if no file
 ls: /tmp/parallel.file*: No such file or directory
 OK
+ls: /tmp/parallel.file*: No such file or directory
+OK
 ### --transfer --cleanup - multiple argument files
 cat /tmp/parallel.file\>fire.file tmp/parallel.file\>fire.file
 file>fire
 file>fire
-cat /tmp/parallel.file/./sub\ dir.file tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file
+cat /tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file
-file/./sub dir
 file : & ) \n*.jpg
-cat /tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file tmp/parallel.file/./sub\ dir.file
 file : & ) \n*.jpg
-file/./sub dir
+ls: /tmp/parallel.file*: No such file or directory
+OK

testsuite/wanted-results/test19

@@ -75,31 +75,37 @@ OK
 ls: tmp/parallel.file*: No such file or directory
 OK
 Input for ssh
--l parallel parallel-server1 rsync --server -lDrRze.iLsf . .
+parallel@parallel-server1 mkdir -p ./.
+-l parallel parallel-server1 rsync --server -lDrRze.iLsf . ./.
 -tt -oLogLevel=quiet parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ 2\;  setenv PARALLEL_PID 00000  || echo PARALLEL_SEQ=2\;export PARALLEL_SEQ\;  PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.'
 'newline2 > tmp/parallel.file.'
 'newline2.out;cat tmp/parallel.file.'
 'newline2 > tmp/parallel.file.'
 'newline2.out2
--l parallel parallel-server1 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.'
+-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.'
 'newline2.out
--l parallel parallel-server1 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.'
+-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.'
 'newline2.out2
-parallel@parallel-server1 rm -f ./tmp/parallel.file.'
+parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
-'newline2;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.'
+'newline2; rmdir ./tmp/ ./ 2>/dev/null;)
-'newline2.out;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.'
+parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
-'newline2.out2;rmdir 2>/dev/null ./tmp;
+'newline2.out; rmdir ./tmp/ ./ 2>/dev/null;)
--l parallel parallel-server2 rsync --server -lDrRze.iLsf . .
+parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
+'newline2.out2; rmdir ./tmp/ ./ 2>/dev/null;)
+parallel@parallel-server2 mkdir -p ./.
+-l parallel parallel-server2 rsync --server -lDrRze.iLsf . ./.
 -tt -oLogLevel=quiet parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ 1\;  setenv PARALLEL_PID 00000  || echo PARALLEL_SEQ=1\;export PARALLEL_SEQ\;  PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.'
 'newline1 > tmp/parallel.file.'
 'newline1.out;cat tmp/parallel.file.'
 'newline1 > tmp/parallel.file.'
 'newline1.out2
--l parallel parallel-server2 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.'
+-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.'
 'newline1.out
--l parallel parallel-server2 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.'
+-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.'
 'newline1.out2
-parallel@parallel-server2 rm -f ./tmp/parallel.file.'
+parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
-'newline1;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.'
+'newline1; rmdir ./tmp/ ./ 2>/dev/null;)
-'newline1.out;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.'
+parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
-'newline1.out2;rmdir 2>/dev/null ./tmp;
+'newline1.out; rmdir ./tmp/ ./ 2>/dev/null;)
+parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
+'newline1.out2; rmdir ./tmp/ ./ 2>/dev/null;)

testsuite/wanted-results/test35

@@ -66,5 +66,3 @@ echo >./a\ b/cd\ /\ ef/efg.5
 ./a b/cd / ef/efg.5
 ./b/bar.5
 ./b/foo.5
-### Test --wd ... --cleanup
-echo >uNiQuE_sTrInG.6

testsuite/wanted-results/test61

@@ -1,5 +1,5 @@
 ### Test --return of weirdly named file
-ssh -tt -oLogLevel=quiet parallel@parallel-server3  'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\;  setenv PARALLEL_PID '$PARALLEL_PID'  || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;  PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;'  tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p .; rsync --rsync-path=cd\ ./.\;\ rsync -rlDzR -essh parallel@parallel-server3:aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b .; exit $_EXIT_status;
+ssh -tt -oLogLevel=quiet parallel@parallel-server3  'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\;  setenv PARALLEL_PID '$PARALLEL_PID'  || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;  PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;'  tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
 ### Test if remote login shell is csh
 ssh -tt -oLogLevel=quiet csh@localhost  'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\;  setenv PARALLEL_PID '$PARALLEL_PID'  || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;  PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;'  tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$PARALLEL_PID\ \$PARALLEL_SEQ\ a\|\ wc\ -w;
 3