diff --git a/NEWS b/NEWS index 517716e3..d1fd9efb 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,67 @@ +20131022 + +* --transfer files with /./ in the path will copy the files relative + to the --workdir. + +* The maximal command length is now cached in a file halfing the + startup time. + +* GNU Parallel was cited in: Investigating speaker gender using rhythm + metrics in Arabic dialects + http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=6602389&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D6602389 + +* GNU Parallel was cited in: Tiedon erist ̈minen ja visualisointi a + lastensuojelun asiakaskertomuksista + https://tampub.uta.fi/handle/10024/84978 + +* Some Additional Useful Bioinformatics Software + http://nix-bio.blogspot.dk/2013/10/additional-software-for-bioinformatics.html + +* GNU Parallel Example Screen Cast (bil to Geotiff conversion) + http://www.youtube.com/watch?v=PpF27eC7Mys + +* Using GNU Parallel to roll-your-own Map Reduce! + http://www.rankfocus.com/hello-world/ + +* Use multiple CPU Cores with your Linux commands + http://www.rankfocus.com/use-cpu-cores-linux-commands/ + +* Using GNU Parallel at HPC @ Uni.lu + https://hpc.uni.lu/users/use_cases/ + +* Scaling up with parallelization + https://www.msi.umn.edu/sites/default/files/AdvPython_1.pdf + +* How to teach yourself to be a technical tester + http://infiniteundo.com/post/62421363463/qa-toc + +* Faster Rasters For All + http://2013.foss4g.org/conf/programme/presentations/52/ + +* Optimizing translated file downloads + http://www.smartling.com/blog/2013/05/20/optimizing-translated-file-downloads/ + +* EMC NetWorker parallel saveset cloning with nsrclone and GNU parallel + http://www.beebotech.com.au/2013/10/emc-networker-parallel-saveset-cloning-with-nsrclone-and-gnu-parallel/ + +* Best Practices for Amazon EMR + http://media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf + +* Computing checksums for backup + https://identi.ca/evan/note/6yf1GzAARtyBhj__xzMvAg + +* Un exemplu de utilizare a programului GNU parallel + http://vundicind.blogspot.dk/2013/10/exemplu-utilizare-gnu-parallel.html + +* 平行化你的工作 + http://www.slideshare.net/drakeguan/part1-23705978 + +* なんか気持ちよかです Gnu Parallel + http://a3-lab.jpn.org/note/%E3%81%AA%E3%82%93%E3%81%8B%E6%B0%97%E6%8C%81%E3%81%A1%E3%82%88%E3%81%8B%E3%81%A7%E3%81%99-gnu-parallel + +* Bug fixes and man page updates. + + 20130922 * PDF-files of documentation is now included. diff --git a/README b/README index 12d4c260..22540790 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20130922.tar.bz2 - bzip2 -dc parallel-20130922.tar.bz2 | tar xvf - - cd parallel-20130922 + wget http://ftpmirror.gnu.org/parallel/parallel-20131022.tar.bz2 + bzip2 -dc parallel-20131022.tar.bz2 | tar xvf - + cd parallel-20131022 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20130922.tar.bz2 - bzip2 -dc parallel-20130922.tar.bz2 | tar xvf - - cd parallel-20130922 + wget http://ftpmirror.gnu.org/parallel/parallel-20131022.tar.bz2 + bzip2 -dc parallel-20131022.tar.bz2 | tar xvf - + cd parallel-20131022 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index d38e72d0..ae0910f0 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.68 for parallel 20130922. +# Generated by GNU Autoconf 2.68 for parallel 20131022. # # Report bugs to . # @@ -559,8 +559,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20130922' -PACKAGE_STRING='parallel 20130922' +PACKAGE_VERSION='20131022' +PACKAGE_STRING='parallel 20131022' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1176,7 +1176,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20130922 to adapt to many kinds of systems. +\`configure' configures parallel 20131022 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1242,7 +1242,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20130922:";; + short | recursive ) echo "Configuration of parallel 20131022:";; esac cat <<\_ACEOF @@ -1309,7 +1309,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20130922 +parallel configure 20131022 generated by GNU Autoconf 2.68 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1326,7 +1326,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20130922, which was +It was created by parallel $as_me 20131022, which was generated by GNU Autoconf 2.68. Invocation command line was $ $0 $@ @@ -2141,7 +2141,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20130922' + VERSION='20131022' cat >>confdefs.h <<_ACEOF @@ -2704,7 +2704,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20130922, which was +This file was extended by parallel $as_me 20131022, which was generated by GNU Autoconf 2.68. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2766,7 +2766,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20130922 +parallel config.status 20131022 configured by $0, generated by GNU Autoconf 2.68, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 16df0792..25cb2771 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20130922], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20131022], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 2a646a56..4bd0bdcf 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -208,14 +208,31 @@ available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: +* --transfer files with /./ in the path will copy the files relative + to the --workdir. + +* The maximal command length is now cached in a file halfing the + startup time. + +* GNU Parallel was cited in: Investigating speaker gender using rhythm + metrics in Arabic dialects + http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=6602389&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D6602389 + +* GNU Parallel was cited in: Tiedon erist ̈minen ja visualisointi a + lastensuojelun asiakaskertomuksista + https://tampub.uta.fi/handle/10024/84978 + +* Some Additional Useful Bioinformatics Software + http://nix-bio.blogspot.dk/2013/10/additional-software-for-bioinformatics.html + +* GNU Parallel Example Screen Cast (bil to Geotiff conversion) + http://www.youtube.com/watch?v=PpF27eC7Mys + * Using GNU Parallel to roll-your-own Map Reduce! http://www.rankfocus.com/hello-world/ -* 平行化你的工作 - http://www.slideshare.net/drakeguan/part1-23705978 - -* Best Practices for Amazon EMR - http://media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf +* Use multiple CPU Cores with your Linux commands + http://www.rankfocus.com/use-cpu-cores-linux-commands/ * Using GNU Parallel at HPC @ Uni.lu https://hpc.uni.lu/users/use_cases/ @@ -223,14 +240,33 @@ New in this release: * Scaling up with parallelization https://www.msi.umn.edu/sites/default/files/AdvPython_1.pdf -* Optimizing translated file downloads - http://www.smartling.com/blog/2013/05/20/optimizing-translated-file-downloads/ - -https://identi.ca/evan/note/6yf1GzAARtyBhj__xzMvAg +* How to teach yourself to be a technical tester + http://infiniteundo.com/post/62421363463/qa-toc * Faster Rasters For All http://2013.foss4g.org/conf/programme/presentations/52/ +* Optimizing translated file downloads + http://www.smartling.com/blog/2013/05/20/optimizing-translated-file-downloads/ + +* EMC NetWorker parallel saveset cloning with nsrclone and GNU parallel + http://www.beebotech.com.au/2013/10/emc-networker-parallel-saveset-cloning-with-nsrclone-and-gnu-parallel/ + +* Best Practices for Amazon EMR + http://media.amazonwebservices.com/AWS_Amazon_EMR_Best_Practices.pdf + +* Computing checksums for backup + https://identi.ca/evan/note/6yf1GzAARtyBhj__xzMvAg + +* Un exemplu de utilizare a programului GNU parallel + http://vundicind.blogspot.dk/2013/10/exemplu-utilizare-gnu-parallel.html + +* 平行化你的工作 + http://www.slideshare.net/drakeguan/part1-23705978 + +* なんか気持ちよかです Gnu Parallel + http://a3-lab.jpn.org/note/%E3%81%AA%E3%82%93%E3%81%8B%E6%B0%97%E6%8C%81%E3%81%A1%E3%82%88%E3%81%8B%E3%81%A7%E3%81%99-gnu-parallel + * Bug fixes and man page updates. diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 1dd5fc2b..4b930b8b 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20130922 +Version: 20131022 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index 13f83191..0b3bb41f 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20130922; +$Global::version = 20131022; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 66dd0da1..1a5cade7 100755 --- a/src/parallel +++ b/src/parallel @@ -32,6 +32,7 @@ use File::Path; use Getopt::Long; # Used to ensure code quality use strict; +use File::Basename; if(not $ENV{SHELL}) { # $ENV{SHELL} is sometimes not set on Mac OS X and Windows @@ -184,7 +185,7 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { ::die_bug("host check too many col0: $_"); } } else { - ::die_bug("host check unmatched short jobline: $_"); + ::die_bug("host check unmatched short jobline ($col[0]): $_"); } } close $host_fh; @@ -207,6 +208,10 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { } if($opt::nonall or $opt::onall) { + +# TODO --transfer foo/./bar --cleanup +# multiple --transfer and --basefile with different /./ + sub tmp_joblog { my $joblog = shift; if(not defined $joblog) { @@ -790,7 +795,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20130927; + $Global::version = 20131022; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1357,6 +1362,40 @@ sub shell_quote_scalar { return $a; } +sub shell_quote_file { + # Quote the string so shell will not expand any special chars and prepend ./ if needed + # Returns: + # string quoted with \ as needed by the shell and ./ if needed + my $a = shell_quote_scalar(shift); + if(defined $a) { + if($a =~ m:^/: or $a =~ m:^\./:) { + # /abs/path or ./rel/path => skip + } else { + # rel/path => ./rel/path + $a = "./".$a; + } + } + return $a; +} + +sub shell_quote_files { + # Quote the strings so shell will not expand any special chars and prepend ./ if needed + # Returns: + # strings quoted with \ as needed by the shell and ./ if needed + my @strings = shell_quote(@_); + for my $a (@strings) { + if(defined $a) { + if($a =~ m:^/: or m:\./:) { + # /abs/path or ./rel/path => skip + } else { + # rel/path => ./rel/path + $a = "./".$a; + } + } + } + return @strings;; +} + sub maybe_quote { # If $Global::quoting then quote the string so shell will not expand any special chars # Else do not quote @@ -1990,20 +2029,18 @@ sub setup_basefile { # This needs to be done before first jobs on $sshlogin is run # Returns: N/A my $cmd = ""; + my $rsync_destdir; + my $workdir; for my $sshlogin (values %Global::host) { - if($sshlogin->string() eq ":") { next } - my $sshcmd = $sshlogin->sshcommand(); - my $serverlogin = $sshlogin->serverlogin(); - my $rsync_opt = "-rlDzR -e".shell_quote_scalar($sshcmd); - for my $file (@opt::basefile) { - my $f = $file; - my $relpath = ($f !~ m:^/:); # Is the path relative? - # Use different subdirs depending on abs or rel path - my $rsync_destdir = ($relpath ? "./" : "/"); - $f =~ s:/\./:/:g; # Rsync treats /./ special. We dont want that - $f = shell_quote_scalar($f); - $cmd .= "rsync $rsync_opt $f $serverlogin:$rsync_destdir &"; - } + if($sshlogin->string() eq ":") { next } + for my $file (@opt::basefile) { + if($file !~ m:^/: and $opt::workdir eq "...") { + ::error("Work dir '...' will not work with relative basefiles\n"); + ::wait_and_exit(255); + } + $workdir ||= Job->new("")->workdir(); + $cmd .= $sshlogin->rsync_transfer_cmd($file,$workdir) . "&"; + } } $cmd .= "wait;"; debug("basesetup: $cmd\n"); @@ -2014,12 +2051,11 @@ sub cleanup_basefile { # Remove the basefiles transferred # Returns: N/A my $cmd=""; + my $workdir = Job->new("")->workdir(); for my $sshlogin (values %Global::host) { if($sshlogin->string() eq ":") { next } - my $sshcmd = $sshlogin->sshcommand(); - my $serverlogin = $sshlogin->serverlogin(); for my $file (@opt::basefile) { - $cmd .= "$sshcmd $serverlogin rm -f ".shell_quote_scalar(shell_quote_scalar($file))."&"; + $cmd .= $sshlogin->cleanup_cmd($file,$workdir)."&"; } } $cmd .= "wait;"; @@ -3627,6 +3663,73 @@ sub control_path_dir { } +sub rsync_transfer_cmd { + # Command to run to transfer a file + # Input: + # $file = filename of file to transfer + # $workdir = destination dir + # Returns: + # $cmd = rsync command to run to transfer $file ("" if unreadable) + my $self = shift; + my $file = shift; + my $workdir = shift; + if(not -r $file) { + ::warning($file, " is not readable and will not be transferred.\n"); + return "true"; + } + my $rsync_destdir; + if($file =~ m:^/:) { + # rsync /foo/bar / + $rsync_destdir = "/"; + } else { + $rsync_destdir = ::shell_quote_file($workdir); + } + $file = ::shell_quote_file($file); + my $sshcmd = $self->sshcommand(); + my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd); + my $serverlogin = $self->serverlogin(); + # Make dir if it does not exist + return "( $sshcmd $serverlogin mkdir -p $rsync_destdir;" . + "rsync $rsync_opt $file $serverlogin:$rsync_destdir )"; +} + +sub cleanup_cmd { + # Command to run to remove the remote file + # Input: + # $file = filename to remove + # $workdir = destination dir + # Returns: + # $cmd = ssh command to run to remove $file and empty parent dirs + my $self = shift; + my $file = shift; + my $workdir = shift; + my $f = $file; + if($f =~ m:/\./:) { + # foo/bar/./baz/quux => workdir/baz/quux + # /foo/bar/./baz/quux => workdir/baz/quux + $f =~ s:.*/\./:$workdir/:; + } elsif($f =~ m:^[^/]:) { + # foo/bar => workdir/foo/bar + $f = $workdir."/".$f; + } + my @subdirs = split m:/:, ::dirname($f); + my @rmdir; + my $dir = ""; + for(@subdirs) { + $dir .= $_."/"; + unshift @rmdir, ::shell_quote_file($dir); + } + my $rmdir = @rmdir ? "rmdir @rmdir 2>/dev/null;" : ""; + if(defined $opt::workdir and $opt::workdir eq "...") { + $rmdir .= "rm -rf " . ::shell_quote_file($workdir).';'; + } + + $f = ::shell_quote_file($f); + my $sshcmd = $self->sshcommand(); + my $serverlogin = $self->serverlogin(); + return "$sshcmd $serverlogin ".::shell_quote_scalar("(rm -f $f; $rmdir)"); +} + package JobQueue; sub new { @@ -4125,9 +4228,9 @@ sub sshlogin_wrap { $remote_pre .= ::shell_quote_scalar('tty >/dev/null && stty isig -onlcr -echo;'); } if($opt::workdir) { - $remote_pre .= - ::shell_quote_scalar("mkdir -p ".$self->workdir()."; " - . "cd ".$self->workdir()." && "); + my $wd = ::shell_quote_file($self->workdir()); + $remote_pre .= ::shell_quote_scalar("mkdir -p ") . $wd . + ::shell_quote_scalar("; cd ") . $wd . ::shell_quote_scalar(" && "); } $self->{'sshlogin_wrap'} = ($pre @@ -4142,6 +4245,8 @@ sub sshlogin_wrap { sub transfer { # Files to transfer + # Returns: + # @transfer - File names of files to transfer my $self = shift; my @transfer = (); $self->{'transfersize'} = 0; @@ -4166,36 +4271,16 @@ sub transfersize { } sub sshtransfer { + # Returns for each transfer file: + # rsync $file remote:$workdir my $self = shift; + my @pre; my $sshlogin = $self->sshlogin(); - my $sshcmd = $sshlogin->sshcommand(); - my $serverlogin = $sshlogin->serverlogin(); - my $rsync_opt = "-rlDzR -e".::shell_quote_scalar($sshcmd); - my $pre = ""; + my $workdir = $self->workdir(); for my $file ($self->transfer()) { - $file =~ s:/\./:/:g; # Rsync treats /./ special. We dont want that - $file =~ s:^\./::g; # Remove ./ if any - my $relpath = ($file !~ m:^/:); # Is the path relative? - # Use different subdirs depending on abs or rel path - # Abs path: rsync -rlDzR /home/tange/dir/subdir/file.gz server:/ - # Rel path: rsync -rlDzR ./subdir/file.gz server:.parallel/tmp/tempid/ - # Rel path: rsync -rlDzR ./subdir/file.gz server:$workdir/ - my $remote_workdir = $self->workdir($file); - my $rsync_destdir = ($relpath ? $remote_workdir : "/"); - if($relpath) { - $file = "./".$file; - } - if(-r $file) { - my $mkremote_workdir = - $remote_workdir eq "." ? "true" : - "$sshcmd $serverlogin mkdir -p $rsync_destdir"; - $pre .= "$mkremote_workdir; rsync $rsync_opt " - . ::shell_quote_scalar($file)." $serverlogin:$rsync_destdir;"; - } else { - ::warning($file, " is not readable and will not be transferred.\n"); - } + push @pre, $sshlogin->rsync_transfer_cmd($file,$workdir).";"; } - return $pre; + return join("",@pre); } sub return { @@ -4226,6 +4311,8 @@ sub returnsize { } sub sshreturn { + # Returns for each return-file: + # rsync remote:$workdir/$file . my $self = shift; my $sshlogin = $self->sshlogin(); my $sshcmd = $sshlogin->sshcommand(); @@ -4242,13 +4329,13 @@ sub sshreturn { # rsync -avR /foo/./bar/baz.c remote:/tmp/ # == (on old systems) # rsync -avR --rsync-path="cd /foo; rsync" remote:bar/baz.c /tmp/ - $wd = ::shell_quote_scalar($self->workdir())."/"; + $wd = ::shell_quote_file($self->workdir()."/"); } # Only load File::Basename if actually needed $Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; - $cd = ::shell_quote_scalar(dirname($file)); + $cd = ::shell_quote_file(dirname($file)); my $rsync_cd = '--rsync-path='.::shell_quote_scalar("cd $wd$cd; rsync"); - my $basename = ::shell_quote_scalar(::shell_quote_scalar(basename($file))); + my $basename = ::shell_quote_scalar(::shell_quote_file(basename($file))); # --return # mkdir -p /home/tange/dir/subdir/; # rsync -rlDzR --rsync-path="cd /home/tange/dir/subdir/; rsync" @@ -4268,28 +4355,14 @@ sub sshcleanup { my $sshcmd = $sshlogin->sshcommand(); my $serverlogin = $sshlogin->serverlogin(); my $workdir = $self->workdir(); - my $removeworkdir = ""; my $cleancmd = ""; for my $file ($self->cleanup()) { my @subworkdirs = parentdirs_of($file); - if(@subworkdirs) { - $removeworkdir = "rmdir 2>/dev/null ". - join(" ",map { ::shell_quote_scalar($workdir."/".$_) } - @subworkdirs).";"; - } - my $relpath = ($file !~ m:^/:); # Is the path relative? - my $cleandir = ($relpath ? $workdir."/" : ""); - $cleancmd .= - ::shell_quote_scalar("rm -f " - . ::shell_quote_scalar($cleandir.$file).';' - . $removeworkdir); + $cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";"; } if(defined $opt::workdir and $opt::workdir eq "...") { - $cleancmd .= "rm -rf " . ::shell_quote_scalar($workdir).'\;'; - } - if($cleancmd) { - $cleancmd = "$sshcmd $serverlogin ".$cleancmd.";"; + $cleancmd .= "$sshcmd $serverlogin rm -rf " . ::shell_quote_scalar($workdir).';'; } return $cleancmd; } diff --git a/src/parallel.pdf b/src/parallel.pdf index df256241..ec392086 100644 Binary files a/src/parallel.pdf and b/src/parallel.pdf differ diff --git a/src/parallel.pod b/src/parallel.pod index 35e65000..0d4c1f19 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -301,9 +301,9 @@ string that is not in the command line. See also: B<:::>. -=item B<--basefile> I +=item B<--basefile> I (alpha testing) -=item B<--bf> I +=item B<--bf> I (alpha testing) I will be transferred to each sshlogin before a jobs is started. It will be removed if B<--cleanup> is active. The file may be @@ -361,7 +361,7 @@ I defaults to 1M. See B<--pipe> for use of this. -=item B<--cleanup> +=item B<--cleanup> (alpha testing) Remove transferred files. B<--cleanup> will remove the transferred files on the remote computer after processing is done. @@ -430,7 +430,7 @@ occurs as a line of input, the rest of the input is ignored. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--delay> I (beta testing) +=item B<--delay> I Delay starting next job I seconds. GNU B will pause I seconds after starting each job. I can be less than 1 @@ -456,7 +456,7 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I (beta testing) +=item B<--env> I Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for @@ -489,7 +489,7 @@ See also B<--bg>, B. Implies B<--semaphore>. -=item B<--filter-hosts> (beta testing) +=item B<--filter-hosts> Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -551,7 +551,7 @@ status will be the exit status from the failing job. =back -=item B<--header> I (beta testing) +=item B<--header> I Use regexp as header. For normal usage the matched header (typically the first line: B<--header '.*\n'>) will be split using B<--colsep> @@ -579,7 +579,7 @@ specified, and for B<-I>{} otherwise. This option is deprecated; use B<-I> instead. -=item B<--joblog> I (alpha testing) +=item B<--joblog> I (beta testing) Logfile for executed jobs. Save a list of the executed jobs to I in the following TAB separated format: sequence number, @@ -705,7 +705,7 @@ B<-l 0> is an alias for B<-l 1>. Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set. -=item B<--line-buffer> (beta testing) +=item B<--line-buffer> Buffer output on line basis. B<--group> will keep the output together for a whole job. B<--ungroup> allows output to mixup with half a line @@ -774,7 +774,7 @@ This is useful for scripts that depend on features only available from a certain version of GNU B. -=item B<--nonall> (alpha testing) +=item B<--nonall> (beta testing) B<--onall> with no arguments. Run the command on all computers given with B<--sshlogin> but take no arguments. GNU B will log @@ -785,7 +785,7 @@ This is useful for running the same command (e.g. uptime) on a list of servers. -=item B<--onall> (alpha testing) +=item B<--onall> (beta testing) Run all the jobs on all computers given with B<--sshlogin>. GNU B will log into B<--jobs> number of computers in parallel @@ -797,19 +797,19 @@ When using B<--group> the output will be grouped by each server, so all the output from one server will be grouped together. -=item B<--output-as-files> (alpha testing) +=item B<--output-as-files> (beta testing) -=item B<--outputasfiles> (alpha testing) +=item B<--outputasfiles> (beta testing) -=item B<--files> (alpha testing) +=item B<--files> (beta testing) Instead of printing the output to stdout (standard output) the output of each job is saved in a file and the filename is then printed. -=item B<--pipe> (beta testing) +=item B<--pipe> -=item B<--spreadstdin> (beta testing) +=item B<--spreadstdin> Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -967,7 +967,7 @@ If the stdin (standard input) only contains whitespace, do not run the command. If used with B<--pipe> this is slow. -=item B<--record-env> (beta testing) +=item B<--record-env> Record current environment variables in ~/.parallel/ignored_vars. This is useful before using B<--env _>. @@ -1015,9 +1015,9 @@ it to the command. Only used with B<--pipe>. -=item B<--results> I (beta testing) +=item B<--results> I -=item B<--res> I (beta testing) +=item B<--res> I Save the output into files. The files will be stored in a directory tree rooted at I. Within this directory tree, each command will result @@ -1091,7 +1091,7 @@ re-use the computers. This is useful if some jobs fail for no apparent reason (such as network failure). -=item B<--return> I +=item B<--return> I (alpha testing) Transfer files from remote computers. B<--return> is used with B<--sshlogin> when the arguments are files on the remote computers. When @@ -1126,9 +1126,9 @@ B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. -=item B<--round-robin> (beta testing) +=item B<--round-robin> -=item B<--round> (beta testing) +=item B<--round> Normally B<--pipe> will give a single block to each instance of the command. With B<--round-robin> all blocks will at random be written to @@ -1293,7 +1293,7 @@ be needed. An I is of the form: - [sshcommand [options]][username@]hostname + [sshcommand [options]] [username@]hostname The sshlogin must not require a password. @@ -1318,9 +1318,9 @@ B<--sshlogin> is often used with B<--transfer>, B<--return>, B<--cleanup>, and B<--trc>. -=item B<--sshloginfile> I (beta testing) +=item B<--sshloginfile> I -=item B<--slf> I (beta testing) +=item B<--slf> I File with sshlogins. The file consists of sshlogins on separate lines. Empty lines and lines starting with '#' are ignored. Example: @@ -1412,7 +1412,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--timeout> I (beta testing) +=item B<--timeout> I Time out for command. If the command runs for longer than I seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms @@ -1445,7 +1445,7 @@ Print the job to be run on stderr (standard error). See also B<-v>, B<-p>. -=item B<--transfer> +=item B<--transfer> (alpha testing) Transfer files to remote computers. B<--transfer> is used with B<--sshlogin> when the arguments are files and should be transferred to @@ -1557,9 +1557,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely. Print the version GNU B and exit. -=item B<--workdir> I (beta testing) +=item B<--workdir> I -=item B<--wd> I (beta testing) +=item B<--wd> I Files transferred using B<--transfer> and B<--return> will be relative to I on remote computers, and the command will be executed in @@ -2163,6 +2163,23 @@ To run commands on a remote computer SSH needs to be set up and you must be able to login without entering a password (The commands B and B may help you do that). +If you need to login to a whole cluster, you typically do not want to +accept the host key for every host. You want to accept them the first +time and be warned if they are ever changed. To do that: + + # Add the servers to the sshloginfile + (echo servera; echo serverb) > .parallel/my_cluster + # Make sure .ssh/config exist + touch .ssh/config + cp .ssh/config .ssh/config.backup + # Disable StrictHostKeyChecking temporarily + (echo 'Host *'; echo StrictHostKeyChecking no) >> .ssh/config + parallel --slf my_cluster --nonall true + # Remove the disabling of StrictHostKeyChecking + mv .ssh/config.backup .ssh/config + +The servers in B<.parallel/my_cluster> are now added in B<.ssh/known_hosts>. + To run B on B: seq 10 | parallel --sshlogin server.example.com echo diff --git a/src/parallel.texi b/src/parallel.texi index 0998aabb..41fc80f2 100644 --- a/src/parallel.texi +++ b/src/parallel.texi @@ -317,11 +317,11 @@ string that is not in the command line. See also: @strong{:::}. -@item @strong{--basefile} @emph{file} -@anchor{@strong{--basefile} @emph{file}} +@item @strong{--basefile} @emph{file} (alpha testing) +@anchor{@strong{--basefile} @emph{file} (alpha testing)} -@item @strong{--bf} @emph{file} -@anchor{@strong{--bf} @emph{file}} +@item @strong{--bf} @emph{file} (alpha testing) +@anchor{@strong{--bf} @emph{file} (alpha testing)} @emph{file} will be transferred to each sshlogin before a jobs is started. It will be removed if @strong{--cleanup} is active. The file may be @@ -381,8 +381,8 @@ than a single record. See @strong{--pipe} for use of this. -@item @strong{--cleanup} -@anchor{@strong{--cleanup}} +@item @strong{--cleanup} (alpha testing) +@anchor{@strong{--cleanup} (alpha testing)} Remove transferred files. @strong{--cleanup} will remove the transferred files on the remote computer after processing is done. @@ -456,8 +456,8 @@ Set the end of file string to eof-str. If the end of file string occurs as a line of input, the rest of the input is ignored. If neither @strong{-E} nor @strong{-e} is used, no end of file string is used. -@item @strong{--delay} @emph{secs} (beta testing) -@anchor{@strong{--delay} @emph{secs} (beta testing)} +@item @strong{--delay} @emph{secs} +@anchor{@strong{--delay} @emph{secs}} Delay starting next job @emph{secs} seconds. GNU @strong{parallel} will pause @emph{secs} seconds after starting each job. @emph{secs} can be less than 1 @@ -483,8 +483,8 @@ because it is POSIX compliant for @strong{xargs} while this option is not. If @emph{eof-str} is omitted, there is no end of file string. If neither @strong{-E} nor @strong{-e} is used, no end of file string is used. -@item @strong{--env} @emph{var} (beta testing) -@anchor{@strong{--env} @emph{var} (beta testing)} +@item @strong{--env} @emph{var} +@anchor{@strong{--env} @emph{var}} Copy environment variable @emph{var}. This will copy @emph{var} to the environment that the command is run in. This is especially useful for @@ -516,8 +516,8 @@ See also @strong{--bg}, @strong{man sem}. Implies @strong{--semaphore}. -@item @strong{--filter-hosts} (beta testing) -@anchor{@strong{--filter-hosts} (beta testing)} +@item @strong{--filter-hosts} +@anchor{@strong{--filter-hosts}} Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -582,8 +582,8 @@ status will be the exit status from the failing job. @end table -@item @strong{--header} @emph{regexp} (beta testing) -@anchor{@strong{--header} @emph{regexp} (beta testing)} +@item @strong{--header} @emph{regexp} +@anchor{@strong{--header} @emph{regexp}} Use regexp as header. For normal usage the matched header (typically the first line: @strong{--header '.*\n'}) will be split using @strong{--colsep} @@ -611,8 +611,8 @@ This option is a synonym for @strong{-I}@emph{replace-str} if @emph{replace-str} specified, and for @strong{-I}@{@} otherwise. This option is deprecated; use @strong{-I} instead. -@item @strong{--joblog} @emph{logfile} (alpha testing) -@anchor{@strong{--joblog} @emph{logfile} (alpha testing)} +@item @strong{--joblog} @emph{logfile} (beta testing) +@anchor{@strong{--joblog} @emph{logfile} (beta testing)} Logfile for executed jobs. Save a list of the executed jobs to @emph{logfile} in the following TAB separated format: sequence number, @@ -757,8 +757,8 @@ standard specifies @strong{-L} instead. Implies @strong{-X} unless @strong{-m}, @strong{--xargs}, or @strong{--pipe} is set. -@item @strong{--line-buffer} (beta testing) -@anchor{@strong{--line-buffer} (beta testing)} +@item @strong{--line-buffer} +@anchor{@strong{--line-buffer}} Buffer output on line basis. @strong{--group} will keep the output together for a whole job. @strong{--ungroup} allows output to mixup with half a line @@ -827,8 +827,8 @@ GNU @strong{parallel} is less than @emph{version} the exit code is This is useful for scripts that depend on features only available from a certain version of GNU @strong{parallel}. -@item @strong{--nonall} (alpha testing) -@anchor{@strong{--nonall} (alpha testing)} +@item @strong{--nonall} (beta testing) +@anchor{@strong{--nonall} (beta testing)} @strong{--onall} with no arguments. Run the command on all computers given with @strong{--sshlogin} but take no arguments. GNU @strong{parallel} will log @@ -838,8 +838,8 @@ computer. @strong{-j} adjusts how many computers to log into in parallel. This is useful for running the same command (e.g. uptime) on a list of servers. -@item @strong{--onall} (alpha testing) -@anchor{@strong{--onall} (alpha testing)} +@item @strong{--onall} (beta testing) +@anchor{@strong{--onall} (beta testing)} Run all the jobs on all computers given with @strong{--sshlogin}. GNU @strong{parallel} will log into @strong{--jobs} number of computers in parallel @@ -850,23 +850,23 @@ adjusts how many computers to log into in parallel. When using @strong{--group} the output will be grouped by each server, so all the output from one server will be grouped together. -@item @strong{--output-as-files} (alpha testing) -@anchor{@strong{--output-as-files} (alpha testing)} +@item @strong{--output-as-files} (beta testing) +@anchor{@strong{--output-as-files} (beta testing)} -@item @strong{--outputasfiles} (alpha testing) -@anchor{@strong{--outputasfiles} (alpha testing)} +@item @strong{--outputasfiles} (beta testing) +@anchor{@strong{--outputasfiles} (beta testing)} -@item @strong{--files} (alpha testing) -@anchor{@strong{--files} (alpha testing)} +@item @strong{--files} (beta testing) +@anchor{@strong{--files} (beta testing)} Instead of printing the output to stdout (standard output) the output of each job is saved in a file and the filename is then printed. -@item @strong{--pipe} (beta testing) -@anchor{@strong{--pipe} (beta testing)} +@item @strong{--pipe} +@anchor{@strong{--pipe}} -@item @strong{--spreadstdin} (beta testing) -@anchor{@strong{--spreadstdin} (beta testing)} +@item @strong{--spreadstdin} +@anchor{@strong{--spreadstdin}} Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -1029,8 +1029,8 @@ If the stdin (standard input) only contains whitespace, do not run the command. If used with @strong{--pipe} this is slow. -@item @strong{--record-env} (beta testing) -@anchor{@strong{--record-env} (beta testing)} +@item @strong{--record-env} +@anchor{@strong{--record-env}} Record current environment variables in ~/.parallel/ignored_vars. This is useful before using @strong{--env _}. @@ -1080,11 +1080,11 @@ it to the command. Only used with @strong{--pipe}. -@item @strong{--results} @emph{prefix} (beta testing) -@anchor{@strong{--results} @emph{prefix} (beta testing)} +@item @strong{--results} @emph{prefix} +@anchor{@strong{--results} @emph{prefix}} -@item @strong{--res} @emph{prefix} (beta testing) -@anchor{@strong{--res} @emph{prefix} (beta testing)} +@item @strong{--res} @emph{prefix} +@anchor{@strong{--res} @emph{prefix}} Save the output into files. The files will be stored in a directory tree rooted at @emph{prefix}. Within this directory tree, each command will result @@ -1165,8 +1165,8 @@ there are fewer than @emph{n} computers in @strong{--sshlogin} GNU @strong{paral re-use the computers. This is useful if some jobs fail for no apparent reason (such as network failure). -@item @strong{--return} @emph{filename} -@anchor{@strong{--return} @emph{filename}} +@item @strong{--return} @emph{filename} (alpha testing) +@anchor{@strong{--return} @emph{filename} (alpha testing)} Transfer files from remote computers. @strong{--return} is used with @strong{--sshlogin} when the arguments are files on the remote computers. When @@ -1206,11 +1206,11 @@ times: @strong{--return} is ignored when used with @strong{--sshlogin :} or when not used with @strong{--sshlogin}. -@item @strong{--round-robin} (beta testing) -@anchor{@strong{--round-robin} (beta testing)} +@item @strong{--round-robin} +@anchor{@strong{--round-robin}} -@item @strong{--round} (beta testing) -@anchor{@strong{--round} (beta testing)} +@item @strong{--round} +@anchor{@strong{--round}} Normally @strong{--pipe} will give a single block to each instance of the command. With @strong{--round-robin} all blocks will at random be written to @@ -1394,7 +1394,7 @@ be needed. An @emph{sshlogin} is of the form: @verbatim - [sshcommand [options]][username@]hostname + [sshcommand [options]] [username@]hostname @end verbatim The sshlogin must not require a password. @@ -1419,11 +1419,11 @@ The remote host must have GNU @strong{parallel} installed. @strong{--sshlogin} is often used with @strong{--transfer}, @strong{--return}, @strong{--cleanup}, and @strong{--trc}. -@item @strong{--sshloginfile} @emph{filename} (beta testing) -@anchor{@strong{--sshloginfile} @emph{filename} (beta testing)} +@item @strong{--sshloginfile} @emph{filename} +@anchor{@strong{--sshloginfile} @emph{filename}} -@item @strong{--slf} @emph{filename} (beta testing) -@anchor{@strong{--slf} @emph{filename} (beta testing)} +@item @strong{--slf} @emph{filename} +@anchor{@strong{--slf} @emph{filename}} File with sshlogins. The file consists of sshlogins on separate lines. Empty lines and lines starting with '#' are ignored. Example: @@ -1516,8 +1516,8 @@ into temporary files in /tmp. By setting @strong{--tmpdir} you can use a different dir for the files. Setting @strong{--tmpdir} is equivalent to setting $TMPDIR. -@item @strong{--timeout} @emph{val} (beta testing) -@anchor{@strong{--timeout} @emph{val} (beta testing)} +@item @strong{--timeout} @emph{val} +@anchor{@strong{--timeout} @emph{val}} Time out for command. If the command runs for longer than @emph{val} seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms @@ -1550,8 +1550,8 @@ Print the job to be run on stderr (standard error). See also @strong{-v}, @strong{-p}. -@item @strong{--transfer} -@anchor{@strong{--transfer}} +@item @strong{--transfer} (alpha testing) +@anchor{@strong{--transfer} (alpha testing)} Transfer files to remote computers. @strong{--transfer} is used with @strong{--sshlogin} when the arguments are files and should be transferred to @@ -1673,11 +1673,11 @@ Use @strong{-v} @strong{-v} to print the wrapping ssh command when running remot Print the version GNU @strong{parallel} and exit. -@item @strong{--workdir} @emph{mydir} (beta testing) -@anchor{@strong{--workdir} @emph{mydir} (beta testing)} +@item @strong{--workdir} @emph{mydir} +@anchor{@strong{--workdir} @emph{mydir}} -@item @strong{--wd} @emph{mydir} (beta testing) -@anchor{@strong{--wd} @emph{mydir} (beta testing)} +@item @strong{--wd} @emph{mydir} +@anchor{@strong{--wd} @emph{mydir}} Files transferred using @strong{--transfer} and @strong{--return} will be relative to @emph{mydir} on remote computers, and the command will be executed in @@ -2317,6 +2317,25 @@ To run commands on a remote computer SSH needs to be set up and you must be able to login without entering a password (The commands @strong{ssh-copy-id} and @strong{ssh-agent} may help you do that). +If you need to login to a whole cluster, you typically do not want to +accept the host key for every host. You want to accept them the first +time and be warned if they are ever changed. To do that: + +@verbatim + # Add the servers to the sshloginfile + (echo servera; echo serverb) > .parallel/my_cluster + # Make sure .ssh/config exist + touch .ssh/config + cp .ssh/config .ssh/config.backup + # Disable StrictHostKeyChecking temporarily + (echo 'Host *'; echo StrictHostKeyChecking no) >> .ssh/config + parallel --slf my_cluster --nonall true + # Remove the disabling of StrictHostKeyChecking + mv .ssh/config.backup .ssh/config +@end verbatim + +The servers in @strong{.parallel/my_cluster} are now added in @strong{.ssh/known_hosts}. + To run @strong{echo} on @strong{server.example.com}: @verbatim diff --git a/src/parallel_tutorial.1 b/src/parallel_tutorial.1 index 92ad88ec..1a7817d0 100644 --- a/src/parallel_tutorial.1 +++ b/src/parallel_tutorial.1 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL_TUTORIAL 1" -.TH PARALLEL_TUTORIAL 1 "2013-09-28" "20130922" "parallel" +.TH PARALLEL_TUTORIAL 1 "2013-09-28" "20131022" "parallel" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/src/sql b/src/sql index d408c835..747aaf64 100755 --- a/src/sql +++ b/src/sql @@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20130922; + $Global::version = 20131022; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/src/sql.pdf b/src/sql.pdf index a71010f5..44ce6288 100644 Binary files a/src/sql.pdf and b/src/sql.pdf differ diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 0b091a28..b8dec1de 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -1,21 +1,22 @@ #!/bin/bash +rm -rf tmp +mkdir tmp +cd tmp + cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j10 -k -L1 echo '### Test bug #34241: --pipe should not spawn unneeded processes' seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end echo '### --env _' - fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DISPLAY' ::: test -echo '### --env _ with explicit mentioning of normally ignored var $DISPLAY' - fUbAr="OK FUBAR" parallel -S csh@lo --env DISPLAY,_ echo '$fUbAr $DISPLAY' ::: test + fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test +echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' + fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test echo 'bug #40137: SHELL not bash: Warning when exporting funcs' . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning -echo '### --filter-hosts --slf <()' - parallel --nonall --filter-hosts --slf <(echo localhost) echo OK - echo '### bug #40002: --files and --nonall seem not to work together:' parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l @@ -23,6 +24,79 @@ echo '### bug #40001: --joblog and --nonall seem not to work together:' parallel --joblog - --nonall -S lo,localhost true | wc -l echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME' - cd && parallel --workdir . -S lo pwd ::: "" + cd && parallel --workdir . -S lo pwd ::: "" +echo '### test filename :' + echo content-of-: > :; + echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; + cat :.:; rm : :.: + +echo '### Test --wd ... --cleanup which should remove the filled tmp dir' + find ~/.parallel/tmp |grep uNiQuE_sTrInG.6 | parallel rm; + stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S lo -v echo ">"{}.6 ::: uNiQuE_sTrInG; + find ~/.parallel/tmp |grep uNiQuE_sTrInG.6 + +echo '### Test --wd --' + stdout parallel --wd -- -S lo echo OK ">"{}.7 ::: uNiQuE_sTrInG; + cat ~/--/uNiQuE_sTrInG.7; + stdout rm ~/--/uNiQuE_sTrInG.7 + +echo '### Test --wd " "' + stdout parallel --wd " " -S lo echo OK ">"{}.8 ::: uNiQuE_sTrInG; + cat ~/" "/uNiQuE_sTrInG.8; + stdout rm ~/" "/uNiQuE_sTrInG.8 + +echo "### Test --wd \"'\"" + stdout parallel --wd "'" -S lo echo OK ">"{}.9 ::: uNiQuE_sTrInG; + cat ~/"'"/uNiQuE_sTrInG.9; + stdout rm ~/"'"/uNiQuE_sTrInG.9 + +echo '### Test --trc --/--foo1' + mkdir -p ./--; echo 'Content --/--foo1' > ./--/--foo1; + stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./--/--foo1; cat ./--/--foo1.1; + stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: --/--foo1; cat ./--/--foo1.2 + +echo '### Test --trc :dir/:foo2' + mkdir -p ./:dir; echo 'Content :dir/:foo2' > ./:dir/:foo2; + stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./:dir/:foo2; cat ./:dir/:foo2.1; + stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: :dir/:foo2; cat ./:dir/:foo2.2 + +echo '### Test --trc " "/" "foo3' + mkdir -p ./" "; echo 'Content _/_foo3' > ./" "/" "foo3; + stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1; + stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: " "/" "foo3; cat ./" "/" "foo3.2 + +#echo '### Test --trc --/./--foo4' +# mkdir -p ./--; echo 'Content --/./--foo4' > ./--/./--foo4; +# stdout parallel --trc {}.1 -S lo '(cat ./--foo4; echo remote{}) > --foo4.1' ::: --/./--foo4; cat ./--/./--foo4.1 +# +#echo '### Test --trc :/:foo2' +# mkdir -p ./:; echo 'Content :/:foo2' > ./:/:foo2; +# stdout parallel --trc {}.1 -S lo '(cat {}; echo remote) > {}.1' ::: ./:/:foo2; cat ./:/:foo2.1 +# +#echo '### Test --trc " "/" "foo3' +# mkdir -p ./" "; echo 'Content _/_foo2' > ./" "/" "foo3; +# stdout parallel --trc {}.1 -S lo '(cat {}; echo remote) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1 +# +# +# +# +# +# +#echo '### Test --trc --/./--foo4 :/./:foo5 " "/./" "foo6 ./foo10/./foo10' +# mkdir ./--; echo 'Content --/--foo4' > ./--/--foo4; +# mkdir ./:; echo 'Content :/:foo5' > ./:/:foo5; +# mkdir ./" "; echo 'Content _/_foo2' > ./" "/" "foo3; + +### echo '### Test --trc --basefile --/./--foo7 :/./:foo8 " "/./" "foo9 ./foo11/./foo11' +### echo missing +### echo '### Test --trc "-- "' +### echo missing +### echo '### Test --trc " --"' +### echo missing +### EOF + +rm -rf tmp +mkdir tmp +cd tmp diff --git a/testsuite/tests-to-run/parallel-local-ssh2.sh b/testsuite/tests-to-run/parallel-local-ssh2.sh new file mode 100644 index 00000000..d7fd4e3d --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-ssh2.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j8 -k -L1 +echo '### --filter-hosts --slf <()' + parallel --nonall --filter-hosts --slf <(echo localhost) echo OK + +EOF diff --git a/testsuite/tests-to-run/test17.sh b/testsuite/tests-to-run/test17.sh index abee4136..88db57cb 100755 --- a/testsuite/tests-to-run/test17.sh +++ b/testsuite/tests-to-run/test17.sh @@ -16,9 +16,8 @@ rm -rf /tmp/parallel.file* stdout ssh $SSHLOGIN1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' stdout ssh $SSHLOGIN2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' # (seq 1 3;echo '>fire';seq 5 10; /bin/echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17 -(echo '>fire';/bin/echo ' : & ) \n*.jpg'; echo '/./sub dir') >/tmp/test17 +(echo '>fire';/bin/echo ' : & ) \n*.jpg') >/tmp/test17 echo '# Create some weirdly files in /tmp' -mkdir -p /tmp/parallel.file cat /tmp/test17 | parallel -k /bin/echo file{} '>'/tmp/parallel.file{}.file cat /tmp/test17 | parallel -k /bin/echo /tmp/parallel.file{}.file >/tmp/test17abs cat /tmp/test17 | parallel -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel @@ -69,14 +68,14 @@ stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out cat /tmp/test17abs | parallel -k --return {.}.out --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p /tmp/parallel.file";"echo {} ">"{.}.out -ls /tmp/parallel.file*out /tmp/parallel.file/*out +stdout ls /tmp/parallel.file*out echo '### --return - relpath' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' rm -rf /tmp/parallel.file*out cat /tmp/test17rel | parallel -k --return {.}.out --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out -ls tmp/parallel.file*out tmp/parallel.file/*out +stdout ls tmp/parallel.file*out echo '### --return - multiple files' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' @@ -84,7 +83,7 @@ stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' -ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +stdout ls tmp/parallel.file*out tmp/parallel.file*done echo '### --return --cleanup - abspath' stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*' @@ -92,19 +91,19 @@ stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done cat /tmp/test17abs | parallel -k --return {.}.out --return {}.done --cleanup \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p /tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK echo '### --return --cleanup - relpath' -rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +rm -rf tmp/parallel.file*out tmp/parallel.file*done stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' -ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +stdout ls tmp/parallel.file*out tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory @@ -113,35 +112,33 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK echo '### --return --cleanup - multiple returns' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' -rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file -stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK -# Should give: No such file or directory -stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK +stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' +stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' echo '### --transfer --return --cleanup - abspath' stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*' -rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 cat {} ">"{.}.out';'cat {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK - echo '### --transfer --return --cleanup - relpath' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' -rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory @@ -150,10 +147,10 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - multiple files' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' -rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK @@ -161,10 +158,10 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK echo '### --trc - abspath' stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*' -rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK # Should give: No such file or directory @@ -173,10 +170,10 @@ stdout ssh $SSHLOGIN2 ls '/tmp/parallel.file*' || echo OK echo '### --trc - relpath' stdout ssh $SSHLOGIN1 'rm -rf tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf tmp/parallel.file*' -rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' -ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +stdout ls tmp/parallel.file*out tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory @@ -185,10 +182,10 @@ stdout ssh $SSHLOGIN2 ls 'tmp/parallel.file*' || echo OK echo '### --trc - multiple files' stdout ssh $SSHLOGIN1 'rm -rf /tmp/parallel.file*' stdout ssh $SSHLOGIN2 'rm -rf /tmp/parallel.file*' -rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SSHLOGIN1,$SSHLOGIN2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' -ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done +stdout ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SSHLOGIN1 ls '/tmp/parallel.file*' || echo OK # Should give: No such file or directory diff --git a/testsuite/tests-to-run/test35.sh b/testsuite/tests-to-run/test35.sh index 033c6f2f..dc3cec60 100755 --- a/testsuite/tests-to-run/test35.sh +++ b/testsuite/tests-to-run/test35.sh @@ -18,15 +18,15 @@ echo '### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs'; find . -name '*.7' | sort echo '### Test --workdir ...' -parallel -j9 -k --workdir ... --basefile 1-col.txt --trc {}.1 -S .. echo ">"{}.1 ::: 2-col.txt +parallel -j9 -k --workdir ... --trc {}.1 -S .. echo ">"{}.1 ::: 2-col.txt find . -name '*.1' | sort echo '### Test --wd ...' -parallel -k --wd ... --basefile 1-col.txt --trc {}.2 -S .. -v echo ">"{}.2 ::: 2-col.txt +parallel -k --wd ... --trc {}.2 -S .. -v echo ">"{}.2 ::: 2-col.txt find . -name '*.2' | sort echo '### Test --wd ... with space dirs' -stdout parallel -j9 -k --wd ... --basefile 1-col.txt --trc {}.3 -S .. -v echo ">"{}.3 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg' +stdout parallel -j9 -k --wd ... --trc {}.3 -S .. -v echo ">"{}.3 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg' # A few rmdir errors are OK as we have multiple files in the same dirs find . -name '*.3' | sort @@ -39,9 +39,5 @@ stdout parallel -k -j9 --wd /tmp/ --basefile 1-col.txt --trc {}.5 -S .. -v echo # A few rmdir errors are OK as we have multiple files in the same dirs find . -name '*.5' | sort -echo '### Test --wd ... --cleanup' -stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S lo -v echo ">"{}.6 ::: uNiQuE_sTrInG -find ~/.parallel/tmp |grep uNiQuE_sTrInG - cd .. rm -rf tmp diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index b834eaff..bf1a1a29 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -7,18 +7,42 @@ Block_end 5 Block_end ### --env _ -DISPLAY: Undefined variable. -### --env _ with explicit mentioning of normally ignored var $DISPLAY -OK FUBAR :0.0 test +DEBEMAIL: Undefined variable. +### --env _ with explicit mentioning of normally ignored var $DEBEMAIL +OK FUBAR ole@tange.dk test bug #40137: SHELL not bash: Warning when exporting funcs no_warning warning parallel: Warning: Shell functions may not be supported in /bin/sh -### --filter-hosts --slf <() -OK ### bug #40002: --files and --nonall seem not to work together: 1 ### bug #40001: --joblog and --nonall seem not to work together: 3 ### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME /home/tange +### test filename : +remote-:.: +content-of-: +### Test --wd ... --cleanup which should remove the filled tmp dir +echo >uNiQuE_sTrInG.6 +### Test --wd -- +OK +### Test --wd " " +OK +### Test --wd "'" +OK +### Test --trc --/--foo1 +Content --/--foo1 +remote1 +Content --/--foo1 +remote2 +### Test --trc :dir/:foo2 +Content :dir/:foo2 +remote1 +Content :dir/:foo2 +remote2 +### Test --trc " "/" "foo3 +Content _/_foo3 +remote1 +Content _/_foo3 +remote2 diff --git a/testsuite/wanted-results/parallel-local-ssh2 b/testsuite/wanted-results/parallel-local-ssh2 new file mode 100644 index 00000000..9c898686 --- /dev/null +++ b/testsuite/wanted-results/parallel-local-ssh2 @@ -0,0 +1,2 @@ +### --filter-hosts --slf <() +OK diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index 77991dc5..97ababb0 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -18,8 +18,8 @@ begin 14 15 16 -17 parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help. +17 18 19 20 diff --git a/testsuite/wanted-results/test17 b/testsuite/wanted-results/test17 index fcde2d56..4a403895 100644 --- a/testsuite/wanted-results/test17 +++ b/testsuite/wanted-results/test17 @@ -3,25 +3,24 @@ ### --transfer - abspath file>fire file : & ) \n*.jpg -file/./sub dir good if no file ls: /tmp/parallel.file*: No such file or directory +ls: /tmp/parallel.file*: No such file or directory ### --transfer - relpath file>fire file : & ) \n*.jpg -file/./sub dir good if no file ls: tmp/parallel.file*: No such file or directory +ls: tmp/parallel.file*: No such file or directory ### --transfer --cleanup - abspath file>fire file : & ) \n*.jpg -file/./sub dir good if no file ls: /tmp/parallel.file*: No such file or directory +ls: /tmp/parallel.file*: No such file or directory ### --transfer --cleanup - relpath file>fire file : & ) \n*.jpg -file/./sub dir good if no file ls: tmp/parallel.file*: No such file or directory OK @@ -29,62 +28,46 @@ ls: tmp/parallel.file*: No such file or directory OK ### --return - abspath /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.out ### --return - relpath tmp/parallel.file : & ) \n*.jpg.out -tmp/parallel.file/sub dir.out tmp/parallel.file>fire.out ### --return - multiple files tmp/parallel.file : & ) \n*.jpg.file.done tmp/parallel.file : & ) \n*.jpg.out -tmp/parallel.file/sub dir.file.done -tmp/parallel.file/sub dir.out tmp/parallel.file>fire.file.done tmp/parallel.file>fire.out ### --return --cleanup - abspath /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file ### --return --cleanup - relpath tmp/parallel.file : & ) \n*.jpg.file.done tmp/parallel.file : & ) \n*.jpg.out -tmp/parallel.file/sub dir.file.done -tmp/parallel.file/sub dir.out tmp/parallel.file>fire.file.done tmp/parallel.file>fire.out good if no file -ls: tmp/parallel.file*: No such file or directory -OK ### --return --cleanup - multiple returns /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file -ls: tmp/parallel.file*: No such file or directory -OK ### --transfer --return --cleanup - abspath /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file ls: /tmp/parallel.file*: No such file or directory OK +ls: /tmp/parallel.file*: No such file or directory +OK ### --transfer --return --cleanup - relpath /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file @@ -95,8 +78,6 @@ OK ### --transfer --return --cleanup - multiple files /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out ls: tmp/parallel.file*: No such file or directory @@ -106,18 +87,16 @@ OK ### --trc - abspath /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file ls: /tmp/parallel.file*: No such file or directory OK +ls: /tmp/parallel.file*: No such file or directory +OK ### --trc - relpath tmp/parallel.file : & ) \n*.jpg.file.done tmp/parallel.file : & ) \n*.jpg.out -tmp/parallel.file/sub dir.file.done -tmp/parallel.file/sub dir.out tmp/parallel.file>fire.file.done tmp/parallel.file>fire.out good if no file @@ -128,20 +107,19 @@ OK ### --trc - multiple files /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file ls: /tmp/parallel.file*: No such file or directory OK +ls: /tmp/parallel.file*: No such file or directory +OK ### --transfer --cleanup - multiple argument files cat /tmp/parallel.file\>fire.file tmp/parallel.file\>fire.file file>fire file>fire -cat /tmp/parallel.file/./sub\ dir.file tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file -file/./sub dir +cat /tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file file : & ) \n*.jpg -cat /tmp/parallel.file\ :\ \&\ \)\ \\n\*.jpg.file tmp/parallel.file/./sub\ dir.file file : & ) \n*.jpg -file/./sub dir +ls: /tmp/parallel.file*: No such file or directory +OK diff --git a/testsuite/wanted-results/test19 b/testsuite/wanted-results/test19 index cccd1dba..8ab86674 100644 --- a/testsuite/wanted-results/test19 +++ b/testsuite/wanted-results/test19 @@ -75,31 +75,37 @@ OK ls: tmp/parallel.file*: No such file or directory OK Input for ssh --l parallel parallel-server1 rsync --server -lDrRze.iLsf . . +parallel@parallel-server1 mkdir -p ./. +-l parallel parallel-server1 rsync --server -lDrRze.iLsf . ./. -tt -oLogLevel=quiet parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 2\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=2\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.' 'newline2 > tmp/parallel.file.' 'newline2.out;cat tmp/parallel.file.' 'newline2 > tmp/parallel.file.' 'newline2.out2 --l parallel parallel-server1 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.' +-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.' 'newline2.out --l parallel parallel-server1 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.' +-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.' 'newline2.out2 -parallel@parallel-server1 rm -f ./tmp/parallel.file.' -'newline2;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.' -'newline2.out;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.' -'newline2.out2;rmdir 2>/dev/null ./tmp; --l parallel parallel-server2 rsync --server -lDrRze.iLsf . . +parallel@parallel-server1 (rm -f ./tmp/parallel.file.' +'newline2; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server1 (rm -f ./tmp/parallel.file.' +'newline2.out; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server1 (rm -f ./tmp/parallel.file.' +'newline2.out2; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server2 mkdir -p ./. +-l parallel parallel-server2 rsync --server -lDrRze.iLsf . ./. -tt -oLogLevel=quiet parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 1\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=1\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.' 'newline1 > tmp/parallel.file.' 'newline1.out;cat tmp/parallel.file.' 'newline1 > tmp/parallel.file.' 'newline1.out2 --l parallel parallel-server2 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.' +-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.' 'newline1.out --l parallel parallel-server2 cd ./tmp; rsync --server --sender -lDrRze.iLsf . parallel.file.' +-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLsf . ./parallel.file.' 'newline1.out2 -parallel@parallel-server2 rm -f ./tmp/parallel.file.' -'newline1;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.' -'newline1.out;rmdir 2>/dev/null ./tmp;rm -f ./tmp/parallel.file.' -'newline1.out2;rmdir 2>/dev/null ./tmp; +parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +'newline1; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +'newline1.out; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +'newline1.out2; rmdir ./tmp/ ./ 2>/dev/null;) diff --git a/testsuite/wanted-results/test35 b/testsuite/wanted-results/test35 index 8d4462a5..90d636b3 100644 --- a/testsuite/wanted-results/test35 +++ b/testsuite/wanted-results/test35 @@ -66,5 +66,3 @@ echo >./a\ b/cd\ /\ ef/efg.5 ./a b/cd / ef/efg.5 ./b/bar.5 ./b/foo.5 -### Test --wd ... --cleanup -echo >uNiQuE_sTrInG.6 diff --git a/testsuite/wanted-results/test61 b/testsuite/wanted-results/test61 index a09c7151..2f99e7b8 100644 --- a/testsuite/wanted-results/test61 +++ b/testsuite/wanted-results/test61 @@ -1,5 +1,5 @@ ### Test --return of weirdly named file -ssh -tt -oLogLevel=quiet parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p .; rsync --rsync-path=cd\ ./.\;\ rsync -rlDzR -essh parallel@parallel-server3:aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b .; exit $_EXIT_status; +ssh -tt -oLogLevel=quiet parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status; ### Test if remote login shell is csh ssh -tt -oLogLevel=quiet csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$PARALLEL_PID\ \$PARALLEL_SEQ\ a\|\ wc\ -w; 3