From ebdc4c641aa95351c862ad1a49be70f81b433ac0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 23 Jun 2019 08:31:28 +0200 Subject: [PATCH] parallel: Fixed bug #56540: parallel echo $(printf "\257\256") ::: $(printf "\257\256"). parallel: added uq() as function for unquoting replacement string. --- doc/haikus | 6 +- src/niceload | 2 +- .../gnuparallel/runwrapper-generator.pl | 183 ++++++++++ src/parallel | 72 ++-- src/parallel.pod | 4 + src/sql | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 5 + testsuite/tests-to-run/parallel-local-10s.sh | 18 + testsuite/tests-to-run/parallel-local-30s.sh | 15 - testsuite/tests-to-run/parallel-local-ssh1.sh | 293 +++++++++------- testsuite/wanted-results/parallel-local-0.3s | 2 + testsuite/wanted-results/parallel-local-10s | 99 ++++++ testsuite/wanted-results/parallel-local-30s | 22 -- testsuite/wanted-results/parallel-local-ssh1 | 330 ++++++++---------- 14 files changed, 673 insertions(+), 380 deletions(-) create mode 100644 src/optional/python/gnuparallel/runwrapper-generator.pl diff --git a/doc/haikus b/doc/haikus index 2e53a563..57eba02b 100644 --- a/doc/haikus +++ b/doc/haikus @@ -3,9 +3,6 @@ Quote of the month: I love so much @GnuParallel to parallelize my SQL requests on @PostgreSQL -- @rmaziere_85 Romain -I want to make a shout-out for @GnuParallel, it's a work of beauty and power - -- Cristian Consonni @CristianCantoro - Parallel is the BEST command. -- Nick @NickInfoSec @@ -47,6 +44,9 @@ It's the MapReduce of our generation! === Used === +I want to make a shout-out for @GnuParallel, it's a work of beauty and power + -- Cristian Consonni @CristianCantoro + Amazingly useful script! -- unxusr@reddit.com diff --git a/src/niceload b/src/niceload index 29a6087e..71eb3cc9 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20190622; +$Global::version = 20190623; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/optional/python/gnuparallel/runwrapper-generator.pl b/src/optional/python/gnuparallel/runwrapper-generator.pl new file mode 100644 index 00000000..3c1794f0 --- /dev/null +++ b/src/optional/python/gnuparallel/runwrapper-generator.pl @@ -0,0 +1,183 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2019 Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +use strict; + +my %opt = get_options_hash(); + +# Reserved words in Python: Replace them +my %reserved = ("return" => "_return", + "0" => "null"); +my @out = <<'_EOS'; +#!/usr/bin/python3 + +# Copyright (C) 2019 Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +import subprocess + +_EOS + +# Make the function definition +push @out, "def parallel(command=None, args=None, input=None, "; + +my @inner; +for my $k (keys %opt) { + $k =~ s/-/_/g; + $k =~ s/[:=].*//; + for my $p (split /\|/, $k) { + push @inner, ($reserved{$p} || $p)."=None"; + } +} +push @out, join ", ", uniq(@inner); +push @out, "):\n"; + +# Documentation string +push @out, ' """ + Python wrapper for GNU Parallel + + Use GNU Parallel options with - replaced with _. + + These: + + parallel(command="echo _{}_",args=[["A B","C D"],[1,2]],jobs="50%") + parallel("echo _{}_",args=[["A B","C D"],[1,2]],jobs="50%") + parallel("echo _{}_",[["A B","C D"],[1,2]],jobs="50%") + + will all run: + + parallel --jobs 50% echo _{}_ ::: "A B" "C D" ::: 1 2 + + This: + + parallel(command="echo _{}_",args=["A B","C D"],tag=True) + + will run: + + parallel --tag echo _{}_ ::: "A B" "C D" + + This: + + parallel(command="echo _{}_",input=b"a\nb\n",keep_order=True) + + will send "a\nb\n" on standard input (stdin) to: + + | parallel --keep-order echo _{}_ + + """',"\n"; +# Build the command for subprocess.run +push @out, " option = ['parallel']\n"; + +for my $k (keys %opt) { + my $type = "bool"; + if($k =~ s/([:=])(.*)//) { + my %typeof = ("i" => "int", + "f" => "float", + "s" => "str"); + $type = $typeof{$2}; + } + for my $p (split /\|/, $k) { + my $o = $p; + $p =~ s/-/_/g; + push @out, " if ".($reserved{$p} || $p)." is not None:\n"; + # --long-option? or -s hort? + my $dashes = ($o =~ /../) ? "--" : "-"; + if($type eq "bool") { + push @out, " option.extend(['$dashes$o'])\n"; + } + if($type eq "str" or $type eq "float" or $type eq "int") { + push @out, " option.extend(['$dashes$o',".($reserved{$p} || $p)."])\n"; + } + } +} +push @out, map { s/^/ /gm;$_ } join"\n", + ("argumentsep = ':::'", + "if argsep is not None:", + " argumentsep = argsep", + "if arg_sep is not None:", + " argumentsep = arg_sep", + "if command is not None:", + " option.append(command)", + "if args is not None:", + " if type(args[0]) is list:", + " for a in args:", + " option.extend([argumentsep])", + " option.extend(map(str,a))", + " else:", + " option.extend([argumentsep])", + " option.extend(map(str,args))", + "out = subprocess.run(option,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE)", + "return(out.stdout,out.stderr)", + ); +push @out, join "\n", + ("",'', + 'print(parallel(command="echo _{}_",args=[["A B","C D"],[1,2]],jobs="50%"))', + 'print(parallel(command="echo _{}_",args=["A B","C D"],tag=True))', + 'print(parallel(command="echo _{}_",v=True,argsep="f",args=["A B","C D"],tag=True))', + 'print(parallel(command="echo _{}_",input=b"a\nb\n",keep_order=True))', + 'print(parallel(args=["pwd","echo foo"]))', + '' + ); + +print @out; + +sub get_options_hash { + sub which { + local $_ = qx{ LANG=C type "$_[0]" }; + my $exit += (s/ is an alias for .*// || + s/ is aliased to .*// || + s/ is a function// || + s/ is a shell function// || + s/ is a shell builtin// || + s/.* is hashed .(\S+).$/$1/ || + s/.* is (a tracked alias for )?//); + if($exit) { + return $_; + } else { + return undef; + } + } + + my $fullpath = which("parallel"); + my $parallelsource = qx{cat $fullpath}; + # Grap options hash from source + $parallelsource =~ /([(]"debug.D=s".*?[)])/s || die; + my $optsource = $1; + my %opt = eval $optsource; + return %opt; +} + +sub uniq { + # Remove duplicates and return unique values + return keys %{{ map { $_ => 1 } @_ }}; +} diff --git a/src/parallel b/src/parallel index a40f8be1..f993e58a 100755 --- a/src/parallel +++ b/src/parallel @@ -1378,7 +1378,7 @@ sub options_hash() { "tmux" => \$opt::tmux, "tmuxpane" => \$opt::tmuxpane, "null|0" => \$opt::null, - "quote|q" => \$opt::q, + "quote|q" => \$opt::quote, # Replacement strings "parens=s" => \$opt::parens, "rpl=s" => \@opt::rpl, @@ -1598,7 +1598,7 @@ sub parse_options(@) { $opt::tagstring = unquote_printf($opt::tagstring); } if(defined $opt::interactive) { $Global::interactive = $opt::interactive; } - if(defined $opt::q) { $Global::quoting = 1; } + if(defined $opt::quote) { $Global::quoting = 1; } if(defined $opt::r) { $Global::ignore_empty = 1; } if(defined $opt::verbose) { $Global::stderr_verbose = 1; } parse_replacement_string_options(); @@ -1984,12 +1984,15 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20190622; + $Global::version = 20190623; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; $Global::verbose = 0; + # Don't quote every part of the command line $Global::quoting = 0; + # Quote replacement strings + $Global::quote_replace = 1; $Global::total_completed = 0; # Read only table with default --rpl values %Global::replace = @@ -10257,8 +10260,8 @@ sub push($) { push @{$self->{'arg_list'}}, $record; # Make @arg available for {= =} *Arg::arg = $self->{'arg_list_flat_orig'}; - - my $quote_arg = $Global::noquote ? 0 : not $Global::quoting; + + my $quote_arg = ($Global::quote_replace and not $Global::quoting); for my $perlexpr (keys %{$self->{'replacecount'}}) { if($perlexpr =~ /^(\d+) /) { # Positional @@ -10285,7 +10288,7 @@ sub pop($) { # pop off arguments from @$record splice @{$self->{'arg_list_flat_orig'}}, -($#$record+1), $#$record+1; splice @{$self->{'arg_list_flat'}}, -($#$record+1), $#$record+1; - my $quote_arg = $Global::noquote ? 0 : not $Global::quoting; + my $quote_arg = ($Global::quote_replace and not $Global::quoting); for my $perlexpr (keys %{$self->{'replacecount'}}) { if($perlexpr =~ /^(\d+) /) { # Positional @@ -10551,14 +10554,14 @@ sub len($) { sub replaced($) { # Uses: - # $Global::noquote + # $Global::quote_replace # $Global::quoting # Returns: # $replaced = command with place holders replaced and prepended my $self = shift; if(not defined $self->{'replaced'}) { # Don't quote arguments if the input is the full command line - my $quote_arg = $Global::noquote ? 0 : not $Global::quoting; + my $quote_arg = ($Global::quote_replace and not $Global::quoting); # or if ($opt::cat or $opt::pipe) as they use $PARALLEL_TMP $quote_arg = ($opt::cat || $opt::fifo) ? 0 : $quote_arg; $self->{'replaced'} = $self-> @@ -10656,7 +10659,7 @@ sub replace_placeholders($$$$) { # except the first CORE::push @replaced, $space, map { $a = $_; - $a =~ + if($a =~ s{\257<(-?\d+)?(.*)\257>} { if($1) { @@ -10671,7 +10674,15 @@ sub replace_placeholders($$$$) { $normal_replace ||= 1; ($arg ? $arg->replace($2,$quote_arg,$self) : ""); } - }sgxe; + }sgxe) { + # Token is \257<..\257> + } else { + if($Global::escape_string_present) { + # Command line contains \257: + # Unescape it \257\256 => \257 + $a =~ s/\257\256/\257/g; + } + } $a } @ctxgroup; $normal_replace or last; @@ -10679,7 +10690,9 @@ sub replace_placeholders($$$$) { } } else { # Context group has no a replacement string: Copy it once - CORE::push @replaced, @ctxgroup; + CORE::push @replaced, map { + $Global::escape_string_present and s/\257\256/\257/g; $_; + } @ctxgroup; } # New context group @ctxgroup=(); @@ -10724,21 +10737,14 @@ sub replace_placeholders($$$$) { } } else { # No replacement - CORE::push @replaced, $t; + CORE::push @replaced, map { + $Global::escape_string_present and s/\257\256/\257/g; $_; + } $t; } } } *Arg::arg = []; ::debug("replace","Replaced: ".join":",@replaced,"\n"); - if($Global::escape_string_present) { - # Command line contains \257: Unescape it \257\256 => \257 - # If a replacement resulted in \257\256 - # it will have been escaped into \\\257\\\\256 - # and will not be matched below - for(@replaced) { - s/\257\256/\257/g; - } - } # Put tokens into groups that may be quoted. my @quotegroup; @@ -10907,7 +10913,7 @@ sub new($) { # argument or variable definition V1={}) # E.g. parallel {}, parallel my_{= s/_//=}, parallel {2} # Do no quote (Otherwise it will fail if the input contains spaces) - $Global::noquote = 1; + $Global::quote_replace = 0; } if($opt::sqlmaster and $Global::sql->append()) { @@ -11071,7 +11077,7 @@ sub get($) { # argument or variable definition V1={}) # E.g. parallel {}, parallel my_{= s/_//=}, parallel {2} # Do no quote (Otherwise it will fail if the input contains spaces) - $Global::noquote = 1; + $Global::quote_replace = 0; } } @@ -11795,6 +11801,10 @@ sub total_jobs() { # shorthand for $job->seq(); $job->seq(); } + sub uq() { + # Do not quote this arg + $Global::unquote_arg = 1; + } sub replace($$$$) { # Calculates the corresponding value for a given perl expression @@ -11802,7 +11812,7 @@ sub total_jobs() { # The calculated string (quoted if asked for) my $self = shift; my $perlexpr = shift; # E.g. $_=$_ or s/.gz// - my $quote = (shift) ? 1 : 0; # should the string be quoted? + my $quote = shift; # should the string be quoted? # This is actually a CommandLine-object, # but it looks nice to be able to say {= $job->slot() =} $job = shift; @@ -11834,10 +11844,20 @@ sub total_jobs() { # Execute the function $perleval{$perlexpr}->($job); $self->{'cache'}{$perlexpr} = $_; + if($Global::unquote_arg) { + # uq() was called in perlexpr + $self->{'cache'}{'unquote'}{$perlexpr} = 1; + # Reset for next perlexpr + $Global::unquote_arg = 0; + } } # Return the value quoted if needed - return($quote ? Q($self->{'cache'}{$perlexpr}) - : $self->{'cache'}{$perlexpr}); + if($self->{'cache'}{'unquote'}{$perlexpr}) { + return($self->{'cache'}{$perlexpr}); + } else { + return($quote ? Q($self->{'cache'}{$perlexpr}) + : $self->{'cache'}{$perlexpr}); + } } } diff --git a/src/parallel.pod b/src/parallel.pod index deb5e2e3..f05432e4 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -277,6 +277,10 @@ shell quote a string perl quote a string +=item Z<> B + +unquote current replacement string + =item Z<> B number of jobs in total diff --git a/src/sql b/src/sql index 5e8b9348..d171d9c7 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20190622; + $Global::version = 20190623; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index fe8111bc..46a99ceb 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -737,6 +737,11 @@ par_rpl_that_is_substring_of_longer_rpl() { 'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f } +par_unquote_replacement_string() { + echo '### Can part of the replacement string be unquoted using uq()?' + parallel echo '{}{=uq()=}' ::: '`echo foo`' +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 47975a69..e7cbe54a 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -4,6 +4,24 @@ # Each should be taking 10-30s and be possible to run in parallel # I.e.: No race conditions, no logins +par_macron() { + echo '### See if \257\256 \257<\257> is replaced correctly' + print_it() { + parallel $2 ::: "echo $1" + parallel $2 echo ::: "$1" + parallel $2 echo "$1" ::: "$1" + parallel $2 echo \""$1"\" ::: "$1" + parallel $2 echo "$1"{} ::: "$1" + parallel $2 echo \""$1"\"{} ::: "$1" + } + export -f print_it + parallel --tag -k print_it \ + ::: "$(perl -e 'print "\257"')" "$(perl -e 'print "\257\256"')" \ + "$(perl -e 'print "\257\257\256"')" \ + "$(perl -e 'print "\257<\257<\257>\257>"')" \ + ::: -X -q -Xq -k +} + par_kill_hup() { echo '### Are children killed if GNU Parallel receives HUP? There should be no sleep at the end' diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 498e8194..787eb630 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -236,21 +236,6 @@ par_max_length_len_128k() { seq 1 60000 | perl -pe 's/$/.gif/' | parallel -X echo {} aa {} |head -n 1 |wc } -par_macron() { - print_it() { - parallel ::: "echo $1" - parallel echo ::: "$1" - parallel echo "$1" ::: "$1" - parallel echo \""$1"\" ::: "$1" - parallel -q echo ::: "$1" - parallel -q echo "$1" ::: "$1" - parallel -q echo \""$1"\" ::: "$1" - } - print_it "$(perl -e 'print "\257"')" - print_it "$(perl -e 'print "\257\256"')" - print_it "$(perl -e 'print "\257<\257<\257>\257>"')" -} - par_round_robin_blocks() { echo "bug #49664: --round-robin does not complete" seq 20000000 | parallel -j8 --block 10M --round-robin --pipe wc -c | wc -l diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 78fe9c96..d1c6d08d 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -7,128 +7,6 @@ cd tmp unset run_test cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj300% -k --joblog /tmp/jl-`basename $0` -L1 -r -echo '### Test --load remote' - ssh parallel@lo 'seq 10 | parallel --nice 19 --timeout 15 -j0 -qN0 perl -e while\(1\)\{\ \}' & - sleep 1; - stdout /usr/bin/time -f %e parallel -S parallel@lo --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' - -echo '**' - -echo '### Stop if all hosts are filtered and there are no hosts left to run on' - stdout parallel --filter-hosts -S no-such.host echo ::: 1 - -echo '### Can csh propagate a variable containing \n'; - export A=$(seq 3); parallel -S csh@lo --env A bash -c \''echo "$A"'\' ::: dummy - -echo '### Test bug #34241: --pipe should not spawn unneeded processes' - seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end - -echo '### bug #40002: --files and --nonall seem not to work together:' - parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l - -echo '### bug #40001: --joblog and --nonall seem not to work together:' - parallel --joblog - --nonall -S lo,localhost true | wc -l - -echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME' - cd && parallel --workdir . -S lo pwd ::: "" - -echo '### use function as $PARALLEL_SSH' - foossh() { echo "FOOSSH" >&2; ssh "$@"; }; - export -f foossh; - PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?' - -echo '### use --ssh' - barssh() { echo "BARSSH" >&2; ssh "$@"; }; - export -f barssh; - parallel --ssh barssh -S 1/lo echo ::: 'Run through BARSSH?' - -echo '### test filename :' - echo content-of-: > :; - echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; - cat :.:; rm : :.: - -echo '### Test --wd ... --cleanup which should remove the filled tmp dir' - ssh sh@lo 'mkdir -p .parallel/tmp; find .parallel/tmp |grep uNiQuE_sTrInG.6 | parallel rm'; - stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S sh@lo -v echo ">"{}.6 ::: uNiQuE_sTrInG; - find ~sh/.parallel/tmp |grep uNiQuE_sTrInG.6 - -echo '### Test --wd --' - stdout parallel --wd -- -S sh@lo echo OK ">"{}.7 ::: uNiQuE_sTrInG; - cat ~sh/--/uNiQuE_sTrInG.7; - stdout ssh sh@lo rm ./--/uNiQuE_sTrInG.7 - -echo '### Test --wd " "' - stdout parallel --wd " " -S sh@lo echo OK ">"{}.8 ::: uNiQuE_sTrInG; - cat ~sh/" "/uNiQuE_sTrInG.8; - stdout ssh sh@lo rm ./'" "'/uNiQuE_sTrInG.8 - -echo "### Test --wd \"'\"" - stdout parallel --wd "'" -S sh@lo echo OK ">"{}.9 ::: uNiQuE_sTrInG; - cat ~sh/"'"/uNiQuE_sTrInG.9; - stdout ssh sh@lo rm ./"\\'"/uNiQuE_sTrInG.9 - -echo '### Test --trc ./--/--foo1' - mkdir -p ./--; echo 'Content --/--foo1' > ./--/--foo1; - stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./--/--foo1; cat ./--/--foo1.1; - stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: --/--foo1; cat ./--/--foo1.2 - -echo '### Test --trc ./:dir/:foo2' - mkdir -p ./:dir; echo 'Content :dir/:foo2' > ./:dir/:foo2; - stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./:dir/:foo2; - cat ./:dir/:foo2.1; - stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: :dir/:foo2; - cat ./:dir/:foo2.2 - -echo '### Test --trc ./" "/" "foo3' - mkdir -p ./" "; echo 'Content _/_foo3' > ./" "/" "foo3; - stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./" "/" "foo3; - cat ./" "/" "foo3.1; - stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: " "/" "foo3; - cat ./" "/" "foo3.2 - -echo '### Test --trc ./--/./--foo4' - mkdir -p ./--; echo 'Content --/./--foo4' > ./--/./--foo4; - stdout parallel --trc {}.1 -S sh@lo '(cat ./--foo4; echo remote{}) > --foo4.1' ::: --/./--foo4; - cat ./--/./--foo4.1 - -echo '### Test --trc ./:/./:foo5' - mkdir -p ./:a; echo 'Content :a/./:foo5' > ./:a/./:foo5; - stdout parallel --trc {}.1 -S sh@lo '(cat ./:foo5; echo remote{}) > ./:foo5.1' ::: ./:a/./:foo5; - cat ./:a/./:foo5.1 - -echo '### Test --trc ./" "/./" "foo6' - mkdir -p ./" "; echo 'Content _/./_foo6' > ./" "/./" "foo6; - stdout parallel --trc {}.1 -S sh@lo '(cat ./" "foo6; echo remote{}) > ./" "foo6.1' ::: ./" "/./" "foo6; - cat ./" "/./" "foo6.1 - -echo '### Test --trc "-- " "-- "' - touch -- '-- ' ' --'; rm -f ./?--.a ./--?.a; - parallel --trc {}.a -S csh@lo,sh@lo touch ./{}.a ::: '-- ' ' --'; ls ./--?.a ./?--.a; - parallel --nonall -k -S csh@lo,sh@lo 'ls ./?-- || echo OK'; - parallel --nonall -k -S csh@lo,sh@lo 'ls ./--? || echo OK'; - parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--.a || echo OK'; - parallel --nonall -k -S csh@lo,sh@lo 'ls ./--?.a || echo OK' - -echo '### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B"' - mkdir -p '/tmp/./--- ' '/tmp/ ---'; - touch -- '/tmp/./--- /A' '/tmp/ ---/B'; - rm -f ./---?/A.a ./?---/B.a; - parallel --trc {=s:.*/./::=}.a -S csh@lo,sh@lo touch ./{=s:.*/./::=}.a ::: '/tmp/./--- /A' '/tmp/./ ---/B'; - ls ./---?/A.a ./?---/B.a | LC_ALL=C sort; - parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--- ./---? || echo OK' | LC_ALL=C sort; - -echo '### bug #46519: --onall ignores --transfer' - touch bug46519.{a,b,c}; rm -f bug46519.?? bug46519.???; - parallel --onall --tf bug46519.{} --trc bug46519.{}{} --trc bug46519.{}{}{} -S csh@lo,sh@lo - 'ls bug46519.{}; touch bug46519.{}{} bug46519.{}{}{}' ::: a b c; - ls bug46519.?? bug46519.???; - parallel --onall -S csh@lo,sh@lo ls bug46519.{}{} bug46519.{}{}{} ::: a b c && echo Cleanup failed - -echo '### Test --nice remote' -stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | - perl -pe 's/\S*parallel-server\S*/one-server/;s:="[0-9]+":="XXXXX":i;' - -echo '**' echo TODO @@ -136,6 +14,177 @@ echo TODO EOF +par_stop_if_no_hosts_left() { + echo '### Stop if all hosts are filtered and there are no hosts left to run on' + stdout parallel --filter-hosts -S no-such.host echo ::: 1 +} + +par_csh_variable_newline() { + echo '### Can csh propagate a variable containing \n' + export A=$(seq 3); parallel -S csh@lo --env A bash -c \''echo "$A"'\' ::: dummy +} + +par_pipe_unneeded_spawn() { + echo '### Test bug #34241: --pipe should not spawn unneeded processes' + seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end +} + +par_files_nonall() { + echo '### bug #40002: --files and --nonall seem not to work together:' + parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l +} + +par_joblog_nonall() { + echo '### bug #40001: --joblog and --nonall seem not to work together:' + parallel --joblog - --nonall -S lo,localhost true | wc -l +} + +par_workdir_home() { + echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME' + cd && parallel --workdir . -S lo pwd ::: "" +} + +par_PARALLEL_SSH_function() { + echo '### use function as $PARALLEL_SSH' + foossh() { echo "FOOSSH" >&2; ssh "$@"; } + export -f foossh + PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?' +} + +par_ssh() { + echo '### use --ssh' + barssh() { echo "BARSSH" >&2; ssh "$@"; } + export -f barssh + parallel --ssh barssh -S 1/lo echo ::: 'Run through BARSSH?' +} + +par_filename_colon() { + echo '### test filename :' + echo content-of-: > : + echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}' + cat :.:; rm : :.: +} + +par_wd_dotdotdot() { + echo '### Test --wd ... --cleanup which should remove the filled tmp dir' + ssh sh@lo 'mkdir -p .parallel/tmp; find .parallel/tmp |grep uNiQuE_sTrInG.6 | parallel rm' + stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S sh@lo -v echo ">"{}.6 ::: uNiQuE_sTrInG + find ~sh/.parallel/tmp |grep uNiQuE_sTrInG.6 +} + +par_wd_dashdash() { + echo '### Test --wd --' + stdout parallel --wd -- -S sh@lo echo OK ">"{}.7 ::: uNiQuE_sTrInG + cat ~sh/--/uNiQuE_sTrInG.7 + stdout ssh sh@lo rm ./--/uNiQuE_sTrInG.7 +} + +par_wd_space() { + echo '### Test --wd " "' + stdout parallel --wd " " -S sh@lo echo OK ">"{}.8 ::: uNiQuE_sTrInG + cat ~sh/" "/uNiQuE_sTrInG.8 + stdout ssh sh@lo rm ./'" "'/uNiQuE_sTrInG.8 +} + +par_wd_quote() { + echo "### Test --wd \"'\"" + stdout parallel --wd "'" -S sh@lo echo OK ">"{}.9 ::: uNiQuE_sTrInG + cat ~sh/"'"/uNiQuE_sTrInG.9 + stdout ssh sh@lo rm ./"\\'"/uNiQuE_sTrInG.9 +} + +par_trc_dashdash() { + echo '### Test --trc ./--/--foo1' + mkdir -p ./--; echo 'Content --/--foo1' > ./--/--foo1 + stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./--/--foo1; cat ./--/--foo1.1 + stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: --/--foo1; cat ./--/--foo1.2 +} + +par_trc_colon() { + echo '### Test --trc ./:dir/:foo2' + mkdir -p ./:dir; echo 'Content :dir/:foo2' > ./:dir/:foo2 + stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./:dir/:foo2 + cat ./:dir/:foo2.1 + stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: :dir/:foo2 + cat ./:dir/:foo2.2 +} + +par_trc_space() { + echo '### Test --trc ./" "/" "foo3' + mkdir -p ./" "; echo 'Content _/_foo3' > ./" "/" "foo3 + stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./" "/" "foo3 + cat ./" "/" "foo3.1 + stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: " "/" "foo3 + cat ./" "/" "foo3.2 +} + +par_trc_dashdashdot() { + echo '### Test --trc ./--/./--foo4' + mkdir -p ./--; echo 'Content --/./--foo4' > ./--/./--foo4 + stdout parallel --trc {}.1 -S sh@lo '(cat ./--foo4; echo remote{}) > --foo4.1' ::: --/./--foo4 + cat ./--/./--foo4.1 +} + +par_trc_colondot() { + echo '### Test --trc ./:/./:foo5' + mkdir -p ./:a; echo 'Content :a/./:foo5' > ./:a/./:foo5 + stdout parallel --trc {}.1 -S sh@lo '(cat ./:foo5; echo remote{}) > ./:foo5.1' ::: ./:a/./:foo5 + cat ./:a/./:foo5.1 +} + +par_trc_spacedot() { + echo '### Test --trc ./" "/./" "foo6' + mkdir -p ./" "; echo 'Content _/./_foo6' > ./" "/./" "foo6 + stdout parallel --trc {}.1 -S sh@lo '(cat ./" "foo6; echo remote{}) > ./" "foo6.1' ::: ./" "/./" "foo6 + cat ./" "/./" "foo6.1 +} + +par_trc_dashdashspace() { + echo '### Test --trc "-- " "-- "' + touch -- '-- ' ' --'; rm -f ./?--.a ./--?.a + parallel --trc {}.a -S csh@lo,sh@lo touch ./{}.a ::: '-- ' ' --'; ls ./--?.a ./?--.a + parallel --nonall -k -S csh@lo,sh@lo 'ls ./" "-- || echo OK' + parallel --nonall -k -S csh@lo,sh@lo 'ls ./--" " || echo OK' + parallel --nonall -k -S csh@lo,sh@lo 'ls ./" "--.a || echo OK' + parallel --nonall -k -S csh@lo,sh@lo 'ls ./--" ".a || echo OK' +} + +par_trc_dashdashdashspace() { + echo '### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B"' + mkdir -p '/tmp/./--- ' '/tmp/ ---' + touch -- '/tmp/./--- /A' '/tmp/ ---/B' + rm -f ./---?/A.a ./?---/B.a + parallel --trc {=s:.*/./::=}.a -S csh@lo,sh@lo touch ./{=s:.*/./::=}.a ::: '/tmp/./--- /A' '/tmp/./ ---/B' + ls ./---?/A.a ./?---/B.a | LC_ALL=C sort + parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--- ./---? || echo OK' | LC_ALL=C sort +} + +par_onall_transfer() { + echo '### bug #46519: --onall ignores --transfer' + touch bug46519.{a,b,c}; rm -f bug46519.?? bug46519.??? + parallel --onall --tf bug46519.{} --trc bug46519.{}{} --trc bug46519.{}{}{} -S csh@lo,sh@lo 'ls bug46519.{}; touch bug46519.{}{} bug46519.{}{}{}' ::: a b c + ls bug46519.?? bug46519.??? + parallel --onall -S csh@lo,sh@lo ls bug46519.{}{} bug46519.{}{}{} ::: a b c && + echo Cleanup failed +} + +par_remote_load() { + echo '### Test --load remote' + ssh parallel@lo 'seq 10 | parallel --nice 19 --timeout 15 -j0 -qN0 perl -e while\(1\)\{\ \}' & + sleep 1 + stdout /usr/bin/time -f %e parallel -S parallel@lo --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' +} + +par_remote_nice() { + echo '### Test --nice remote' + stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | + perl -pe 's/\S*parallel-server\S*/one-server/;s:="[0-9]+":="XXXXX":i;' +} + +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | LC_ALL=C sort | + parallel --timeout 130 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' + cd .. rm -rf tmp mkdir tmp diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 27f1deaf..31ad1f3b 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -977,6 +977,8 @@ par_tricolonplus 3 22 b cc par_tricolonplus 3 23 c aa par_tricolonplus 3 23 c bb par_tricolonplus 3 23 c cc +par_unquote_replacement_string ### Can part of the replacement string be unquoted using uq()? +par_unquote_replacement_string `echo foo`foo par_verbose_t ### Test --verbose and -t par_verbose_t echo bar par_verbose_t echo car diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index b2c1beba..887502e6 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -272,6 +272,105 @@ par_line_buffer These must diff: 1 par_long_line_remote ### Deal with long command lines on remote servers par_long_line_remote 2 6 30006 par_long_line_remote 3 50 250050 +par_macron ### See if \257\256 \257<\257> is replaced correctly +par_macron ¯ -X ¯ +par_macron ¯ -X ¯ +par_macron ¯ -X ¯ ¯ +par_macron ¯ -X ¯ ¯ +par_macron ¯ -X ¯¯ +par_macron ¯ -X ¯¯ +par_macron ¯ -q ¯ +par_macron ¯ -q ¯ ¯ +par_macron ¯ -q "¯" ¯ +par_macron ¯ -q ¯¯ +par_macron ¯ -q "¯"¯ +par_macron ¯ -q /bin/bash: $'echo \257': command not found +par_macron ¯ -Xq ¯ +par_macron ¯ -Xq ¯ ¯ +par_macron ¯ -Xq "¯" ¯ +par_macron ¯ -Xq ¯¯ +par_macron ¯ -Xq "¯"¯ +par_macron ¯ -Xq /bin/bash: $'echo \257': command not found +par_macron ¯ -k ¯ +par_macron ¯ -k ¯ +par_macron ¯ -k ¯ ¯ +par_macron ¯ -k ¯ ¯ +par_macron ¯ -k ¯¯ +par_macron ¯ -k ¯¯ +par_macron ¯® -X ¯® +par_macron ¯® -X ¯® +par_macron ¯® -X ¯® ¯® +par_macron ¯® -X ¯® ¯® +par_macron ¯® -X ¯®¯® +par_macron ¯® -X ¯®¯® +par_macron ¯® -q ¯® +par_macron ¯® -q ¯® ¯® +par_macron ¯® -q "¯®" ¯® +par_macron ¯® -q ¯®¯® +par_macron ¯® -q "¯®"¯® +par_macron ¯® -q /bin/bash: $'echo \257\256': command not found +par_macron ¯® -Xq ¯® +par_macron ¯® -Xq ¯® ¯® +par_macron ¯® -Xq "¯®" ¯® +par_macron ¯® -Xq ¯®¯® +par_macron ¯® -Xq "¯®"¯® +par_macron ¯® -Xq /bin/bash: $'echo \257\256': command not found +par_macron ¯® -k ¯® +par_macron ¯® -k ¯® +par_macron ¯® -k ¯® ¯® +par_macron ¯® -k ¯® ¯® +par_macron ¯® -k ¯®¯® +par_macron ¯® -k ¯®¯® +par_macron ¯¯® -X ¯¯® +par_macron ¯¯® -X ¯¯® +par_macron ¯¯® -X ¯¯® ¯¯® +par_macron ¯¯® -X ¯¯® ¯¯® +par_macron ¯¯® -X ¯¯®¯¯® +par_macron ¯¯® -X ¯¯®¯¯® +par_macron ¯¯® -q ¯¯® +par_macron ¯¯® -q ¯¯® ¯¯® +par_macron ¯¯® -q "¯¯®" ¯¯® +par_macron ¯¯® -q ¯¯®¯¯® +par_macron ¯¯® -q "¯¯®"¯¯® +par_macron ¯¯® -q /bin/bash: $'echo \257\257\256': command not found +par_macron ¯¯® -Xq ¯¯® +par_macron ¯¯® -Xq ¯¯® ¯¯® +par_macron ¯¯® -Xq "¯¯®" ¯¯® +par_macron ¯¯® -Xq ¯¯®¯¯® +par_macron ¯¯® -Xq "¯¯®"¯¯® +par_macron ¯¯® -Xq /bin/bash: $'echo \257\257\256': command not found +par_macron ¯¯® -k ¯¯® +par_macron ¯¯® -k ¯¯® +par_macron ¯¯® -k ¯¯® ¯¯® +par_macron ¯¯® -k ¯¯® ¯¯® +par_macron ¯¯® -k ¯¯®¯¯® +par_macron ¯¯® -k ¯¯®¯¯® +par_macron ¯<¯<¯>¯> -X ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -X ¯<¯<¯>¯> ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -X ¯<¯<¯>¯>¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -X /bin/bash: -c: line 0: syntax error near unexpected token `newline' +par_macron ¯<¯<¯>¯> -X /bin/bash: -c: line 0: `echo ¯<¯<¯>¯>' +par_macron ¯<¯<¯>¯> -X /bin/bash: ¯: No such file or directory +par_macron ¯<¯<¯>¯> -X /bin/bash: ¯: No such file or directory +par_macron ¯<¯<¯>¯> -q ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -q ¯<¯<¯>¯> ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -q "¯<¯<¯>¯>" ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -q ¯<¯<¯>¯>¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -q "¯<¯<¯>¯>"¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -q /bin/bash: $'echo \257<\257<\257>\257>': command not found +par_macron ¯<¯<¯>¯> -Xq ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -Xq ¯<¯<¯>¯> ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -Xq "¯<¯<¯>¯>" ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -Xq ¯<¯<¯>¯>¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -Xq "¯<¯<¯>¯>"¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -Xq /bin/bash: $'echo \257<\257<\257>\257>': command not found +par_macron ¯<¯<¯>¯> -k ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -k ¯<¯<¯>¯> ¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -k ¯<¯<¯>¯>¯<¯<¯>¯> +par_macron ¯<¯<¯>¯> -k /bin/bash: -c: line 0: syntax error near unexpected token `newline' +par_macron ¯<¯<¯>¯> -k /bin/bash: -c: line 0: `echo ¯<¯<¯>¯>' +par_macron ¯<¯<¯>¯> -k /bin/bash: ¯: No such file or directory +par_macron ¯<¯<¯>¯> -k /bin/bash: ¯: No such file or directory par_maxlinelen_X_I ### Test max line length -X -I par_maxlinelen_X_I 3cfc69ee81b0fe7fdbe8eb059ad2da61 - par_maxlinelen_X_I Chars per line (817788/7): 116826 diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index aa57ce37..ecacf90e 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -101,28 +101,6 @@ par_linebuffer_matters_compress ### (--linebuffer) --compress should give diffe par_linebuffer_matters_compress OK: --linebuffer makes a difference par_linebuffer_matters_compress_tag ### (--linebuffer) --compress --tag should give different output par_linebuffer_matters_compress_tag OK: --linebuffer makes a difference -par_macron ¯ -par_macron ¯ -par_macron ¯ ¯ -par_macron ¯ ¯ -par_macron ¯ -par_macron ¯ ¯ -par_macron "¯" ¯ -par_macron ¯® -par_macron ¯® -par_macron ¯® ¯ -par_macron ¯® ¯ -par_macron ¯® -par_macron ¯® ¯ -par_macron "¯®" ¯ -par_macron /bin/bash: -c: line 0: syntax error near unexpected token `newline' -par_macron /bin/bash: -c: line 0: `echo ¯<¯<¯>¯>' -par_macron ¯<¯<¯>¯> -par_macron /bin/bash: ¯: No such file or directory -par_macron ¯<¯<¯>¯> ¯<¯<¯>¯> -par_macron ¯<¯<¯>¯> -par_macron ¯<¯<¯>¯> ¯<¯<¯>¯> -par_macron "¯<¯<¯>¯>" ¯<¯<¯>¯> par_max_length_len_128k ### BUG: The length for -X is not close to max (131072) par_max_length_len_128k 1 8293 131004 par_max_length_len_128k 1 6662 131026 diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 030e2826..5a916c80 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -1,193 +1,143 @@ -echo '### Test --load remote' -### Test --load remote - ssh parallel@lo 'seq 10 | parallel --nice 19 --timeout 15 -j0 -qN0 perl -e while\(1\)\{\ \}' & sleep 1; stdout /usr/bin/time -f %e parallel -S parallel@lo --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' -OK -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -parallel: Warning: This job was killed because it timed out: -parallel: Warning: perl -e 'while(1){ }' -echo '**' -** -echo '### Stop if all hosts are filtered and there are no hosts left to run on' -### Stop if all hosts are filtered and there are no hosts left to run on - stdout parallel --filter-hosts -S no-such.host echo ::: 1 -parallel: Warning: Removed no-such.host. -parallel: Error: Cannot run any jobs. -echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); parallel -S csh@lo --env A bash -c \''echo "$A"'\' ::: dummy -### Can csh propagate a variable containing \n -1 -2 -3 -echo '### Test bug #34241: --pipe should not spawn unneeded processes' -### Test bug #34241: --pipe should not spawn unneeded processes - seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end -1 -2 -Block_end -3 -4 -Block_end -5 -Block_end -echo '### bug #40002: --files and --nonall seem not to work together:' -### bug #40002: --files and --nonall seem not to work together: - parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l -1 -echo '### bug #40001: --joblog and --nonall seem not to work together:' -### bug #40001: --joblog and --nonall seem not to work together: - parallel --joblog - --nonall -S lo,localhost true | wc -l -3 -echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME' -### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME - cd && parallel --workdir . -S lo pwd ::: "" -~ -echo '### use function as $PARALLEL_SSH' -### use function as $PARALLEL_SSH - foossh() { echo "FOOSSH" >&2; ssh "$@"; }; export -f foossh; PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?' -Run through FOOSSH? -FOOSSH -echo '### use --ssh' -### use --ssh - barssh() { echo "BARSSH" >&2; ssh "$@"; }; export -f barssh; parallel --ssh barssh -S 1/lo echo ::: 'Run through BARSSH?' -Run through BARSSH? -BARSSH -echo '### test filename :' -### test filename : - echo content-of-: > :; echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; cat :.:; rm : :.: -remote-:.: -content-of-: -echo '### Test --wd ... --cleanup which should remove the filled tmp dir' -### Test --wd ... --cleanup which should remove the filled tmp dir - ssh sh@lo 'mkdir -p .parallel/tmp; find .parallel/tmp |grep uNiQuE_sTrInG.6 | parallel rm'; stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S sh@lo -v echo ">"{}.6 ::: uNiQuE_sTrInG; find ~sh/.parallel/tmp |grep uNiQuE_sTrInG.6 -echo >uNiQuE_sTrInG.6 -echo '### Test --wd --' -### Test --wd -- - stdout parallel --wd -- -S sh@lo echo OK ">"{}.7 ::: uNiQuE_sTrInG; cat ~sh/--/uNiQuE_sTrInG.7; stdout ssh sh@lo rm ./--/uNiQuE_sTrInG.7 -OK -echo '### Test --wd " "' -### Test --wd " " - stdout parallel --wd " " -S sh@lo echo OK ">"{}.8 ::: uNiQuE_sTrInG; cat ~sh/" "/uNiQuE_sTrInG.8; stdout ssh sh@lo rm ./'" "'/uNiQuE_sTrInG.8 -OK -echo "### Test --wd \"'\"" -### Test --wd "'" - stdout parallel --wd "'" -S sh@lo echo OK ">"{}.9 ::: uNiQuE_sTrInG; cat ~sh/"'"/uNiQuE_sTrInG.9; stdout ssh sh@lo rm ./"\\'"/uNiQuE_sTrInG.9 -OK -echo '### Test --trc ./--/--foo1' -### Test --trc ./--/--foo1 - mkdir -p ./--; echo 'Content --/--foo1' > ./--/--foo1; stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./--/--foo1; cat ./--/--foo1.1; stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: --/--foo1; cat ./--/--foo1.2 -Content --/--foo1 -remote1 -Content --/--foo1 -remote2 -echo '### Test --trc ./:dir/:foo2' -### Test --trc ./:dir/:foo2 - mkdir -p ./:dir; echo 'Content :dir/:foo2' > ./:dir/:foo2; stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./:dir/:foo2; cat ./:dir/:foo2.1; stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: :dir/:foo2; cat ./:dir/:foo2.2 -Content :dir/:foo2 -remote1 -Content :dir/:foo2 -remote2 -echo '### Test --trc ./" "/" "foo3' -### Test --trc ./" "/" "foo3 - mkdir -p ./" "; echo 'Content _/_foo3' > ./" "/" "foo3; stdout parallel --trc {}.1 -S sh@lo '(cat {}; echo remote1) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1; stdout parallel --trc {}.2 -S sh@lo '(cat ./{}; echo remote2) > {}.2' ::: " "/" "foo3; cat ./" "/" "foo3.2 -Content _/_foo3 -remote1 -Content _/_foo3 -remote2 -echo '### Test --trc ./--/./--foo4' -### Test --trc ./--/./--foo4 - mkdir -p ./--; echo 'Content --/./--foo4' > ./--/./--foo4; stdout parallel --trc {}.1 -S sh@lo '(cat ./--foo4; echo remote{}) > --foo4.1' ::: --/./--foo4; cat ./--/./--foo4.1 -Content --/./--foo4 -remote--/./--foo4 -echo '### Test --trc ./:/./:foo5' -### Test --trc ./:/./:foo5 - mkdir -p ./:a; echo 'Content :a/./:foo5' > ./:a/./:foo5; stdout parallel --trc {}.1 -S sh@lo '(cat ./:foo5; echo remote{}) > ./:foo5.1' ::: ./:a/./:foo5; cat ./:a/./:foo5.1 -Content :a/./:foo5 -remote./:a/./:foo5 -echo '### Test --trc ./" "/./" "foo6' -### Test --trc ./" "/./" "foo6 - mkdir -p ./" "; echo 'Content _/./_foo6' > ./" "/./" "foo6; stdout parallel --trc {}.1 -S sh@lo '(cat ./" "foo6; echo remote{}) > ./" "foo6.1' ::: ./" "/./" "foo6; cat ./" "/./" "foo6.1 -Content _/./_foo6 -remote./ /./ foo6 -echo '### Test --trc "-- " "-- "' -### Test --trc "-- " "-- " - touch -- '-- ' ' --'; rm -f ./?--.a ./--?.a; parallel --trc {}.a -S csh@lo,sh@lo touch ./{}.a ::: '-- ' ' --'; ls ./--?.a ./?--.a; parallel --nonall -k -S csh@lo,sh@lo 'ls ./?-- || echo OK'; parallel --nonall -k -S csh@lo,sh@lo 'ls ./--? || echo OK'; parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--.a || echo OK'; parallel --nonall -k -S csh@lo,sh@lo 'ls ./--?.a || echo OK' -./ --.a -./-- .a -OK -OK -OK -OK -OK -OK -OK -OK -ls: No match. -ls: cannot access './?--': No such file or directory -ls: No match. -ls: cannot access './--?': No such file or directory -ls: No match. -ls: cannot access './?--.a': No such file or directory -ls: No match. -ls: cannot access './--?.a': No such file or directory -echo '### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B"' -### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B" - mkdir -p '/tmp/./--- ' '/tmp/ ---'; touch -- '/tmp/./--- /A' '/tmp/ ---/B'; rm -f ./---?/A.a ./?---/B.a; parallel --trc {=s:.*/./::=}.a -S csh@lo,sh@lo touch ./{=s:.*/./::=}.a ::: '/tmp/./--- /A' '/tmp/./ ---/B'; ls ./---?/A.a ./?---/B.a | LC_ALL=C sort; parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--- ./---? || echo OK' | LC_ALL=C sort; echo '### bug #46519: --onall ignores --transfer' -./ ---/B.a -./--- /A.a -OK -OK -### bug #46519: --onall ignores --transfer -ls: No match. -ls: cannot access './?---': No such file or directory -ls: cannot access './---?': No such file or directory - touch bug46519.{a,b,c}; rm -f bug46519.?? bug46519.???; parallel --onall --tf bug46519.{} --trc bug46519.{}{} --trc bug46519.{}{}{} -S csh@lo,sh@lo 'ls bug46519.{}; touch bug46519.{}{} bug46519.{}{}{}' ::: a b c; ls bug46519.?? bug46519.???; parallel --onall -S csh@lo,sh@lo ls bug46519.{}{} bug46519.{}{}{} ::: a b c && echo Cleanup failed -bug46519.a -bug46519.b -bug46519.c -bug46519.a -bug46519.b -bug46519.c -bug46519.aa -bug46519.aaa -bug46519.bb -bug46519.bbb -bug46519.cc -bug46519.ccc -ls: cannot access 'bug46519.aa': No such file or directory -ls: cannot access 'bug46519.aaa': No such file or directory -ls: cannot access 'bug46519.bb': No such file or directory -ls: cannot access 'bug46519.bbb': No such file or directory -ls: cannot access 'bug46519.cc': No such file or directory -ls: cannot access 'bug46519.ccc': No such file or directory -ls: cannot access 'bug46519.aa': No such file or directory -ls: cannot access 'bug46519.aaa': No such file or directory -ls: cannot access 'bug46519.bb': No such file or directory -ls: cannot access 'bug46519.bbb': No such file or directory -ls: cannot access 'bug46519.cc': No such file or directory -ls: cannot access 'bug46519.ccc': No such file or directory -echo '### Test --nice remote' -### Test --nice remote -stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | perl -pe 's/\S*parallel-server\S*/one-server/;s:="[0-9]+":="XXXXX":i;' -ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; -a b -echo '**' -** echo TODO TODO ## echo '### Test --trc --basefile --/./--foo7 :/./:foo8 " "/./" "foo9 ./foo11/./foo11' +par_PARALLEL_SSH_function ### use function as $PARALLEL_SSH +par_PARALLEL_SSH_function Run through FOOSSH? +par_PARALLEL_SSH_function FOOSSH +par_csh_variable_newline ### Can csh propagate a variable containing \n +par_csh_variable_newline 1 +par_csh_variable_newline 2 +par_csh_variable_newline 3 +par_filename_colon ### test filename : +par_filename_colon remote-:.: +par_filename_colon content-of-: +par_files_nonall ### bug #40002: --files and --nonall seem not to work together: +par_files_nonall 1 +par_joblog_nonall ### bug #40001: --joblog and --nonall seem not to work together: +par_joblog_nonall 3 +par_onall_transfer ### bug #46519: --onall ignores --transfer +par_onall_transfer bug46519.a +par_onall_transfer bug46519.b +par_onall_transfer bug46519.c +par_onall_transfer bug46519.a +par_onall_transfer bug46519.b +par_onall_transfer bug46519.c +par_onall_transfer bug46519.aa +par_onall_transfer bug46519.aaa +par_onall_transfer bug46519.bb +par_onall_transfer bug46519.bbb +par_onall_transfer bug46519.cc +par_onall_transfer bug46519.ccc +par_onall_transfer ls: cannot access 'bug46519.aa': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.aaa': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.bb': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.bbb': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.cc': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.ccc': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.aa': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.aaa': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.bb': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.bbb': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.cc': No such file or directory +par_onall_transfer ls: cannot access 'bug46519.ccc': No such file or directory +par_pipe_unneeded_spawn ### Test bug #34241: --pipe should not spawn unneeded processes +par_pipe_unneeded_spawn 1 +par_pipe_unneeded_spawn 2 +par_pipe_unneeded_spawn Block_end +par_pipe_unneeded_spawn 3 +par_pipe_unneeded_spawn 4 +par_pipe_unneeded_spawn Block_end +par_pipe_unneeded_spawn 5 +par_pipe_unneeded_spawn Block_end +par_remote_load ### Test --load remote +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load parallel: Warning: This job was killed because it timed out: +par_remote_load parallel: Warning: perl -e 'while(1){ }' +par_remote_load OK +par_remote_nice ### Test --nice remote +par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; +par_remote_nice a b +par_ssh ### use --ssh +par_ssh Run through BARSSH? +par_ssh BARSSH +par_stop_if_no_hosts_left ### Stop if all hosts are filtered and there are no hosts left to run on +par_stop_if_no_hosts_left parallel: Warning: Removed no-such.host. +par_stop_if_no_hosts_left parallel: Error: Cannot run any jobs. +par_trc_colon ### Test --trc ./:dir/:foo2 +par_trc_colon Content :dir/:foo2 +par_trc_colon remote1 +par_trc_colon Content :dir/:foo2 +par_trc_colon remote2 +par_trc_colondot ### Test --trc ./:/./:foo5 +par_trc_colondot Content :a/./:foo5 +par_trc_colondot remote./:a/./:foo5 +par_trc_dashdash ### Test --trc ./--/--foo1 +par_trc_dashdash Content --/--foo1 +par_trc_dashdash remote1 +par_trc_dashdash Content --/--foo1 +par_trc_dashdash remote2 +par_trc_dashdashdashspace ### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B" +par_trc_dashdashdashspace ./ ---/B.a +par_trc_dashdashdashspace ./--- /A.a +par_trc_dashdashdashspace ls: No match. +par_trc_dashdashdashspace ls: cannot access './?---': No such file or directory +par_trc_dashdashdashspace ls: cannot access './---?': No such file or directory +par_trc_dashdashdashspace OK +par_trc_dashdashdashspace OK +par_trc_dashdashdot ### Test --trc ./--/./--foo4 +par_trc_dashdashdot Content --/./--foo4 +par_trc_dashdashdot remote--/./--foo4 +par_trc_dashdashspace ### Test --trc "-- " "-- " +par_trc_dashdashspace ./ --.a +par_trc_dashdashspace ./-- .a +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './ --': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './ --': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './-- ': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './-- ': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './ --.a': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './ --.a': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './-- .a': No such file or directory +par_trc_dashdashspace OK +par_trc_dashdashspace ls: cannot access './-- .a': No such file or directory +par_trc_space ### Test --trc ./" "/" "foo3 +par_trc_space Content _/_foo3 +par_trc_space remote1 +par_trc_space Content _/_foo3 +par_trc_space remote2 +par_trc_spacedot ### Test --trc ./" "/./" "foo6 +par_trc_spacedot Content _/./_foo6 +par_trc_spacedot remote./ /./ foo6 +par_wd_dashdash ### Test --wd -- +par_wd_dashdash OK +par_wd_dotdotdot ### Test --wd ... --cleanup which should remove the filled tmp dir +par_wd_dotdotdot echo >uNiQuE_sTrInG.6 +par_wd_quote ### Test --wd "'" +par_wd_quote OK +par_wd_space ### Test --wd " " +par_wd_space OK +par_workdir_home ### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME +par_workdir_home ~