parallel: --xapply => --link.

This commit is contained in:
Ole Tange 2016-08-13 19:11:15 +02:00
parent 393cfd177a
commit a7b50a054e
14 changed files with 631 additions and 234 deletions

View file

@ -219,7 +219,7 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20160722 ('Munich/Erdogan') released <<[stable]>>
Subject: GNU Parallel 20160722 ('Og Nomekop/Munich/Erdogan') released <<[stable]>>
GNU Parallel 20160722 ('Munich/Erdogan') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/

View file

@ -875,8 +875,8 @@ sub options_hash {
"block|block-size|blocksize=s" => \$opt::blocksize,
"tollef" => \$opt::tollef,
"gnu" => \$opt::gnu,
"xapply" => \$opt::xapply,
"xapplyinputsource=i" => \@opt::xapplyinputsource,
"link|xapply" => \$opt::link,
"linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource,
"bibtex|citation" => \$opt::citation,
"wc|willcite|will-cite|nn|nonotice|no-notice" => \$opt::willcite,
# Termination and retries
@ -1697,12 +1697,12 @@ sub read_args_from_command_line {
# Arguments given on the command line after:
# ::: ($Global::arg_sep)
# :::: ($Global::arg_file_sep)
# :::+ ($Global::arg_sep with --xapply)
# ::::+ ($Global::arg_file_sep with --xapply)
# :::+ ($Global::arg_sep with --link)
# ::::+ ($Global::arg_file_sep with --link)
# Removes the arguments from @ARGV and:
# - puts filenames into -a
# - puts arguments into files and add the files to -a
# - adds --xapplyinputsource with 0/1 for each -a depending on :::+/::::+
# - adds --linkinputsource with 0/1 for each -a depending on :::+/::::+
# Input:
# @::ARGV = command option ::: arg arg arg :::: argfiles
# Uses:
@ -1722,7 +1722,7 @@ sub read_args_from_command_line {
$arg eq $Global::arg_file_sep
or
$arg eq $Global::arg_file_sep."+") {
my $group = $arg; # This group of arguments is args or argfiles
my $group_sep = $arg; # This group of arguments is args or argfiles
my @group;
while(defined ($arg = shift @ARGV)) {
if($arg eq $Global::arg_sep
@ -1740,24 +1740,29 @@ sub read_args_from_command_line {
push @group, $arg;
}
}
if($group=~/\+$/) {
# :::+ or ::::+
push @opt::xapplyinputsource, 1;
if($group_sep eq $Global::arg_sep."+") {
# :::+
push @opt::linkinputsource, 1;
} elsif($group_sep eq $Global::arg_file_sep."+") {
# ::::+
for(@group) {
push @opt::linkinputsource, 1;
}
} else {
push @opt::xapplyinputsource, 0;
push @opt::linkinputsource, 0;
}
if($group eq $Global::arg_file_sep
if($group_sep eq $Global::arg_file_sep
or
$group eq $Global::arg_file_sep."+"
$group_sep eq $Global::arg_file_sep."+"
or ($opt::internal_pipe_means_argfiles and $opt::pipe)
) {
# Group of file names on the command line.
# Append args into -a
push @opt::a, @group;
} elsif($group eq $Global::arg_sep
} elsif($group_sep eq $Global::arg_sep
or
$group eq $Global::arg_sep."+") {
$group_sep eq $Global::arg_sep."+") {
# Group of arguments on the command line.
# Put them into a file.
# Create argfile
@ -1770,7 +1775,7 @@ sub read_args_from_command_line {
# Append filehandle to -a
push @opt::a, $outfh;
} else {
::die_bug("Unknown command line group: $group");
::die_bug("Unknown command line group: $group_sep");
}
if(defined($arg)) {
# $arg is ::: or ::::
@ -9582,8 +9587,8 @@ sub new {
sub get {
my $self = shift;
if($opt::xapply) {
return $self->xapply_get();
if($opt::link) {
return $self->link_get();
} else {
return $self->nest_get();
}
@ -9606,7 +9611,7 @@ sub empty {
return $empty;
}
sub xapply_get {
sub link_get {
my $self = shift;
if(@{$self->{'unget'}}) {
return shift @{$self->{'unget'}};
@ -9689,8 +9694,8 @@ sub nest_get {
my @combarg = ();
for (my $fhn = 0; $fhn < $no_of_inputsources; $fhn++) {
push(@combarg, [0, $#{$self->{'arg_matrix'}[$fhn]}],
# Is input source --xapply linked to the next?
$opt::xapplyinputsource[$fhn+1]);
# Is input source --link'ed to the next?
$opt::linkinputsource[$fhn+1]);
}
$combarg[2*$fhno] = [$len,$len]; # Find only combinations with this new entry
# map combinations
@ -9777,14 +9782,14 @@ sub expand_combinations {
# Returns: ([x,y,...],[x,y,...])
# where xmin <= x <= xmax and ymin <= y <= ymax
my $minmax_ref = shift;
my $xapply = shift; # This is linked to the next input source
my $link = shift; # This is linked to the next input source
my $xmin = $$minmax_ref[0];
my $xmax = $$minmax_ref[1];
my @p;
if(@_) {
my @rest = expand_combinations(@_);
if($xapply) {
# Linked to next col with xapply
if($link) {
# Linked to next col with --link/:::+/::::+
# TODO BUG does not wrap values if not same number of vals
push(@p, map { [$$_[0], @$_] }
grep { $xmin <= $$_[0] and $$_[0] <= $xmax } @rest);

View file

@ -336,9 +336,9 @@ B<:::> and B<::::> can be mixed. So these are equivalent:
=item B<:::+> I<arguments>
Like B<:::> but linked like B<--xapply> to the previous input source.
Like B<:::> but linked like B<--link> to the previous input source.
Contrary to B<--xapply> values do not wrap: The shortest input source
Contrary to B<--link>, values do not wrap: The shortest input source
determines the length.
Example:
@ -352,14 +352,14 @@ Another way to write B<-a> I<argfile1> B<-a> I<argfile2> ...
B<:::> and B<::::> can be mixed.
See B<-a>, B<:::> and B<--xapply>.
See B<-a>, B<:::> and B<--link>.
=item B<::::+> I<argfiles>
Like B<::::+> but linked like B<--xapply> to the previous input source.
Like B<::::> but linked like B<--link> to the previous input source.
Contrary to B<--xapply> values do not wrap: The shortest input source
Contrary to B<--link>, values do not wrap: The shortest input source
determines the length.
@ -387,7 +387,7 @@ contains B<a b c>. B<-a foo> B<-a bar> will result in the combinations
(1,a) (1,b) (1,c) (2,a) (2,b) (2,c). This is useful for replacing
nested for-loops.
See also B<--xapply> and B<{>I<n>B<}>.
See also B<--link> and B<{>I<n>B<}>.
=item B<--arg-file-sep> I<sep-str>
@ -551,8 +551,8 @@ Compress temporary files. If the output is big and very compressible
this will take up less disk space in $TMPDIR and possibly be faster
due to less disk I/O.
GNU B<parallel> will try B<lz4>, B<pigz>, B<lzop>, B<plzip>,
B<pbzip2>, B<pxz>, B<gzip>, B<lzma>, B<xz>, B<bzip2>, B<lzip> in that
GNU B<parallel> will try B<lz4>, B<lzop>, B<pbzip2>, B<pigz>,
B<plzip>, B<lzip> B<gzip>, B<pxz>, B<lzma>, B<xz>, B<bzip2>, in that
order, and use the first available.
@ -1004,6 +1004,30 @@ limiting factor.
See also: B<--group> B<--ungroup>
=item B<--xapply> (alpha testing)
=item B<--link> (alpha testing)
Link input sources. Read multiple input sources like B<xapply>. If
multiple input sources are given, one argument will be read from each
of the input sources. The arguments can be accessed in the command as
B<{1}> .. B<{>I<n>B<}>, so B<{1}> will be a line from the first input
source, and B<{6}> will refer to the line with the same line number
from the 6th input source.
Compare these two:
parallel echo {1} {2} ::: 1 2 3 ::: a b c
parallel --link echo {1} {2} ::: 1 2 3 ::: a b c
Arguments will be recycled if one input source has more arguments than the others:
parallel --link echo {1} {2} {3} \
::: 1 2 ::: I II III ::: a b c d e f g
See also B<--header>, B<:::+>, B<::::+>.
=item B<--load> I<max-load>
Do not start new jobs on a given computer unless the number of running
@ -1113,6 +1137,7 @@ of each job is saved in a file and the filename is then printed.
See also: B<--results>
=item B<--pipe>
=item B<--spreadstdin>
@ -1155,7 +1180,6 @@ If B<--block> is left out, B<--pipepart> will use a block size that
will result in 10 jobs per jobslot, except if run with
B<--round-robin> in which case it will result in 1 job per jobslot.
B<--pipepart> has a few limitations:
=over 3
@ -2302,27 +2326,6 @@ See also B<-m>.
Exit if the size (see the B<-s> option) is exceeded.
=item B<--xapply>
Read multiple input sources like B<xapply>. If multiple input sources
are given, one argument will be read from each of the input
sources. The arguments can be accessed in the command as B<{1}>
.. B<{>I<n>B<}>, so B<{1}> will be a line from the first input source, and
B<{6}> will refer to the line with the same line number from the 6th
input source.
Compare these two:
parallel echo {1} {2} ::: 1 2 3 ::: a b c
parallel --xapply echo {1} {2} ::: 1 2 3 ::: a b c
Arguments will be recycled if one input source has more arguments than the others:
parallel --xapply echo {1} {2} {3} \
::: 1 2 ::: I II III ::: a b c d e f g
See also B<--header>.
=back
=head1 EXAMPLE: Working as xargs -n1. Argument appending
@ -3339,13 +3342,13 @@ Count in binary:
Print the number on the opposing sides of a six sided die:
parallel --xapply -a <(seq 6) -a <(seq 6 -1 1) echo
parallel --xapply echo :::: <(seq 6) <(seq 6 -1 1)
parallel --link -a <(seq 6) -a <(seq 6 -1 1) echo
parallel --link echo :::: <(seq 6) <(seq 6 -1 1)
Convert files from all subdirs to PNG-files with consecutive numbers
(useful for making input PNG's for B<ffmpeg>):
parallel --xapply -a <(find . -type f | sort) \
parallel --link -a <(find . -type f | sort) \
-a <(seq $(find . -type f|wc -l)) convert {1} {2}.png
Alternative version:
@ -3613,11 +3616,12 @@ system/batch manager: You have to submit JobSlot number of jobs before
they will start, and after that you can submit one at a time, and job
will start immediately if free slots are available. Output from the
running or completed jobs are held back and will only be printed when
JobSlots more jobs has been started (unless you use --ungroup or -u,
in which case the output from the jobs are printed immediately).
E.g. if you have 10 jobslots then the output from the first completed
job will only be printed when job 11 has started, and the output of
second completed job will only be printed when job 12 has started.
JobSlots more jobs has been started (unless you use --ungroup or
--line-buffer, in which case the output from the jobs are printed
immediately). E.g. if you have 10 jobslots then the output from the
first completed job will only be printed when job 11 has started, and
the output of second completed job will only be printed when job 12
has started.
=head1 EXAMPLE: GNU Parallel as dir processor
@ -4432,7 +4436,7 @@ B<2> parallel diff {} ../version5/{} < manifest | more
B<3> xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1
B<3> parallel --xapply diff {1} {2} :::: manifest1 checklist1
B<3> parallel --link diff {1} {2} :::: manifest1 checklist1
B<4> xapply 'indent' *.c

View file

@ -15,13 +15,15 @@ To run this tutorial you must have the following:
=over 9
=item parallel >= version 20160222
=item parallel >= version 20160822
Install the newest version using your package manager or with:
Install the newest version using your package manager or with this
command:
(wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
This will also install the newest version of the tutorial:
This will also install the newest version of the tutorial which you
can see by running this:
man parallel_tutorial
@ -30,71 +32,71 @@ Most of the tutorial will work on older versions, too.
=item abc-file:
The file can be generated by:
The file can be generated by this command:
parallel -k echo ::: A B C > abc-file
=item def-file:
The file can be generated by:
The file can be generated by this command:
parallel -k echo ::: D E F > def-file
=item abc0-file:
The file can be generated by:
The file can be generated by this command:
perl -e 'printf "A\0B\0C\0"' > abc0-file
=item abc_-file:
The file can be generated by:
The file can be generated by this command:
perl -e 'printf "A_B_C_"' > abc_-file
=item tsv-file.tsv
The file can be generated by:
The file can be generated by this command:
perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' > tsv-file.tsv
=item num8
The file can be generated by:
The file can be generated by this command:
perl -e 'for(1..8){print "$_\n"}' > num8
=item num128
The file can be generated by:
The file can be generated by this command:
perl -e 'for(1..128){print "$_\n"}' > num128
=item num30000
The file can be generated by:
The file can be generated by this command:
perl -e 'for(1..30000){print "$_\n"}' > num30000
=item num1000000
The file can be generated by:
The file can be generated by this command:
perl -e 'for(1..1000000){print "$_\n"}' > num1000000
=item num_%header
The file can be generated by:
The file can be generated by this command:
(echo %head1; echo %head2; perl -e 'for(1..10){print "$_\n"}') > num_%header
=item For remote running: ssh login on 2 servers with no password in
$SERVER1 and $SERVER2
$SERVER1 and $SERVER2 must work.
SERVER1=server.example.com
SERVER2=server2.example.net
You must be able to:
So you must be able to do this:
ssh $SERVER1 echo works
ssh $SERVER2 echo works
@ -180,11 +182,12 @@ Output: Same as above.
Output: Same as above.
=head3 Matching arguments from all input sources
=head3 Linking arguments from input sources
With B<--xapply> you can get one argument from each input source:
With B<--link> you can link the input sources and get one argument
from each input source:
parallel --xapply echo ::: A B C ::: D E F
parallel --link echo ::: A B C ::: D E F
Output (the order may be different):
@ -194,7 +197,7 @@ Output (the order may be different):
If one of the input sources is too short, its values will wrap:
parallel --xapply echo ::: A B C D E ::: F G
parallel --link echo ::: A B C D E ::: F G
Output (the order may be different):
@ -204,6 +207,53 @@ Output (the order may be different):
D G
E F
For more flexible linking you can use B<:::+> and B<::::+>. They work
like B<:::> and B<::::> except they link the previous input source to
this input source.
This will link ABC to GHI:
parallel echo :::: abc-file :::+ G H I :::: def-file
Output (the order may be different):
A G D
A G E
A G F
B H D
B H E
B H F
C I D
C I E
C I F
This will link GHI to DEF:
parallel echo :::: abc-file ::: G H I ::::+ def-file
Output (the order may be different):
A G D
A H E
A I F
B G D
B H E
B I F
C G D
C H E
C I F
If one of the input sources is too short when using B<:::+> or
B<::::+>, the rest will be ignored:
parallel echo ::: A B C D E :::+ F G
Output (the order may be different):
A F
B G
=head2 Changing the argument separator.
GNU B<parallel> can use other separators than B<:::> or B<::::>. This is
@ -243,7 +293,7 @@ Output (the order may be different):
B
C
NULL can be given as B<\0>:
NUL can be given as B<\0>:
parallel -d '\0' echo :::: abc0-file

View file

@ -11,7 +11,7 @@ INSTALL="echo sudo aptitude -y install"
# The testsuite depends on this:
$INSTALL imagemagick expect autossh sshpass jq libpod-simple-perl pod2pdf
$INSTALL lua5.2 clisp php7.0-cli nodejs-legacy mono-csharp-shell
# DEBIAN package
$INSTALL dpkg-dev build-essential debhelper

View file

@ -120,8 +120,10 @@ echo '### TMUX not found'
echo '**'
parallel -j4 --halt 2 ::: 'sleep 1' burnP6 false; killall burnP6 && echo ERROR: burnP6 should be killed
parallel -j4 --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burnP5 should be killed
parallel -j4 --halt 2 ::: 'sleep 1' burnP6 false;
killall burnP6 && echo ERROR: burnP6 should already have been killed
parallel -j4 --halt -2 ::: 'sleep 1' burnP5 true;
killall burnP5 && echo ERROR: burnP5 should already have been killed
parallel --halt error echo ::: should not print
parallel --halt soon echo ::: should not print
@ -304,9 +306,9 @@ echo '### test too long args'
perl -e 'print "z"x1000000' | parallel echo 2>&1
perl -e 'print "z"x1000000' | xargs echo 2>&1
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout xargs -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -kX -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -km -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort xargs -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -kX -s 10 echo
echo '### Test -x'
@ -631,6 +633,12 @@ echo '**'
echo '**'
echo '### bug #48745: :::+ bug'
parallel -k echo ::: 11 22 33 ::::+ <(seq 3) <(seq 21 23) ::: c d e :::+ cc dd ee
echo '**'
EOF
echo '### 1 .par file from --files expected'
find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l

View file

@ -1,8 +1,7 @@
#!/bin/bash
seq 1 2 >/tmp/in12
seq 4 5 >/tmp/in45
par_basic_shebang_wrap() {
echo "### Test basic --shebang-wrap"
cat <<EOF > /tmp/basic--shebang-wrap
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl
@ -19,7 +18,11 @@ echo "### Test basic --shebang-wrap stdin"
echo "### Test basic --shebang-wrap Same as"
(echo arg1; echo arg2) | parallel -k /usr/bin/perl /tmp/basic--shebang-wrap
rm /tmp/basic--shebang-wrap
}
par_shebang_with_parser_options() {
seq 1 2 >/tmp/in12
seq 4 5 >/tmp/in45
echo "### Test --shebang-wrap with parser options"
cat <<EOF > /tmp/with-parser--shebang-wrap
@ -55,3 +58,236 @@ rm /tmp/pipe--shebang-wrap
rm /tmp/in12
rm /tmp/in45
}
par_shebang_wrap_perl() {
F=/tmp/shebang_wrap_perl
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl
print "Arguments @ARGV\n";
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_python() {
F=/tmp/shebang_wrap_python
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/python
import sys
print 'Arguments', str(sys.argv)
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_bash() {
F=/tmp/shebang_wrap_bash
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /bin/bash
echo Arguments "$@"
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_sh() {
F=/tmp/shebang_wrap_sh
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /bin/sh
echo Arguments "$@"
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_ksh() {
F=/tmp/shebang_wrap_ksh
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ksh
echo Arguments "$@"
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_zsh() {
F=/tmp/shebang_wrap_zsh
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/zsh
echo Arguments "$@"
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_csh() {
F=/tmp/shebang_wrap_csh
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /bin/csh
echo Arguments "$argv"
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_tcl() {
F=/tmp/shebang_wrap_tcl
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/tclsh
puts "Arguments $argv"
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_R() {
F=/tmp/shebang_wrap_R
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave
args <- commandArgs(trailingOnly = TRUE)
print(paste("Arguments ",args))
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_gnuplot() {
F=/tmp/shebang_wrap_gnuplot
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k ARG={} /usr/bin/gnuplot
print "Arguments ", system('echo $ARG')
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_ruby() {
F=/tmp/shebang_wrap_ruby
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ruby
print "Arguments "
puts ARGV
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_octave() {
F=/tmp/shebang_wrap_octave
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/octave
printf ("Arguments");
arg_list = argv ();
for i = 1:nargin
printf (" %s", arg_list{i});
endfor
printf ("\n");
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_clisp() {
F=/tmp/shebang_wrap_clisp
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/clisp
(format t "~&~S~&" 'Arguments)
(format t "~&~S~&" *args*)
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_php() {
F=/tmp/shebang_wrap_php
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/php
<?php
echo "Arguments";
foreach(array_slice($argv,1) as $v)
{
echo " $v";
}
echo "\n";
?>
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_nodejs() {
F=/tmp/shebang_wrap_nodejs
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/node
var myArgs = process.argv.slice(2);
console.log('Arguments ', myArgs);
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_lua() {
F=/tmp/shebang_wrap_lua
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/lua
io.write "Arguments"
for a = 1, #arg do
io.write(" ")
io.write(arg[a])
end
print("")
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
par_shebang_wrap_csharp() {
F=/tmp/shebang_wrap_csharp
cat <<'EOF' > $F
#!/usr/local/bin/parallel --shebang-wrap -k ARGV={} /usr/bin/csharp
var argv = Environment.GetEnvironmentVariable("ARGV");
print("Arguments "+argv);
EOF
chmod 755 $F
$F arg1 arg2 arg3
rm $F
}
export -f $(compgen -A function | grep par_)
# Tested with -j1..8
# -j6 was fastest
#compgen -A function | grep par_ | sort | parallel -j$P --tag -k '{} 2>&1'
compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1'

View file

@ -2,7 +2,7 @@
# Test xargs compatibility
rm -f ~/.parallel/will-cite
#rm -f ~/.parallel/will-cite
echo '### Test -p --interactive'
cat >/tmp/parallel-script-for-expect <<_EOF

View file

@ -153,11 +153,11 @@ echo '### TMUX not found'
parallel: Error: not-existing not found in $PATH.
echo '**'
**
parallel -j4 --halt 2 ::: 'sleep 1' burnP6 false; killall burnP6 && echo ERROR: burnP6 should be killed
parallel -j4 --halt 2 ::: 'sleep 1' burnP6 false; killall burnP6 && echo ERROR: burnP6 should already have been killed
parallel: This job failed:
false
burnP6: no process found
parallel -j4 --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burnP5 should be killed
parallel -j4 --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burnP5 should already have been killed
parallel: This job succeeded:
true
burnP5: no process found
@ -308,11 +308,11 @@ OK
/mnt/4tb/home/tange/privat/parallel/testsuite
OK
parallel --wd ... 'pwd; echo $OLDPWD; echo' ::: OK | perl -pe 's/\d+/0/g'
/mnt/0tb/home/tange/.parallel/tmp/aspire-0-0
/home/tange/.parallel/tmp/aspire-0-0
/mnt/0tb/home/tange/privat/parallel/testsuite
OK
parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK
/mnt/4tb/home/tange/privat/parallel/testsuite
/home/tange/privat/parallel/testsuite
/mnt/4tb/home/tange/privat/parallel/testsuite
OK
echo '**'
@ -597,21 +597,21 @@ e
parallel: Error: Command line too long (1000005 >= 65528) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
perl -e 'print "z"x1000000' | xargs echo 2>&1
xargs: argument line too long
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -km -s 10 echo
1 2
3 4
5 6
7 8
9 10
parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout xargs -s 10 echo
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort xargs -s 10 echo
1 2
3 4
5 6
7 8
xargs: argument line too long
9 10
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -kX -s 10 echo
xargs: argument line too long
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -kX -s 10 echo
1 2
3 4
5 6
@ -1689,5 +1689,19 @@ echo '**'
env_parallel
echo '**'
**
echo '### bug #48745: :::+ bug'
### bug #48745: :::+ bug
parallel -k echo ::: 11 22 33 ::::+ <(seq 3) <(seq 21 23) ::: c d e :::+ cc dd ee
11 1 21 c cc
11 1 21 d dd
11 1 21 e ee
22 2 22 c cc
22 2 22 d dd
22 2 22 e ee
33 3 23 c cc
33 3 23 d dd
33 3 23 e ee
echo '**'
**
### 1 .par file from --files expected
0

View file

@ -20,22 +20,22 @@ echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
** test_zsh
FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_zsh
HOME=/mnt/4tb/home/tange
HOME=/home/tange
echo '** test_zsh_filter'
** test_zsh_filter
FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_zsh_filter
HOME=/mnt/4tb/home/tange
HOME=/home/tange
echo '** test_csh'
** test_csh
FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_csh
HOME=/mnt/4tb/home/tange
HOME=/home/tange
echo '** test_csh_filter'
** test_csh_filter
FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
FOO=test_csh_filter
HOME=/mnt/4tb/home/tange
HOME=/home/tange
echo '** bug #41805 done'
** bug #41805 done
echo '### Deal with long command lines on remote servers'

View file

@ -1,67 +1,121 @@
### Test basic --shebang-wrap
Shebang from perl with args arg1
Shebang from perl with args arg2
### Test basic --shebang-wrap Same as
Shebang from perl with args arg1
Shebang from perl with args arg2
### Test basic --shebang-wrap stdin
Shebang from perl with args arg1
Shebang from perl with args arg2
### Test basic --shebang-wrap Same as
Shebang from perl with args arg1
Shebang from perl with args arg2
### Test --shebang-wrap with parser options
Shebang from perl with args
1
Shebang from perl with args
2
Shebang from perl with args
4
Shebang from perl with args
5
### Test --shebang-wrap with parser options Same as
Shebang from perl with args
1
Shebang from perl with args
2
Shebang from perl with args
4
Shebang from perl with args
5
### Test --shebang-wrap with parser options stdin
Shebang from perl with args
1
Shebang from perl with args
2
Shebang from perl with args
4
Shebang from perl with args
5
### Test --shebang-wrap with parser options Same as
Shebang from perl with args
1
Shebang from perl with args
2
Shebang from perl with args
4
Shebang from perl with args
5
### Test --shebang-wrap --pipe with parser options
### Test --shebang-wrap --pipe with parser options stdin
Shebang from perl with args
1
Shebang from perl with args
2
Shebang from perl with args
4
Shebang from perl with args
5
### Test --shebang-wrap --pipe with parser options Same as
Shebang from perl with args
1
Shebang from perl with args
2
Shebang from perl with args
4
Shebang from perl with args
5
par_basic_shebang_wrap ### Test basic --shebang-wrap
par_basic_shebang_wrap Shebang from perl with args arg1
par_basic_shebang_wrap Shebang from perl with args arg2
par_basic_shebang_wrap ### Test basic --shebang-wrap Same as
par_basic_shebang_wrap Shebang from perl with args arg1
par_basic_shebang_wrap Shebang from perl with args arg2
par_basic_shebang_wrap ### Test basic --shebang-wrap stdin
par_basic_shebang_wrap Shebang from perl with args arg1
par_basic_shebang_wrap Shebang from perl with args arg2
par_basic_shebang_wrap ### Test basic --shebang-wrap Same as
par_basic_shebang_wrap Shebang from perl with args arg1
par_basic_shebang_wrap Shebang from perl with args arg2
par_shebang_with_parser_options ### Test --shebang-wrap with parser options
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 1
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 2
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 4
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 5
par_shebang_with_parser_options ### Test --shebang-wrap with parser options Same as
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 1
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 2
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 4
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 5
par_shebang_with_parser_options ### Test --shebang-wrap with parser options stdin
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 1
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 2
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 4
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 5
par_shebang_with_parser_options ### Test --shebang-wrap with parser options Same as
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 1
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 2
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 4
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 5
par_shebang_with_parser_options ### Test --shebang-wrap --pipe with parser options
par_shebang_with_parser_options ### Test --shebang-wrap --pipe with parser options stdin
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 1
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 2
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 4
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 5
par_shebang_with_parser_options ### Test --shebang-wrap --pipe with parser options Same as
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 1
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 2
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 4
par_shebang_with_parser_options Shebang from perl with args
par_shebang_with_parser_options 5
par_shebang_wrap_R [1] "Arguments arg1"
par_shebang_wrap_R [1] "Arguments arg2"
par_shebang_wrap_R [1] "Arguments arg3"
par_shebang_wrap_bash Arguments arg1
par_shebang_wrap_bash Arguments arg2
par_shebang_wrap_bash Arguments arg3
par_shebang_wrap_clisp ARGUMENTS
par_shebang_wrap_clisp ("arg1")
par_shebang_wrap_clisp ARGUMENTS
par_shebang_wrap_clisp ("arg2")
par_shebang_wrap_clisp ARGUMENTS
par_shebang_wrap_clisp ("arg3")
par_shebang_wrap_csh Arguments arg1
par_shebang_wrap_csh Arguments arg2
par_shebang_wrap_csh Arguments arg3
par_shebang_wrap_csharp Arguments arg1
par_shebang_wrap_csharp Arguments arg2
par_shebang_wrap_csharp Arguments arg3
par_shebang_wrap_gnuplot Arguments arg1
par_shebang_wrap_gnuplot Arguments arg2
par_shebang_wrap_gnuplot Arguments arg3
par_shebang_wrap_ksh Arguments arg1
par_shebang_wrap_ksh Arguments arg2
par_shebang_wrap_ksh Arguments arg3
par_shebang_wrap_lua Arguments arg1
par_shebang_wrap_lua Arguments arg2
par_shebang_wrap_lua Arguments arg3
par_shebang_wrap_nodejs Arguments [ 'arg1' ]
par_shebang_wrap_nodejs Arguments [ 'arg2' ]
par_shebang_wrap_nodejs Arguments [ 'arg3' ]
par_shebang_wrap_octave Arguments arg1
par_shebang_wrap_octave Arguments arg2
par_shebang_wrap_octave Arguments arg3
par_shebang_wrap_perl Arguments arg1
par_shebang_wrap_perl Arguments arg2
par_shebang_wrap_perl Arguments arg3
par_shebang_wrap_php Arguments arg1
par_shebang_wrap_php Arguments arg2
par_shebang_wrap_php Arguments arg3
par_shebang_wrap_python Arguments ['/tmp/shebang_wrap_python', 'arg1']
par_shebang_wrap_python Arguments ['/tmp/shebang_wrap_python', 'arg2']
par_shebang_wrap_python Arguments ['/tmp/shebang_wrap_python', 'arg3']
par_shebang_wrap_ruby Arguments arg1
par_shebang_wrap_ruby Arguments arg2
par_shebang_wrap_ruby Arguments arg3
par_shebang_wrap_sh Arguments arg1
par_shebang_wrap_sh Arguments arg2
par_shebang_wrap_sh Arguments arg3
par_shebang_wrap_tcl Arguments arg1
par_shebang_wrap_tcl Arguments arg2
par_shebang_wrap_tcl Arguments arg3
par_shebang_wrap_zsh Arguments arg1
par_shebang_wrap_zsh Arguments arg2
par_shebang_wrap_zsh Arguments arg3

View file

@ -1,13 +1,13 @@
bug #46120: Suspend should suspend (at least local) children
1024 SHA256:SXgag2Z2L91JsrT5WjNBcARD1EpyCCj4JctVJ6Zpzm0 /mnt/4tb/home/tange/.ssh/id_dsa (DSA)
8192 SHA256:lYn04AefJq/5r0e4FftceviJ7JVnq9NGKY3CW9XMpO8 /mnt/4tb/home/tange/.ssh/id_rsa (RSA)
4096 SHA256:jUQ9ysfprs7UOckttjjVb+j3qikUmKWDEWC+eEJkbDQ /mnt/4tb/home/tange/.ssh/id_rsa_openindiana (RSA)
1024 SHA256:SXgag2Z2L91JsrT5WjNBcARD1EpyCCj4JctVJ6Zpzm0 /home/tange/.ssh/id_dsa (DSA)
8192 SHA256:lYn04AefJq/5r0e4FftceviJ7JVnq9NGKY3CW9XMpO8 /home/tange/.ssh/id_rsa (RSA)
4096 SHA256:jUQ9ysfprs7UOckttjjVb+j3qikUmKWDEWC+eEJkbDQ /home/tange/.ssh/id_rsa_openindiana (RSA)
stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 ::: 1 | grep -q CPUTIME=1
Zero=OK 0
1024 SHA256:SXgag2Z2L91JsrT5WjNBcARD1EpyCCj4JctVJ6Zpzm0 /mnt/4tb/home/tange/.ssh/id_dsa (DSA)
8192 SHA256:lYn04AefJq/5r0e4FftceviJ7JVnq9NGKY3CW9XMpO8 /mnt/4tb/home/tange/.ssh/id_rsa (RSA)
4096 SHA256:jUQ9ysfprs7UOckttjjVb+j3qikUmKWDEWC+eEJkbDQ /mnt/4tb/home/tange/.ssh/id_rsa_openindiana (RSA)
1024 SHA256:SXgag2Z2L91JsrT5WjNBcARD1EpyCCj4JctVJ6Zpzm0 /home/tange/.ssh/id_dsa (DSA)
8192 SHA256:lYn04AefJq/5r0e4FftceviJ7JVnq9NGKY3CW9XMpO8 /home/tange/.ssh/id_rsa (RSA)
4096 SHA256:jUQ9ysfprs7UOckttjjVb+j3qikUmKWDEWC+eEJkbDQ /home/tange/.ssh/id_rsa_openindiana (RSA)
echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 | grep -q CPUTIME=1
Zero=OK 0

View file

@ -82,16 +82,39 @@ B F
C D
C E
C F
parallel --xapply echo ::: A B C ::: D E F
parallel --link echo ::: A B C ::: D E F
A D
B E
C F
parallel --xapply echo ::: A B C D E ::: F G
parallel --link echo ::: A B C D E ::: F G
A F
B G
C F
D G
E F
parallel echo :::: abc-file :::+ G H I :::: def-file
A G D
A G E
A G F
B H D
B H E
B H F
C I D
C I E
C I F
parallel echo :::: abc-file ::: G H I ::::+ def-file
A G D
A H E
A I F
B G D
B H E
B I F
C G D
C H E
C I F
parallel echo ::: A B C D E :::+ F G
A F
B G
parallel --arg-sep ,, echo ,, A B C :::: def-file
A D
A E
@ -524,7 +547,7 @@ BASE64-Message: GtkDialog mapped without a transient parent. This is discouraged
cat /tmp/log;
parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log;
parallel --resume-failed --joblog /tmp/log
parallel --retry-failed --joblog /tmp/log
cat /tmp/log;
parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
@ -564,6 +587,9 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
9
9
9
@ -709,34 +735,34 @@ Unknown option: :::
Unknown option: green
env_parallel only works if it is a function. Do the below and restart your shell.
bash: Put this in /mnt/4tb/home/tange/.bashrc: . /usr/local/bin/env_parallel.bash
E.g. by doing: echo '. /usr/local/bin/env_parallel.bash' >> /mnt/4tb/home/tange/.bashrc
bash: Put this in /home/tange/.bashrc: . /usr/local/bin/env_parallel.bash
E.g. by doing: echo '. /usr/local/bin/env_parallel.bash' >> /home/tange/.bashrc
Supports: aliases, functions, variables, arrays
zsh: Put this in /mnt/4tb/home/tange/.zshrc: . /usr/local/bin/env_parallel.zsh
E.g. by doing: echo '. /usr/local/bin/env_parallel.zsh' >> /mnt/4tb/home/tange/.zshenv
zsh: Put this in /home/tange/.zshrc: . /usr/local/bin/env_parallel.zsh
E.g. by doing: echo '. /usr/local/bin/env_parallel.zsh' >> /home/tange/.zshenv
Supports: functions, variables, arrays
fish: Put this in /mnt/4tb/home/tange/.config/fish/config.fish:
fish: Put this in /home/tange/.config/fish/config.fish:
. (which env_parallel.fish)
E.g. by doing:
echo '. (which env_parallel.fish)' >> /mnt/4tb/home/tange/.config/fish/config.fish
echo '. (which env_parallel.fish)' >> /home/tange/.config/fish/config.fish
Supports: aliases, functions, variables, arrays
ksh: Put this in /mnt/4tb/home/tange/.kshrc: source /usr/local/bin/env_parallel.ksh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.ksh' >> /mnt/4tb/home/tange/.kshrc
ksh: Put this in /home/tange/.kshrc: source /usr/local/bin/env_parallel.ksh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.ksh' >> /home/tange/.kshrc
Supports: aliases, functions, variables, arrays
pdksh: Put this in /mnt/4tb/home/tange/.profile: source /usr/local/bin/env_parallel.pdksh
E.g. by doing: echo '. /usr/local/bin/env_parallel.pdksh' >> /mnt/4tb/home/tange/.profile
pdksh: Put this in /home/tange/.profile: source /usr/local/bin/env_parallel.pdksh
E.g. by doing: echo '. /usr/local/bin/env_parallel.pdksh' >> /home/tange/.profile
Supports: aliases, functions, variables, arrays
csh: Put this in /mnt/4tb/home/tange/.cshrc: source /usr/local/bin/env_parallel.csh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.csh' >> /mnt/4tb/home/tange/.cshrc
csh: Put this in /home/tange/.cshrc: source /usr/local/bin/env_parallel.csh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.csh' >> /home/tange/.cshrc
Supports: aliases, variables, arrays with no special chars
tcsh: Put this in /mnt/4tb/home/tange/.tcshrc: source /usr/local/bin/env_parallel.tcsh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.tcsh' >> /mnt/4tb/home/tange/.tcshrc
tcsh: Put this in /home/tange/.tcshrc: source /usr/local/bin/env_parallel.tcsh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.tcsh' >> /home/tange/.tcshrc
Supports: aliases, variables, arrays with no special chars
To install in all shells run:

View file

@ -56,7 +56,7 @@ Error:
sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
### Test dburl :
Error:
: is not defined in /mnt/4tb/home/tange/.sql/aliases /mnt/4tb/home/tange/.dburl.aliases /etc/sql/aliases /usr/local/bin/dburl.aliases /usr/local/bin/dburl.aliases.dist
: is not defined in /home/tange/.sql/aliases /home/tange/.dburl.aliases /etc/sql/aliases /usr/local/bin/dburl.aliases /usr/local/bin/dburl.aliases.dist
sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
### Test oracle with multiple arguments on the command line
@ -90,13 +90,13 @@ arg3
OBJECT_NAME
--------------------------------------------------------------------------------------------------------------------------------
JOB_HISTORY
EMPLOYEES
JOBS
DEPARTMENTS
LOCATIONS
COUNTRIES
REGIONS
COUNTRIES
LOCATIONS
DEPARTMENTS
JOBS
EMPLOYEES
JOB_HISTORY
7 rows selected.