mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-24 23:17:55 +00:00
parallel: bug #54647: parset ignores empty lines.
parallel: minix is working again. parallel: bug #54128: command too long when exporting big env testsuite: polarhome parallelized.
This commit is contained in:
Ole Tange
parent
24de94187a
commit
36a8238a20
|
|
@ -206,7 +206,7 @@ to:parallel@gnu.org, bug-parallel@gnu.org
|
|||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
|
||||
Subject: GNU Parallel 20180922 ('Myanmar/Jacksonville/Chemnitz') released <<[stable]>>
|
||||
Subject: GNU Parallel 20180922 ('Danske/Korea/India/Myanmar/Jacksonville/Chemnitz') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20180922 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
|
||||
|
|
@ -219,7 +219,10 @@ Quote of the month:
|
|||
|
||||
New in this release:
|
||||
|
||||
*
|
||||
* Updated graph showing GNU Parallel's overhead per job for each version. https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf
|
||||
|
||||
http://carc.unm.edu/education--training/workshops-and-seminars.html
|
||||
|
||||
|
||||
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2016,2017,2018
|
||||
# Copyright (C) 2016-2018
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2016,2017,2018
|
||||
# Copyright (C) 2016-2018
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2016,2017,2018
|
||||
# Copyright (C) 2016-2018
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
@ -181,12 +181,12 @@ env_parallel() {
|
|||
# type returns:
|
||||
# ll is an alias for ls -l (in ash)
|
||||
# bash is a tracked alias for /bin/bash
|
||||
# true is a shell builtin
|
||||
# true is a shell builtin (in bash)
|
||||
# myfunc is a function (in bash)
|
||||
# myfunc is a shell function (in zsh)
|
||||
# which is /usr/bin/which
|
||||
# which is /usr/bin/which (in sh, bash)
|
||||
# which is hashed (/usr/bin/which)
|
||||
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
|
||||
# gi is aliased to `grep -i' (in bash)
|
||||
# Return 0 if found, 1 otherwise
|
||||
LANG=C type "$@" |
|
||||
perl -pe '$exit += (s/ is an alias for .*// ||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2016,2017,2018
|
||||
# Copyright (C) 2016-2018
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
82
src/parallel
82
src/parallel
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env perl
|
||||
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,
|
||||
# 2017,2018 Ole Tange and Free Software Foundation, Inc.
|
||||
# Copyright (C) 2007-2018 Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
|
|
@ -1555,7 +1554,7 @@ sub check_invalid_option_combinations {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20180824;
|
||||
$Global::version = 20180916;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -3257,19 +3256,8 @@ sub get_job_with_sshlogin {
|
|||
return undef;
|
||||
}
|
||||
}
|
||||
|
||||
my $clean_command = $job->replaced();
|
||||
if($clean_command =~ /^\s*$/) {
|
||||
# Do not run empty lines
|
||||
if(not $Global::JobQueue->empty()) {
|
||||
return get_job_with_sshlogin($sshlogin);
|
||||
} else {
|
||||
return undef;
|
||||
}
|
||||
}
|
||||
$job->set_sshlogin($sshlogin);
|
||||
if($opt::retries and $clean_command and
|
||||
$job->failed_here()) {
|
||||
if($opt::retries and $job->failed_here()) {
|
||||
# This command with these args failed for this sshlogin
|
||||
my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed();
|
||||
# Only look at the Global::host that have > 0 jobslots
|
||||
|
|
@ -4366,17 +4354,17 @@ sub die_bug {
|
|||
|
||||
sub version {
|
||||
# Returns: N/A
|
||||
print join("\n",
|
||||
"GNU $Global::progname $Global::version",
|
||||
"Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018",
|
||||
"Ole Tange and Free Software Foundation, Inc.",
|
||||
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
|
||||
"This is free software: you are free to change and redistribute it.",
|
||||
"GNU $Global::progname comes with no warranty.",
|
||||
"",
|
||||
"Web site: http://www.gnu.org/software/${Global::progname}\n",
|
||||
"When using programs that use GNU Parallel to process data for publication",
|
||||
"please cite as described in 'parallel --citation'.\n",
|
||||
print join
|
||||
("\n",
|
||||
"GNU $Global::progname $Global::version",
|
||||
"Copyright (C) 2007-2018 Ole Tange and Free Software Foundation, Inc.",
|
||||
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
|
||||
"This is free software: you are free to change and redistribute it.",
|
||||
"GNU $Global::progname comes with no warranty.",
|
||||
"",
|
||||
"Web site: http://www.gnu.org/software/${Global::progname}\n",
|
||||
"When using programs that use GNU Parallel to process data for publication",
|
||||
"please cite as described in 'parallel --citation'.\n",
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -4489,8 +4477,8 @@ sub embed {
|
|||
}
|
||||
print "#!$Global::shell
|
||||
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,
|
||||
# 2017,2018 $user, Ole Tange and Free Software Foundation, Inc.
|
||||
# Copyright (C) 2007-2018 $user, Ole Tange and Free Software
|
||||
# Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
|
|
@ -5022,9 +5010,10 @@ sub which {
|
|||
# Filter for SysV-style `ps`
|
||||
my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
|
||||
q(s/^.{$s}//; print "@F[1,2] $_"' );
|
||||
# Crazy msys: ' is not accepted on the cmd line, but " are treated as '
|
||||
my $msys = q( ps -ef | perl -ane "1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
|
||||
q(s/^.{$s}//; print qq{@F[1,2] $_}" );
|
||||
# Minix uses cols 2,3 and can have newlines in the command
|
||||
# so lines not having numbers in cols 2,3 must be ignored
|
||||
my $minix = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
|
||||
q(s/^.{$s}// and $F[2]>0 and $F[3]>0 and print "@F[2,3] $_"' );
|
||||
# BSD-style `ps`
|
||||
my $bsd = q(ps -o pid,ppid,command -ax);
|
||||
%pid_parentpid_cmd =
|
||||
|
|
@ -5040,7 +5029,8 @@ sub which {
|
|||
'hpux' => $sysv,
|
||||
'linux' => $sysv,
|
||||
'mirbsd' => $bsd,
|
||||
'msys' => $msys,
|
||||
'minix' => $minix,
|
||||
'msys' => $sysv,
|
||||
'MSWin32' => $sysv,
|
||||
'netbsd' => $bsd,
|
||||
'nto' => $sysv,
|
||||
|
|
@ -5639,7 +5629,8 @@ sub swap_activity {
|
|||
# Should we update the swap_activity file?
|
||||
my $update_swap_activity_file = 0;
|
||||
if(-r $self->{'swap_activity_file'}) {
|
||||
open(my $swap_fh, "<", $self->{'swap_activity_file'}) || ::die_bug("swap_activity_file-r");
|
||||
open(my $swap_fh, "<", $self->{'swap_activity_file'}) ||
|
||||
::die_bug("swap_activity_file-r");
|
||||
my $swap_out = <$swap_fh>;
|
||||
close $swap_fh;
|
||||
if($swap_out =~ /^(\d+)$/) {
|
||||
|
|
@ -6170,6 +6161,8 @@ sub compute_number_of_processes {
|
|||
return 0;
|
||||
} else {
|
||||
$job = $Global::JobQueue->get();
|
||||
# Replacement must happen here due to seq()
|
||||
$job and $job->replaced();
|
||||
push(@jobs, $job);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -9954,6 +9947,10 @@ sub len {
|
|||
$self->{'replacecount'}{$replstring};
|
||||
}
|
||||
}
|
||||
if(defined $Global::parallel_env) {
|
||||
# If we are using --env, add the prefix for that, too.
|
||||
$len += length $Global::parallel_env;
|
||||
}
|
||||
if($Global::quoting) {
|
||||
# Pessimistic length if -q is set
|
||||
# Worse than worst case: every char needs to be quoted with \
|
||||
|
|
@ -9964,8 +9961,12 @@ sub len {
|
|||
# Worse than worst case: every char needs to be quoted with \ twice
|
||||
$len *= 4;
|
||||
}
|
||||
# If we are using --env, add the prefix for that, too.
|
||||
$len += 0;
|
||||
if(@opt::sshlogin) {
|
||||
# Pessimistic length if remote
|
||||
# Worst case is BASE64 encoding 3 bytes -> 4 bytes
|
||||
$len = int($len*4/3);
|
||||
}
|
||||
|
||||
return $len;
|
||||
}
|
||||
|
||||
|
|
@ -10513,14 +10514,9 @@ sub get {
|
|||
"(e.g. 'cat').");
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
} else {
|
||||
if($cmd_line->number_of_args() == 0) {
|
||||
# We did not get more args - maybe at EOF string?
|
||||
return undef;
|
||||
} elsif($cmd_line->replaced() eq "") {
|
||||
# Empty command - get the next instead
|
||||
return $self->get();
|
||||
}
|
||||
} elsif($cmd_line->number_of_args() == 0) {
|
||||
# We did not get more args - maybe at EOF string?
|
||||
return undef;
|
||||
}
|
||||
$self->set_seq($self->seq()+1);
|
||||
return $cmd_line;
|
||||
|
|
|
|||
|
|
@ -624,17 +624,20 @@ special characters:
|
|||
echo "The Cure" > My\ brother\'s\ 12\"\ records
|
||||
|
||||
ls | \
|
||||
map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' | bc
|
||||
map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' |
|
||||
bc
|
||||
|
||||
It works with GNU B<parallel>:
|
||||
|
||||
ls | \
|
||||
parallel 'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc
|
||||
parallel \
|
||||
'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc
|
||||
|
||||
And you can even get the file name prepended:
|
||||
|
||||
ls | \
|
||||
parallel --tag '(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc'
|
||||
parallel --tag \
|
||||
'(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc'
|
||||
|
||||
B<map> has no support for grouping. So this gives the wrong results
|
||||
without any warnings:
|
||||
|
|
@ -655,23 +658,22 @@ completely for certain jobs (and may even be considered less readable):
|
|||
|
||||
parallel echo -n {} ::: 1 2 3
|
||||
|
||||
map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | sort | cut -f2- -d:
|
||||
map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | \
|
||||
sort | cut -f2- -d:
|
||||
|
||||
B<map>s replacement strings (% %D %B %E) can be simulated in GNU
|
||||
B<parallel> by putting this in B<~/.parallel/config>:
|
||||
|
||||
--rpl '%'
|
||||
--rpl '%D $_=::shell_quote(::dirname($_));'
|
||||
--rpl '%D $_=Q(::dirname($_));'
|
||||
--rpl '%B s:.*/::;s:\.[^/.]+$::;'
|
||||
--rpl '%E s:.*\.::'
|
||||
|
||||
B<map> cannot handle bundled options: B<map -vp 0 echo this fails>
|
||||
|
||||
B<map> does not have an argument separator on the command line, but
|
||||
uses the first argument as command. This makes quoting harder which again
|
||||
may affect readability. Compare:
|
||||
|
||||
map -p 2 perl\\\ -ne\\\ \\\'/^\\\\S+\\\\s+\\\\S+\\\$/\\\ and\\\ print\\\ \\\$ARGV,\\\"\\\\n\\\"\\\' *
|
||||
map -p 2 'perl -ne '"'"'/^\S+\s+\S+$/ and print $ARGV,"\n"'"'" *
|
||||
|
||||
parallel -q perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ::: *
|
||||
|
||||
|
|
@ -680,16 +682,11 @@ context replace:
|
|||
|
||||
parallel --xargs echo 'BEGIN{'{}'}END' ::: 1 2 3
|
||||
|
||||
B<map> does not set exit value according to whether one of the jobs
|
||||
failed:
|
||||
|
||||
parallel false ::: 1 || echo Job failed
|
||||
|
||||
map false 1 || echo Never run
|
||||
map "echo 'BEGIN{'%'}END'" 1 2 3
|
||||
|
||||
B<map> requires Perl v5.10.0 making it harder to use on old systems.
|
||||
|
||||
B<map> has no way of using % in the command (GNU Parallel has -I to
|
||||
B<map> has no way of using % in the command (GNU B<parallel> has -I to
|
||||
specify another replacement string than B<{}>).
|
||||
|
||||
By design B<map> is option incompatible with B<xargs>, it does not
|
||||
|
|
@ -710,18 +707,21 @@ B<ladon> only works on files and the only way to specify files is
|
|||
using a quoted glob string (such as \*.jpg). It is not possible to
|
||||
list the files manually.
|
||||
|
||||
As replacement strings it uses FULLPATH DIRNAME BASENAME EXT RELDIR RELPATH
|
||||
As replacement strings it uses FULLPATH DIRNAME BASENAME EXT RELDIR
|
||||
RELPATH
|
||||
|
||||
These can be simulated using GNU B<parallel> by putting this in B<~/.parallel/config>:
|
||||
These can be simulated using GNU B<parallel> by putting this in
|
||||
B<~/.parallel/config>:
|
||||
|
||||
--rpl 'FULLPATH $_=::shell_quote($_);chomp($_=qx{readlink -f $_});'
|
||||
--rpl 'DIRNAME $_=::shell_quote(::dirname($_));chomp($_=qx{readlink -f $_});'
|
||||
--rpl 'FULLPATH $_=Q($_);chomp($_=qx{readlink -f $_});'
|
||||
--rpl 'DIRNAME $_=Q(::dirname($_));chomp($_=qx{readlink -f $_});'
|
||||
--rpl 'BASENAME s:.*/::;s:\.[^/.]+$::;'
|
||||
--rpl 'EXT s:.*\.::'
|
||||
--rpl 'RELDIR $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;$_=::dirname($_);'
|
||||
--rpl 'RELPATH $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;'
|
||||
--rpl 'RELDIR $_=Q($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;$_=::dirname($_);'
|
||||
--rpl 'RELPATH $_=Q($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;'
|
||||
|
||||
B<ladon> deals badly with filenames containing " and newline, and it fails for output larger than 200k:
|
||||
B<ladon> deals badly with filenames containing " and newline, and it
|
||||
fails for output larger than 200k:
|
||||
|
||||
ladon '*' -- seq 36000 | wc
|
||||
|
||||
|
|
@ -755,17 +755,23 @@ B<jobflow> can run multiple jobs in parallel.
|
|||
|
||||
Just like B<xargs> output from B<jobflow> jobs running in parallel mix
|
||||
together by default. B<jobflow> can buffer into files (placed in
|
||||
/run/shm), but these are not cleaned up - not even if B<jobflow> dies
|
||||
unexpectedly. If the total output is big (in the order of RAM+swap)
|
||||
it can cause the system to run out of memory.
|
||||
/run/shm), but these are not cleaned up if B<jobflow> dies
|
||||
unexpectedly (e.g. by Ctrl-C). If the total output is big (in the
|
||||
order of RAM+swap) it can cause the system to slow to a crawl and
|
||||
eventually run out of memory.
|
||||
|
||||
B<jobflow> gives no error if the command is unknown, and like B<xargs>
|
||||
redirection requires wrapping with B<bash -c>.
|
||||
redirection and composed commands require wrapping with B<bash -c>.
|
||||
|
||||
Input lines can at most be 4096 bytes. You can at most have 16 {}'s in
|
||||
the command template. More than that either crashes the program or
|
||||
simple does not execute the command.
|
||||
|
||||
B<jobflow> has no equivalent for B<--pipe>, or B<--sshlogin>.
|
||||
|
||||
B<jobflow> makes it possible to set resource limits on the running
|
||||
jobs. This can be emulated by GNU B<parallel> using B<bash>'s B<ulimit>:
|
||||
|
||||
|
||||
jobflow -limits=mem=100M,cpu=3,fsize=20M,nofiles=300 myjob
|
||||
|
||||
parallel 'ulimit -v 102400 -t 3 -f 204800 -n 300 myjob'
|
||||
|
|
@ -832,7 +838,7 @@ setting SSHPASS and by using B<--ssh "sshpass ssh">.
|
|||
|
||||
To make the emulation easier, make a simple alias:
|
||||
|
||||
alias par_emul="parallel -j0 --ssh 'sshpass ssh' --nonall --tag --linebuffer"
|
||||
alias par_emul="parallel -j0 --ssh 'sshpass ssh' --nonall --tag --lb"
|
||||
|
||||
If you want to supply a password run:
|
||||
|
||||
|
|
@ -850,8 +856,10 @@ If the above is set up you can then do:
|
|||
orgalorg -o frontend1 -o frontend2 -p -C top -bid 1
|
||||
par_emul -S frontend1 -S frontend2 top -bid 1
|
||||
|
||||
orgalorg -o frontend1 -o frontend2 -p -er /tmp -n 'md5sum /tmp/bigfile' -S bigfile
|
||||
par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp md5sum /tmp/bigfile
|
||||
orgalorg -o frontend1 -o frontend2 -p -er /tmp -n \
|
||||
'md5sum /tmp/bigfile' -S bigfile
|
||||
par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp \
|
||||
md5sum /tmp/bigfile
|
||||
|
||||
B<orgalorg> has a progress indicator for the transferring of a
|
||||
file. GNU B<parallel> does not.
|
||||
|
|
@ -865,8 +873,6 @@ Rust parallel focuses on speed. It is almost as fast as B<xargs>. It
|
|||
implements a few features from GNU B<parallel>, but lacks many
|
||||
functions. All these fail:
|
||||
|
||||
# Show what would be executed
|
||||
parallel --dry-run echo ::: a
|
||||
# Read arguments from file
|
||||
parallel -a file echo
|
||||
# Changing the delimiter
|
||||
|
|
@ -874,8 +880,6 @@ functions. All these fail:
|
|||
|
||||
These do something different from GNU B<parallel>
|
||||
|
||||
# Read more arguments at a time -n
|
||||
parallel -n 2 echo ::: 1 a 2 b
|
||||
# -q to protect quoted $ and space
|
||||
parallel -q perl -e '$a=shift; print "$a"x10000000' ::: a b c
|
||||
# Generation of combination of inputs
|
||||
|
|
@ -885,7 +889,7 @@ These do something different from GNU B<parallel>
|
|||
# --pipe
|
||||
seq 100000 | parallel --pipe wc
|
||||
# linked arguments
|
||||
parallel echo ::: S M L :::+ small medium large ::: R G B :::+ red green blue
|
||||
parallel echo ::: S M L :::+ sml med lrg ::: R G B :::+ red grn blu
|
||||
# Run different shell dialects
|
||||
zsh -c 'parallel echo \={} ::: zsh && true'
|
||||
csh -c 'parallel echo \$\{\} ::: shell && true'
|
||||
|
|
@ -903,6 +907,14 @@ will fail. A malicious user can setup the right permissions and
|
|||
symlink the output file to one of the user's files and next time the
|
||||
user uses Rust parallel it will overwrite this file.
|
||||
|
||||
attacker$ mkdir /tmp/parallel
|
||||
attacker$ chmod a+rwX /tmp/parallel
|
||||
# Symlink to the file the attacker wants to zero out
|
||||
attacker$ ln -s ~victim/.important-file /tmp/parallel/stderr_1
|
||||
victim$ seq 1000 | parallel echo
|
||||
# This file is now overwritten with stderr from 'echo'
|
||||
victim$ cat ~victim/.important-file
|
||||
|
||||
If /tmp/parallel runs full during the run, Rust parallel does not
|
||||
report this, but finishes with success - thereby risking data loss.
|
||||
|
||||
|
|
@ -1286,7 +1298,8 @@ E.g.
|
|||
|
||||
With GNU B<parallel> this can be emulated by:
|
||||
|
||||
echo foosuffix | parallel --plus 'p={%suffix}; echo ${p}_new_suffix'
|
||||
echo foosuffix |
|
||||
parallel --plus 'p={%suffix}; echo ${p}_new_suffix'
|
||||
|
||||
Opposite B<rush> GNU B<parallel> works fine if the input contains
|
||||
double space, ' and ":
|
||||
|
|
@ -1501,9 +1514,9 @@ https://github.com/soveran/map
|
|||
|
||||
=head2 Todo
|
||||
|
||||
Url for map, spread
|
||||
Url for spread
|
||||
|
||||
machma. Requires Go >= 1.7.
|
||||
https://github.com/fd0/machma Requires Go >= 1.7.
|
||||
|
||||
https://github.com/k-bx/par requires Haskell to work. This limits the
|
||||
number of platforms this can work on.
|
||||
|
|
@ -1518,6 +1531,8 @@ https://github.com/Julian/Verge
|
|||
|
||||
https://github.com/amattn/paral
|
||||
|
||||
https://github.com/mmstick/concurr
|
||||
|
||||
pyargs
|
||||
|
||||
|
||||
|
|
@ -1559,7 +1574,8 @@ This test stresses whether output mixes.
|
|||
chmod +x mycommand
|
||||
|
||||
# Run 30 jobs in parallel
|
||||
seq 30 | $paralleltool ./mycommand > >(tr -s abcdef) 2> >(tr -s abcdef >&2)
|
||||
seq 30 |
|
||||
$paralleltool ./mycommand > >(tr -s abcdef) 2> >(tr -s abcdef >&2)
|
||||
|
||||
# 'a c e' and 'b d f' should always stay together
|
||||
# and there should only be a single line per job
|
||||
|
|
@ -1673,6 +1689,18 @@ Some tools can only insert the argument once.
|
|||
echo bar | $paralleltool echo {} foo {}
|
||||
|
||||
|
||||
=head2 INPUTSIZE: Length of input should not be limited
|
||||
|
||||
Some tools limit the length of the input lines artificially with no good
|
||||
reason. GNU B<parallel> does not:
|
||||
|
||||
perl -e 'print "foo."."x"x100_000_000' | parallel echo {.}
|
||||
|
||||
GNU B<parallel> limits the command to run to 128 KB due to execve(1):
|
||||
|
||||
perl -e 'print "x"x131_000' | parallel echo {} | wc
|
||||
|
||||
|
||||
=head2 NUMWORDS: Speed depends on number of words
|
||||
|
||||
Some tools become very slow if output lines have many words.
|
||||
|
|
@ -1705,10 +1733,10 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
|||
|
||||
Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk
|
||||
Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2010,2011,2012,2013,2014,2015,2016,2017,2018 Ole Tange,
|
||||
http://ole.tange.dk and Free Software Foundation, Inc.
|
||||
Copyright (C) 2010-2018 Ole Tange, http://ole.tange.dk and Free
|
||||
Software Foundation, Inc.
|
||||
|
||||
Parts of the manual concerning B<xargs> compatibility is inspired by
|
||||
the manual of B<xargs> from GNU findutils 4.4.2.
|
||||
|
|
@ -1716,8 +1744,7 @@ the manual of B<xargs> from GNU findutils 4.4.2.
|
|||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018
|
||||
Free Software Foundation, Inc.
|
||||
Copyright (C) 2007-2018 Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ testsuite: 3
|
|||
make stopvm
|
||||
|
||||
1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
|
||||
TRIES=1 time bash Start.sh '' 'mem|polarhome|tutorial' || true
|
||||
TRIES=1 time bash Start.sh '' '100s|mem|polarhome|tutorial' || true
|
||||
touch ~/.parallel/will-cite
|
||||
make stopvm
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
echo '### Test niceload -q'
|
||||
niceload -q perl -e '$a = "works";$b="This $a\n"; print($b);'
|
||||
echo
|
||||
echo
|
||||
|
||||
freepl >/dev/null
|
||||
freepl >/dev/null &
|
||||
|
|
@ -21,7 +21,7 @@ freepl >/dev/null &
|
|||
|
||||
# niceload -q -l 5 perl -e '$a=join"",<>; while(1){push @a,$a}' &
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### --rm and --runmem'
|
||||
niceload -H --rm 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm
|
||||
niceload -H --runmem 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ int() {
|
|||
}
|
||||
export -f int
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
# The seq 30000000 should take > 1 cpu sec to run.
|
||||
echo '### --soft -f and test if child is actually suspended and thus takes longer'
|
||||
niceload --soft -f 0.5 'seq 30000000 | nice wc;echo This should finish last' &
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ par_load() {
|
|||
echo '### Test --load (must give 1=true)'
|
||||
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 &
|
||||
parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 |
|
||||
parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}'
|
||||
parallel -k --colsep '\t' --header : echo '{=4 $_=$_>5=}'
|
||||
}
|
||||
|
||||
par_env_parallel() {
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ export -f stdsort
|
|||
# Test amount of parallelization
|
||||
# parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5}
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj13 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj13 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test bug #45619: "--halt" erroneous error exit code (should give 0)';
|
||||
seq 10 | parallel --halt now,fail=1 true;
|
||||
echo $?
|
||||
|
|
@ -837,6 +837,17 @@ par_perlexpr_with_newline() {
|
|||
rm *"Dad's \"famous\" 1' pizza"
|
||||
}
|
||||
|
||||
par_empty_command() {
|
||||
echo 'bug #54647: parset ignores empty lines'
|
||||
# really due to this. Should give an empty line due to -v:
|
||||
parallel -v :::: <(echo)
|
||||
. `which env_parallel.bash`
|
||||
parset a,b,c :::: <(echo echo A; echo; echo echo C)
|
||||
echo Empty: $b
|
||||
parset a,b,c :::: <(echo echo A; echo echo B; echo echo C)
|
||||
echo B: $b
|
||||
}
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
compgen -A function | grep par_ | sort |
|
||||
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ mkdir tmp
|
|||
cd tmp
|
||||
unset run_test
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj300% -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj300% -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test --load remote'
|
||||
ssh parallel@lo 'seq 10 | parallel --nice 19 --timeout 15 -j0 -qN0 perl -e while\(1\)\{\ \}' &
|
||||
sleep 1;
|
||||
|
|
@ -20,17 +20,6 @@ echo '### Stop if all hosts are filtered and there are no hosts left to run on'
|
|||
echo '### Can csh propagate a variable containing \n';
|
||||
export A=$(seq 3); parallel -S csh@lo --env A bash -c \''echo "$A"'\' ::: dummy
|
||||
|
||||
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
|
||||
echo '** test_zsh'
|
||||
FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** test_zsh_filter'
|
||||
FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** test_csh'
|
||||
FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** test_csh_filter'
|
||||
FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** bug #41805 done'
|
||||
|
||||
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
|
||||
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
|
||||
# SSH only allowed to localhost/lo
|
||||
cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### trailing space in sshlogin'
|
||||
echo 'sshlogin trailing space' | parallel --sshlogin "ssh -l parallel localhost " echo
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ unset run_once
|
|||
|
||||
# SSH only allowed to localhost/lo
|
||||
# --retries if ssh dies
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### --env _'
|
||||
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
|
||||
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
|
||||
|
|
|
|||
|
|
@ -1833,20 +1833,20 @@ par_mksh_environment_too_big() {
|
|||
echo 'bug #50815: env_parallel should warn if the environment is too big'
|
||||
. `which env_parallel.mksh`;
|
||||
|
||||
bigvar="$(perl -e 'print "x"x119000')"
|
||||
bigvar="$(perl -e 'print "x"x110000')"
|
||||
env_parallel echo ::: OK_bigvar
|
||||
env_parallel -S lo echo ::: OK_bigvar_remote
|
||||
|
||||
bigvar="$(perl -e 'print "\""x119000')"
|
||||
bigvar="$(perl -e 'print "\""x110000')"
|
||||
env_parallel echo ::: OK_bigvar_quote
|
||||
env_parallel -S lo echo ::: OK_bigvar_quote_remote
|
||||
|
||||
bigvar=u
|
||||
eval 'bigfunc() { a="'"$(perl -e 'print "x"x119000')"'"; };'
|
||||
eval 'bigfunc() { a="'"$(perl -e 'print "x"x110000')"'"; };'
|
||||
env_parallel echo ::: OK_bigfunc
|
||||
env_parallel -S lo echo ::: OK_bigfunc_remote
|
||||
|
||||
eval 'bigfunc() { a="'"$(perl -e 'print "\""x119000')"'"; };'
|
||||
eval 'bigfunc() { a="'"$(perl -e 'print "\""x110000')"'"; };'
|
||||
env_parallel echo ::: OK_bigfunc_quote
|
||||
env_parallel -S lo echo ::: OK_bigfunc_quote_remote
|
||||
bigfunc() { true; }
|
||||
|
|
|
|||
|
|
@ -131,6 +131,28 @@ _EOF
|
|||
ssh zsh@lo "$myscript"
|
||||
}
|
||||
|
||||
par_propagate_env() {
|
||||
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
|
||||
echo '** test_zsh'
|
||||
FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** test_zsh_filter'
|
||||
FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** test_csh'
|
||||
FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** test_csh_filter'
|
||||
FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
echo '** bug #41805 done'
|
||||
}
|
||||
|
||||
par_env_parallel_big_env() {
|
||||
echo '### bug #54128: command too long when exporting big env'
|
||||
. `which env_parallel.bash`
|
||||
a=`rand | head -c 75000`
|
||||
env_parallel -Slo echo should not ::: fail 2>&1
|
||||
a=`rand | head -c 76000`
|
||||
env_parallel -Slo echo should not ::: fail 2>/dev/null || echo OK
|
||||
}
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
#compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1'
|
||||
#compgen -A function | grep par_ | sort |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo "bug #43654: --bar with command not using {} - only last output line "
|
||||
COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
|
||||
seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
|
||||
echo echo 1-4 + 1-4
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ stdsort() {
|
|||
}
|
||||
export -f stdsort
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj100 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj100 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### -0 -n3 echo < files0.xi'
|
||||
stdout xargs -0 -n3 echo < files0.xi
|
||||
stdout parallel -k -0 -n3 echo < files0.xi
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ median() { perl -e '@a=sort {$a<=>$b} <>;print $a[$#a/2]';}
|
|||
export -f median
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### bug #41565: Print happens in blocks - not after each job complete'
|
||||
echo 'The timing here is important: a full second between each'
|
||||
perl -e 'for(1..30){print("$_\n");`sleep 1`}' | parallel -j3 'echo {#}' | timestamp -dd | perl -pe '$_=int($_+0.3)."\n"' | median
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name'
|
||||
echo "echo echo from ./local_test_profile" > local_test_profile;
|
||||
parallel --profile local_test_profile echo ::: 1;
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ ls | parallel -kv rm -- {.}/abc-{.}-{} 2>&1
|
|||
#test05.sh:find . -type d -print0 | perl -0 -pe 's:^./::' | parallel -0 -v rmdir -- {} 2>&1 \
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 --joblog /tmp/jl-`basename $0`
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 --joblog /tmp/jl-`basename $0` -r
|
||||
echo '### Test compress'
|
||||
seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10'
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ forceload () {
|
|||
# Force load avg > number of cpu cores
|
||||
forceload $(parallel --number-of-cores)
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first job"
|
||||
/usr/bin/time -f %e parallel --load 100% true ::: a 2>&1 |
|
||||
perl -ne '$_ > 1 and print "More than 1 secs wall clock: OK\n"'
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test of xargs -m command lines > 130k';
|
||||
seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$;
|
||||
wait;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ cp -a input-files/testdir2 tmp
|
|||
NICEPAR="nice nice parallel"
|
||||
export NICEPAR
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### bug #42329: --line-buffer gives wrong output';
|
||||
$NICEPAR --line-buffer --tag seq ::: 10000000 | wc -c;
|
||||
$NICEPAR --line-buffer seq ::: 10000000 | wc -c
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### -L -n with pipe'
|
||||
seq 14 | parallel --pipe -k -L 3 -n 2 'cat;echo 6 Ln line record'
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ echo '### Test --pipe'
|
|||
seq 1 1000000 >/tmp/parallel-seq
|
||||
shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj2 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test 200M records with too small block';
|
||||
(
|
||||
echo start;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ export -f par_tmux
|
|||
# echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground'
|
||||
# stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null | perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"'
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 100 --retries 1 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 100 --retries 1 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
|
||||
echo '### tmux-1.9'
|
||||
seq 0000 10 1000 | PARALLEL_TMUX=tmux-1.9 par_tmux
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ export XAP
|
|||
NICEPAR="nice nice parallel"
|
||||
export NICEPAR
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo 'bug #41412: --timeout + --delay causes deadlock';
|
||||
seq 10 | parallel -j10 --timeout 1 --delay 0.3 echo;
|
||||
parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3;
|
||||
|
|
|
|||
|
|
@ -8,14 +8,17 @@ env_parallel --session
|
|||
P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax ultrix minix irix hurd beaglebone cubieboard2"
|
||||
P="$P_ALL"
|
||||
|
||||
# 2018-04-22 MAXTIME=20
|
||||
MAXTIME=25
|
||||
RETRIES=4
|
||||
MAXPROC=150
|
||||
# tru64 takes 22s to run 4 parallels
|
||||
MAXTIME=50
|
||||
RETRIES=2
|
||||
MAXPROC=${maxproc:-11}
|
||||
MAXINNERPROC=${maxinnerproc:-3}
|
||||
|
||||
# select a running master (debian or ubuntu)
|
||||
MASTER=$(parallel -j0 --halt now,success=1 ssh {} echo {} \
|
||||
::: {suse,ubuntu,debian}.polarhome.com)
|
||||
export PARALLEL_SSH="ssh -oLogLevel=quiet"
|
||||
|
||||
# select a running master (suse, ubuntu, or debian)
|
||||
MASTER=$(parallel -j0 --delay 0.1 --halt now,success=1 $PARALLEL_SSH {} echo {} \
|
||||
::: {ubuntu,suse,debian}.polarhome.com)
|
||||
|
||||
parallel -j0 --delay 0.1 --retries $RETRIES \
|
||||
rsync -a /usr/local/bin/{parallel,env_parallel,env_parallel.*[^~],parcat} \
|
||||
|
|
@ -29,26 +32,37 @@ doit() {
|
|||
export RETRIES
|
||||
export MAXPROC
|
||||
export RET_TIME_K="-k --retries $RETRIES --timeout $MAXTIME"
|
||||
|
||||
echo MAXTIME=$MAXTIME RETRIES=$RETRIES MAXPROC=$MAXPROC
|
||||
|
||||
echo MAXTIME=$MAXTIME RETRIES=$RETRIES MAXPROC=$MAXPROC MAXINNERPROC=$MAXINNERPROC
|
||||
|
||||
echo '### Filter out working servers'
|
||||
POLAR="` bin/parallel -j0 $RET_TIME_K $PARALLEL_SSH {} echo {} ::: $P`"
|
||||
# syllable often gives false positive
|
||||
parallel --timeout $MAXTIME -j10 ssh syllable true ::: {1..10} 2>/dev/null >/dev/null &
|
||||
POLAR_ALL="`bin/parallel -j0 -k --timeout 10 echo {} ::: $P`"
|
||||
POLAR="`bin/parallel -j0 -k --timeout 10 $PARALLEL_SSH {} echo {} ::: $P`"
|
||||
diff <(echo "$POLAR_ALL") <(echo "$POLAR")
|
||||
S_POLAR=`bin/parallel -j0 $RET_TIME_K echo -S 1/{} ::: $POLAR`
|
||||
|
||||
#" -S '1/sshminix minix'"
|
||||
|
||||
sshwithpass() {
|
||||
# Minix requires sshpass. The other servers will use ssh-keys
|
||||
sshpass -f ~/.ssh/minix.password ssh -oLogLevel=quiet "$@"
|
||||
}
|
||||
export -f sshwithpass
|
||||
|
||||
copy() {
|
||||
# scp, but atomic (avoid half files if disconnected)
|
||||
host=$1
|
||||
src="$2"
|
||||
dst="$3"
|
||||
cat "$src" |
|
||||
stdout ssh -oLogLevel=quiet $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'"
|
||||
sshwithpass $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'" 2>&1
|
||||
}
|
||||
export -f copy
|
||||
|
||||
|
||||
par_nonall() {
|
||||
parallel -j$MAXPROC $RET_TIME_K --delay 0.1 --tag \
|
||||
--nonall $S_POLAR --argsep ,:- \
|
||||
--nonall $S_POLAR -S "1/sshminix minix" --argsep ,:- \
|
||||
'source setupenv >&/dev/null || . `pwd`/setupenv;' "$@"
|
||||
}
|
||||
export -f par_nonall
|
||||
|
|
@ -56,53 +70,56 @@ doit() {
|
|||
echo '### Copy commands to servers'
|
||||
env_parallel -vj$MAXPROC $RET_TIME_K --delay 0.03 --tag copy {2} {1} {1/} \
|
||||
::: bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \
|
||||
::: $POLAR
|
||||
::: $POLAR minix
|
||||
echo Done copying
|
||||
|
||||
env_parallel -d '\n\n' -vkj$MAXINNERPROC --delay 2 <<'EOF'
|
||||
|
||||
echo
|
||||
echo '### Works on ...'
|
||||
echo
|
||||
par_nonall parallel echo Works on {} ::: '`hostname`' 2>&1
|
||||
|
||||
# Test empty command
|
||||
test_empty_cmd() {
|
||||
echo '### Test if empty command in process list causes problems'
|
||||
perl -e '$0=" ";sleep 10' &
|
||||
parallel echo ::: OK_with_empty_cmd
|
||||
}
|
||||
export -f test_empty_cmd
|
||||
PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1
|
||||
echo
|
||||
echo '### --number-of-cores/--number-of-cpus should work with no error'
|
||||
echo
|
||||
par_nonall 'parallel --number-of-sockets; parallel --number-of-cores' 2>&1
|
||||
par_nonall 'parallel --number-of-threads; parallel --number-of-cpus' 2>&1
|
||||
|
||||
echo
|
||||
echo '### Fails if tmpdir is R/O'
|
||||
echo
|
||||
par_nonall "stdout parallel --tmpdir / echo ::: test read-only tmp |" \
|
||||
"perl -pe '\$exit += s:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; \$exit += s/[0-9][0-9][0-9][0-9]/0000/gi; END { exit not \$exit }' &&" \
|
||||
"echo OK readonly tmp" 2>&1
|
||||
par_nonall "stdout parallel --tmpdir / echo ::: test read-only tmp |
|
||||
perl -pe '\$exit += s:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; \$exit += s/[0-9][0-9][0-9][0-9]/0000/gi; END { exit not \$exit }' &&
|
||||
echo OK readonly tmp" 2>&1
|
||||
|
||||
echo
|
||||
echo '### --number-of-cores/--number-of-cpus should work with no error'
|
||||
echo
|
||||
par_nonall parallel --number-of-sockets 2>&1
|
||||
par_nonall parallel --number-of-cores 2>&1
|
||||
par_nonall parallel --number-of-threads 2>&1
|
||||
par_nonall parallel --number-of-cpus 2>&1
|
||||
|
||||
echo
|
||||
echo '### Does exporting a bash function kill parallel'
|
||||
echo '### Does exporting a bash function make parallel fail?'
|
||||
echo 'If login shell is not bash compatible it fails'
|
||||
echo
|
||||
# http://zmwangx.github.io/blog/2015-11-25-bash-function-exporting-fiasco.html
|
||||
par_nonall 'func() { cat <(echo bash only A); };export -f func; bin/parallel func ::: 1' 2>&1
|
||||
par_nonall 'echo test funcA
|
||||
funcA() {
|
||||
cat <(echo bash only A)
|
||||
}
|
||||
export -f funcA;
|
||||
bin/parallel funcA ::: 1' 2>&1
|
||||
|
||||
echo
|
||||
echo '### Does PARALLEL_SHELL help exporting a bash function not kill parallel'
|
||||
echo '### Does PARALLEL_SHELL help exporting a bash function'
|
||||
echo 'If login shell is not bash compatible it should work'
|
||||
echo
|
||||
(
|
||||
mkdir -p tmp/bin;
|
||||
cp /bin/bash tmp/bin
|
||||
cd tmp
|
||||
PARALLEL_SHELL=bin/bash par_nonall 'func() { cat <(echo bash only B); };export -f func; bin/parallel func ::: 1'
|
||||
)
|
||||
mkdir -p tmp/bin
|
||||
cp /bin/bash tmp/bin
|
||||
cd tmp
|
||||
export PARALLEL_SHELL=bin/bash
|
||||
par_nonall 'echo test funcB
|
||||
funcB() {
|
||||
cat <(echo bash only B)
|
||||
}
|
||||
export -f funcB
|
||||
export PARALLEL_SHELL=bin/bash
|
||||
bin/parallel funcB ::: 1' 2>&1
|
||||
|
||||
echo
|
||||
echo '### env_parallel echo :::: <(echo OK)'
|
||||
|
|
@ -110,10 +127,22 @@ doit() {
|
|||
echo
|
||||
par_nonall 'bin/env_parallel --install && echo install-OK' 2>&1
|
||||
par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1
|
||||
par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' |
|
||||
# csh on NetBSD does not support process substitution
|
||||
# csh on NetBSD does not support process substitution
|
||||
par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 |
|
||||
grep -v ': /tmp/.*: No such file or directory'
|
||||
|
||||
# Test empty command name in process list
|
||||
test_empty_cmd() {
|
||||
echo '### Test if empty command name in process list causes problems'
|
||||
perl -e '$0=" ";sleep 1000' &
|
||||
pid=$!
|
||||
parallel echo ::: OK_with_empty_cmd
|
||||
kill $pid
|
||||
}
|
||||
export -f test_empty_cmd
|
||||
export PARALLEL_SHELL=bin/bash
|
||||
PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1
|
||||
|
||||
echo
|
||||
echo '### parset arr seq ::: 2 3 4'
|
||||
echo '(bash ksh mksh zsh only)'
|
||||
|
|
@ -129,6 +158,8 @@ doit() {
|
|||
par_nonall 'parset var1,var2,var3 seq ::: 2 3 4; echo $var1,$var2,$var3' 2>&1
|
||||
echo '### env_parset var1,var2,var3 seq ::: 2 3 4'
|
||||
par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
env_parallel -u -S$MASTER doit ::: 1
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2
|
|||
chmod 755 /tmp/myssh1 /tmp/myssh2
|
||||
seq 1 100 | parallel --sshdelay 0.03 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj3 -k -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj3 -k -L1 -r
|
||||
echo '### test --timeout --retries'
|
||||
parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 $SSHLOGIN1 $SSHLOGIN2 $SSHLOGIN3
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test mutex. This should not mix output';
|
||||
parallel --semaphore --id mutex -u seq 1 10 '|' pv -qL 20;
|
||||
parallel --semaphore --id mutex -u seq 11 20 '|' pv -qL 100;
|
||||
|
|
@ -55,4 +55,4 @@ echo '### Test --st +1/-1'
|
|||
stdout sem --id st --wait
|
||||
|
||||
|
||||
EOF
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test of --eta'
|
||||
seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ SERVER1=parallel-server3
|
|||
SERVER2=parallel-server1
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin'
|
||||
seq 1 150 | stdout nice parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null"
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ SSHLOGIN1=parallel@$SERVER1
|
|||
SSHLOGIN2=parallel@$SERVER2
|
||||
|
||||
# -L1 will join lines ending in ' '
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r
|
||||
echo '### Test --onall';
|
||||
parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
SERVER1=parallel-server1
|
||||
SERVER2=parallel-server2
|
||||
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 | perl -pe 's/(PARALLEL_PID....)\d+/$1XXXXX/g'
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r | perl -pe 's/(PARALLEL_PID....)\d+/$1XXXXX/g'
|
||||
echo '### Test --return of weirdly named file'
|
||||
stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b' |
|
||||
perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'; rm 'aa<${#}" b'
|
||||
|
|
|
|||
|
|
@ -389,7 +389,7 @@ d
|
|||
(echo a_b' ';echo c;echo d) | xargs -L1 echo
|
||||
a_b c
|
||||
d
|
||||
echo '### xargs -L2 echo'
|
||||
echo '### xargs -L2 echo'
|
||||
### xargs -L2 echo
|
||||
(echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo
|
||||
a_b c d
|
||||
|
|
@ -400,7 +400,7 @@ e
|
|||
(echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo
|
||||
a_b c d
|
||||
e
|
||||
echo '### xargs -l echo'
|
||||
echo '### xargs -l echo'
|
||||
### xargs -l echo
|
||||
(echo a_b' ';echo c;echo d;echo e) | parallel -l -km echo # This behaves wrong
|
||||
a_b c
|
||||
|
|
@ -414,7 +414,7 @@ e
|
|||
a_b c
|
||||
d
|
||||
e
|
||||
echo '### xargs -l2 echo'
|
||||
echo '### xargs -l2 echo'
|
||||
### xargs -l2 echo
|
||||
(echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo
|
||||
a_b c d
|
||||
|
|
@ -425,7 +425,7 @@ e
|
|||
(echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo
|
||||
a_b c d
|
||||
e
|
||||
echo '### xargs -l1 echo'
|
||||
echo '### xargs -l1 echo'
|
||||
### xargs -l1 echo
|
||||
(echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo
|
||||
a_b c
|
||||
|
|
@ -439,7 +439,7 @@ e
|
|||
a_b c
|
||||
d
|
||||
e
|
||||
echo '### xargs --max-lines=2 echo'
|
||||
echo '### xargs --max-lines=2 echo'
|
||||
### xargs --max-lines=2 echo
|
||||
(echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo
|
||||
a_b c d
|
||||
|
|
@ -450,7 +450,7 @@ e
|
|||
(echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo
|
||||
a_b c d
|
||||
e
|
||||
echo '### xargs --max-lines echo'
|
||||
echo '### xargs --max-lines echo'
|
||||
### xargs --max-lines echo
|
||||
(echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -km echo # This behaves wrong
|
||||
a_b c
|
||||
|
|
@ -464,7 +464,7 @@ e
|
|||
a_b c
|
||||
d
|
||||
e
|
||||
echo '### test too long args'
|
||||
echo '### test too long args'
|
||||
### test too long args
|
||||
perl -e 'print "z"x1000000' | parallel echo 2>&1
|
||||
parallel: Error: Command line too long (1000005 >= 131049) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
|
|
@ -491,7 +491,7 @@ xargs: argument line too long
|
|||
7 8
|
||||
9 10
|
||||
parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
echo '### Test -x'
|
||||
echo '### Test -x'
|
||||
### Test -x
|
||||
(seq 1 10; echo 12345; seq 12 15) | stdsort parallel -j1 -km -s 10 -x echo
|
||||
1 2
|
||||
|
|
@ -546,7 +546,7 @@ xargs: argument line too long
|
|||
5 6
|
||||
7 8
|
||||
9 10
|
||||
echo '### Test -a and --arg-file: Read input from file instead of stdin'
|
||||
echo '### Test -a and --arg-file: Read input from file instead of stdin'
|
||||
### Test -a and --arg-file: Read input from file instead of stdin
|
||||
seq 1 10 >/tmp/parallel_$$-1; parallel -k -a /tmp/parallel_$$-1 echo; rm /tmp/parallel_$$-1
|
||||
1
|
||||
|
|
@ -570,7 +570,7 @@ xargs: argument line too long
|
|||
8
|
||||
9
|
||||
10
|
||||
echo '### Test -i and --replace: Replace with argument'
|
||||
echo '### Test -i and --replace: Replace with argument'
|
||||
### Test -i and --replace: Replace with argument
|
||||
(echo a; echo END; echo b) | parallel -k -i -eEND echo repl{}ce
|
||||
replace
|
||||
|
|
@ -590,13 +590,13 @@ replace
|
|||
replace
|
||||
(echo a; echo END; echo b) | parallel -k -I^ -eEND echo repl^ce
|
||||
replace
|
||||
echo '### Test -E: Artificial end-of-file'
|
||||
echo '### Test -E: Artificial end-of-file'
|
||||
### Test -E: Artificial end-of-file
|
||||
(echo include this; echo END; echo not this) | parallel -k -E END echo
|
||||
include this
|
||||
(echo include this; echo END; echo not this) | parallel -k -EEND echo
|
||||
include this
|
||||
echo '### Test -e and --eof: Artificial end-of-file'
|
||||
echo '### Test -e and --eof: Artificial end-of-file'
|
||||
### Test -e and --eof: Artificial end-of-file
|
||||
(echo include this; echo END; echo not this) | parallel -k -e END echo
|
||||
include this
|
||||
|
|
@ -606,7 +606,7 @@ include this
|
|||
include this
|
||||
(echo include this; echo END; echo not this) | parallel -k --eof END echo
|
||||
include this
|
||||
echo '### Test -n and --max-args: Max number of args per line (only with -X and -m)'
|
||||
echo '### Test -n and --max-args: Max number of args per line (only with -X and -m)'
|
||||
### Test -n and --max-args: Max number of args per line (only with -X and -m)
|
||||
(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -m echo
|
||||
line 1
|
||||
|
|
@ -650,7 +650,7 @@ line 3
|
|||
(echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 echo
|
||||
line 1 line 1
|
||||
line 2
|
||||
echo '### Test --max-procs and -P: Number of processes'
|
||||
echo '### Test --max-procs and -P: Number of processes'
|
||||
### Test --max-procs and -P: Number of processes
|
||||
seq 1 10 | parallel -k --max-procs +0 echo max proc
|
||||
max proc 1
|
||||
|
|
@ -674,7 +674,7 @@ max proc 10
|
|||
200% proc 8
|
||||
200% proc 9
|
||||
200% proc 10
|
||||
echo '### Test --delimiter and -d: Delimiter instead of newline'
|
||||
echo '### Test --delimiter and -d: Delimiter instead of newline'
|
||||
### Test --delimiter and -d: Delimiter instead of newline
|
||||
echo '# Yes there is supposed to be an extra newline for -d N'
|
||||
# Yes there is supposed to be an extra newline for -d N
|
||||
|
|
@ -696,7 +696,7 @@ line 3
|
|||
delimiter TAB line 1
|
||||
line 2
|
||||
line 3
|
||||
echo '### Test --max-chars and -s: Max number of chars in a line'
|
||||
echo '### Test --max-chars and -s: Max number of chars in a line'
|
||||
### Test --max-chars and -s: Max number of chars in a line
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k --max-chars 25 -X echo
|
||||
line 1 line 1
|
||||
|
|
@ -704,11 +704,11 @@ line 2
|
|||
(echo line 1;echo line 1;echo line 2) | parallel -k -s 25 -X echo
|
||||
line 1 line 1
|
||||
line 2
|
||||
echo '### Test --no-run-if-empty and -r: This should give no output'
|
||||
echo '### Test --no-run-if-empty and -r: This should give no output'
|
||||
### Test --no-run-if-empty and -r: This should give no output
|
||||
echo " " | parallel -r echo
|
||||
echo " " | parallel --no-run-if-empty echo
|
||||
echo '### Test --help and -h: Help output (just check we get the same amount of lines)'
|
||||
echo '### Test --help and -h: Help output (just check we get the same amount of lines)'
|
||||
### Test --help and -h: Help output (just check we get the same amount of lines)
|
||||
echo Output from -h and --help
|
||||
Output from -h and --help
|
||||
|
|
@ -716,11 +716,11 @@ Output from -h and --help
|
|||
37
|
||||
parallel --help | wc -l
|
||||
37
|
||||
echo '### Test --version: Version output (just check we get the same amount of lines)'
|
||||
echo '### Test --version: Version output (just check we get the same amount of lines)'
|
||||
### Test --version: Version output (just check we get the same amount of lines)
|
||||
parallel --version | wc -l
|
||||
11
|
||||
echo '### Test --verbose and -t'
|
||||
10
|
||||
echo '### Test --verbose and -t'
|
||||
### Test --verbose and -t
|
||||
(echo b; echo c; echo f) | parallel -k -t echo {}ar 2>&1 >/dev/null
|
||||
echo bar
|
||||
|
|
@ -730,7 +730,7 @@ echo far
|
|||
echo bar
|
||||
echo car
|
||||
echo far
|
||||
echo '### Test --show-limits'
|
||||
echo '### Test --show-limits'
|
||||
### Test --show-limits
|
||||
(echo b; echo c; echo f) | parallel -k --show-limits echo {}ar
|
||||
Maximal size of command: 131049
|
||||
|
|
@ -750,11 +750,11 @@ Execution of will continue now, and it will try to read its input
|
|||
and run commands; if this is not what you wanted to happen, please
|
||||
press CTRL-D or CTRL-C
|
||||
bar car far
|
||||
echo '### Test empty line as input'
|
||||
echo '### Test empty line as input'
|
||||
### Test empty line as input
|
||||
echo | parallel echo empty input line
|
||||
empty input line
|
||||
echo '### Tests if (cat | sh) works'
|
||||
echo '### Tests if (cat | sh) works'
|
||||
### Tests if (cat | sh) works
|
||||
perl -e 'for(1..25) {print "echo a $_; echo b $_\n"}' | parallel 2>&1 | sort
|
||||
a 1
|
||||
|
|
@ -807,7 +807,7 @@ b 6
|
|||
b 7
|
||||
b 8
|
||||
b 9
|
||||
echo '### Test if xargs-mode works'
|
||||
echo '### Test if xargs-mode works'
|
||||
### Test if xargs-mode works
|
||||
perl -e 'for(1..25) {print "a $_\nb $_\n"}' | parallel echo 2>&1 | sort
|
||||
a 1
|
||||
|
|
@ -860,12 +860,12 @@ b 6
|
|||
b 7
|
||||
b 8
|
||||
b 9
|
||||
echo '### Test -q'
|
||||
echo '### Test -q'
|
||||
### Test -q
|
||||
parallel -kq perl -e '$ARGV[0]=~/^\S+\s+\S+$/ and print $ARGV[0],"\n"' ::: "a b" c "d e f" g "h i"
|
||||
a b
|
||||
h i
|
||||
echo '### Test -q {#}'
|
||||
echo '### Test -q {#}'
|
||||
### Test -q {#}
|
||||
parallel -kq echo {#} ::: a b
|
||||
1
|
||||
|
|
@ -876,7 +876,7 @@ h i
|
|||
parallel -kq echo {\\#} ::: a b
|
||||
{\#} a
|
||||
{\#} b
|
||||
echo '### Test long commands do not take up all memory'
|
||||
echo '### Test long commands do not take up all memory'
|
||||
### Test long commands do not take up all memory
|
||||
seq 1 100 | parallel -j0 -qv perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 2>/dev/null | sort
|
||||
1
|
||||
|
|
@ -1079,7 +1079,7 @@ perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000
|
|||
perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 97
|
||||
perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 98
|
||||
perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 99
|
||||
echo '### Test 0-arguments'
|
||||
echo '### Test 0-arguments'
|
||||
### Test 0-arguments
|
||||
seq 1 2 | parallel -k -n0 echo n0
|
||||
n0
|
||||
|
|
@ -1090,34 +1090,34 @@ L0
|
|||
seq 1 2 | parallel -k -N0 echo N0
|
||||
N0
|
||||
N0
|
||||
echo '### Because of --tollef -l, then -l0 == -l1, sorry'
|
||||
echo '### Because of --tollef -l, then -l0 == -l1, sorry'
|
||||
### Because of --tollef -l, then -l0 == -l1, sorry
|
||||
seq 1 2 | parallel -k -l0 echo l0
|
||||
l0 1
|
||||
l0 2
|
||||
echo '### Test replace {}'
|
||||
echo '### Test replace {}'
|
||||
### Test replace {}
|
||||
seq 1 2 | parallel -k -N0 echo replace {} curlies
|
||||
replace curlies
|
||||
replace curlies
|
||||
echo '### Test arguments on commandline'
|
||||
echo '### Test arguments on commandline'
|
||||
### Test arguments on commandline
|
||||
parallel -k -N0 echo args on cmdline ::: 1 2
|
||||
args on cmdline
|
||||
args on cmdline
|
||||
echo '### Test --nice locally'
|
||||
echo '### Test --nice locally'
|
||||
### Test --nice locally
|
||||
parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b
|
||||
PAR=a bash -c "echo \$PAR b"
|
||||
a b
|
||||
echo '### Test distribute arguments at EOF to 2 jobslots'
|
||||
echo '### Test distribute arguments at EOF to 2 jobslots'
|
||||
### Test distribute arguments at EOF to 2 jobslots
|
||||
seq 1 92 | parallel -j2 -kX -s 100 echo
|
||||
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
|
||||
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
|
||||
66 67 68 69 70 71 72 73 74 75 76 77 78 79
|
||||
80 81 82 83 84 85 86 87 88 89 90 91 92
|
||||
echo '### Test distribute arguments at EOF to 5 jobslots'
|
||||
echo '### Test distribute arguments at EOF to 5 jobslots'
|
||||
### Test distribute arguments at EOF to 5 jobslots
|
||||
seq 1 92 | parallel -j5 -kX -s 100 echo
|
||||
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
|
||||
|
|
@ -1127,7 +1127,7 @@ a b
|
|||
78 79 80 81 82 83
|
||||
84 85 86 87 88 89
|
||||
90 91 92
|
||||
echo '### Test distribute arguments at EOF to infinity jobslots'
|
||||
echo '### Test distribute arguments at EOF to infinity jobslots'
|
||||
### Test distribute arguments at EOF to infinity jobslots
|
||||
seq 1 92 | parallel -j0 -kX -s 100 echo 2>/dev/null
|
||||
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
|
||||
|
|
@ -1159,26 +1159,26 @@ a b
|
|||
90
|
||||
91
|
||||
92
|
||||
echo '### Test -N is not broken by distribution - single line'
|
||||
echo '### Test -N is not broken by distribution - single line'
|
||||
### Test -N is not broken by distribution - single line
|
||||
seq 9 | parallel -N 10 echo
|
||||
1 2 3 4 5 6 7 8 9
|
||||
echo '### Test -N is not broken by distribution - two lines'
|
||||
echo '### Test -N is not broken by distribution - two lines'
|
||||
### Test -N is not broken by distribution - two lines
|
||||
seq 19 | parallel -k -N 10 echo
|
||||
1 2 3 4 5 6 7 8 9 10
|
||||
11 12 13 14 15 16 17 18 19
|
||||
echo '### Test -N context replace'
|
||||
echo '### Test -N context replace'
|
||||
### Test -N context replace
|
||||
seq 19 | parallel -k -N 10 echo a{}b
|
||||
a1b a2b a3b a4b a5b a6b a7b a8b a9b a10b
|
||||
a11b a12b a13b a14b a15b a16b a17b a18b a19b
|
||||
echo '### Test -L context replace'
|
||||
echo '### Test -L context replace'
|
||||
### Test -L context replace
|
||||
seq 19 | parallel -k -L 10 echo a{}b
|
||||
a1b a2b a3b a4b a5b a6b a7b a8b a9b a10b
|
||||
a11b a12b a13b a14b a15b a16b a17b a18b a19b
|
||||
echo '**'
|
||||
echo '**'
|
||||
**
|
||||
echo '### Test {} multiple times in different commands'
|
||||
### Test {} multiple times in different commands
|
||||
|
|
@ -1313,6 +1313,10 @@ par_dryrun_append_joblog echo 4
|
|||
par_dryrun_append_joblog 4
|
||||
par_empty bug #:
|
||||
par_empty true
|
||||
par_empty_command bug #54647: parset ignores empty lines
|
||||
par_empty_command
|
||||
par_empty_command Empty:
|
||||
par_empty_command B: B
|
||||
par_empty_line ### Test bug: empty line for | sh with -k
|
||||
par_empty_line a
|
||||
par_empty_line b
|
||||
|
|
|
|||
|
|
@ -258,7 +258,7 @@ par_linebuffer_tag_slow_output b b
|
|||
par_linebuffer_tag_slow_output b
|
||||
par_long_line_remote ### Deal with long command lines on remote servers
|
||||
par_long_line_remote 2 6 30006
|
||||
par_long_line_remote 2 50 250050
|
||||
par_long_line_remote 3 50 250050
|
||||
par_maxlinelen_X_I ### Test max line length -X -I
|
||||
par_maxlinelen_X_I 3cfc69ee81b0fe7fdbe8eb059ad2da61 -
|
||||
par_maxlinelen_X_I Chars per line (817788/7): 116826
|
||||
|
|
|
|||
|
|
@ -34,30 +34,6 @@ echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); para
|
|||
1
|
||||
2
|
||||
3
|
||||
echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
|
||||
### bug #41805: Idea: propagate --env for parallel --number-of-cores
|
||||
echo '** test_zsh'
|
||||
** test_zsh
|
||||
FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
FOO=test_zsh
|
||||
HOME=~
|
||||
echo '** test_zsh_filter'
|
||||
** test_zsh_filter
|
||||
FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
FOO=test_zsh_filter
|
||||
HOME=~
|
||||
echo '** test_csh'
|
||||
** test_csh
|
||||
FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
FOO=test_csh
|
||||
HOME=~
|
||||
echo '** test_csh_filter'
|
||||
** test_csh_filter
|
||||
FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME'
|
||||
FOO=test_csh_filter
|
||||
HOME=~
|
||||
echo '** bug #41805 done'
|
||||
** bug #41805 done
|
||||
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
|
||||
### Test bug #34241: --pipe should not spawn unneeded processes
|
||||
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
|
||||
|
|
|
|||
|
|
@ -40,6 +40,20 @@ par_sh_embed your
|
|||
par_sh_embed code
|
||||
par_sh_embed here
|
||||
par_sh_embed Put your code here
|
||||
par_propagate_env ### bug #41805: Idea: propagate --env for parallel --number-of-cores
|
||||
par_propagate_env ** test_zsh
|
||||
par_propagate_env FOO=test_zsh
|
||||
par_propagate_env HOME=~
|
||||
par_propagate_env ** test_zsh_filter
|
||||
par_propagate_env FOO=test_zsh_filter
|
||||
par_propagate_env HOME=~
|
||||
par_propagate_env ** test_csh
|
||||
par_propagate_env FOO=test_csh
|
||||
par_propagate_env HOME=~
|
||||
par_propagate_env ** test_csh_filter
|
||||
par_propagate_env FOO=test_csh_filter
|
||||
par_propagate_env HOME=~
|
||||
par_propagate_env ** bug #41805 done
|
||||
par_ksh_embed --embed
|
||||
par_ksh_embed Redirect the output to a file and add your changes at the end:
|
||||
par_ksh_embed /usr/local/bin/parallel --embed > new_script
|
||||
|
|
@ -66,6 +80,9 @@ par_ksh_embed code
|
|||
par_ksh_embed here
|
||||
par_ksh_embed Put your code here
|
||||
par_fish_embed Not implemented
|
||||
par_env_parallel_big_env ### bug #54128: command too long when exporting big env
|
||||
par_env_parallel_big_env should not fail
|
||||
par_env_parallel_big_env OK
|
||||
par_csh_embed Not implemented
|
||||
par_bash_embed --embed
|
||||
par_bash_embed Redirect the output to a file and add your changes at the end:
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ a13.gifb13c13 a14.gifb14c14 a15.gifb15c15
|
|||
a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif 7.gifb1 2 3 4 5 6 7c1 2 3 4 5 6 7
|
||||
a8.gif 9.gif 10.gif 11.gif 12.gif 13.gifb8 9 10 11 12 13c8 9 10 11 12 13
|
||||
a14.gif 15.gifb14 15c14 15
|
||||
echo '### Test -I with shell meta chars'
|
||||
echo '### Test -I with shell meta chars'
|
||||
### Test -I with shell meta chars
|
||||
seq 10000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l
|
||||
2
|
||||
|
|
@ -145,7 +145,7 @@ a14.gif 15.gifb14 15c14 15
|
|||
2
|
||||
seq 10000 | parallel -j1 -I '>' -X echo 'a>b>c>' | wc -l
|
||||
2
|
||||
echo '### Test {.}'
|
||||
echo '### Test {.}'
|
||||
### Test {.}
|
||||
echo a | parallel -qX echo "'"{.}"' "
|
||||
'a'
|
||||
|
|
|
|||
|
|
@ -61,8 +61,8 @@ echo '### Check that 4 processes are really used'
|
|||
echo '### --version must have higher priority than retired options'
|
||||
### --version must have higher priority than retired options
|
||||
$NICEPAR --version -g -Y -U -W -T | tail
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018
|
||||
Ole Tange and Free Software Foundation, Inc.
|
||||
GNU parallel 20180916
|
||||
Copyright (C) 2007-2018 Ole Tange and Free Software Foundation, Inc.
|
||||
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
||||
This is free software: you are free to change and redistribute it.
|
||||
GNU parallel comes with no warranty.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue