tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: FreeBSD specific bugs.

Browse source
This commit is contained in:
Ole Tange 2015-03-29 05:28:57 +02:00
parent fa72e99c16
commit b5c67823d6
5 changed files with 72 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/parallel
View file

@ -521,7 +521,7 @@ sub index64 {
my $block_size = 2**31-1; my $block_size = 2**31-1;
my $strlen = length($$ref); my $strlen = length($$ref);
# No point in doing extra work if we don't need to. # No point in doing extra work if we don't need to.
if($strlen < $block_size) { if($strlen < $block_size or $] > 5.022) {
return index($$ref, $match, $pos); return index($$ref, $match, $pos);
} }
@ -3401,16 +3401,20 @@ sub which {
# #
# ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh # ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh
# posh rbash rush rzsh sash sh static-sh tcsh yash zsh # posh rbash rush rzsh sash sh static-sh tcsh yash zsh
my @shells = qw(ash bash csh dash fdsh fish fizsh ksh my @shells = (qw(ash bash csh dash fdsh fish fizsh ksh
ksh93 mksh pdksh posh rbash rush rzsh ksh93 mksh pdksh posh rbash rush rzsh
sash sh static-sh tcsh yash zsh -sh -csh); sash sh static-sh tcsh yash zsh -sh -csh),
'-sh (sh)' # sh on FreeBSD
);
# Can be formatted as: # Can be formatted as:
# [sh] -sh sh busybox sh # [sh] -sh sh busybox sh -sh (sh)
# /bin/sh /sbin/sh /opt/csw/sh # /bin/sh /sbin/sh /opt/csw/sh
# NOT: foo.sh sshd crash flush pdflush scosh fsflush ssh # NOT: foo.sh sshd crash flush pdflush scosh fsflush ssh
my $shell = "(?:".join("|",@shells).")"; my $shell = "(?:".join("|",map { "\Q$_\E" } @shells).")";
$regexp = '^((\[)('. $shell. ')(\])|(|\S+/|busybox )('. $shell. '))($| )'; $regexp = '^((\[)('. $shell. ')(\])|(|\S+/|busybox )('. $shell. '))($| [^(])';
%fakename = ( %fakename = (
# sh disguises itself as -sh (sh) on FreeBSD
"-sh (sh)" => ["sh"],
# csh and tcsh disguise themselves as -sh/-csh # csh and tcsh disguise themselves as -sh/-csh
"-sh" => ["csh", "tcsh"], "-sh" => ["csh", "tcsh"],
"-csh" => ["tcsh", "csh"], "-csh" => ["tcsh", "csh"],
@ -4180,7 +4184,8 @@ sub loadavg {
local $/ = undef; local $/ = undef;
my $load_out = <$load_fh>; my $load_out = <$load_fh>;
close $load_fh; close $load_fh;
my $load =()= ($load_out=~/(^\s?[DOR]\S* [^\[])/gm); # Count lines starting with D,O,R but command does not start with [
my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm);
if($load > 0) { if($load > 0) {
# load is overestimated by 1 # load is overestimated by 1
$self->{'loadavg'} = $load - 1; $self->{'loadavg'} = $load - 1;
@ -6187,13 +6192,14 @@ sub sshlogin_wrap {
if(not $monitor_parent_sshd_script) { if(not $monitor_parent_sshd_script) {
$monitor_parent_sshd_script = $monitor_parent_sshd_script =
# This will be packed in ', so only use " # This will be packed in ', so only use "
::spacefree(0,q{ ::spacefree(0,'$shell = "'.($ENV{'PARALLEL_SHELL'} || '$ENV{SHELL}').'";'.
q{
$SIG{CHLD} = sub { $done = 1; }; $SIG{CHLD} = sub { $done = 1; };
$pid = fork; $pid = fork;
unless($pid) { unless($pid) {
# Make own process group to be able to kill HUP it later # Make own process group to be able to kill HUP it later
setpgrp; setpgrp;
exec $ENV{SHELL}, "-c", ($bashfunc."@ARGV"); exec $shell, "-c", ($bashfunc."@ARGV");
die "exec: $!\n"; die "exec: $!\n";
} }
do { do {

38
src/parallel_tutorial.html
View file

@ -47,7 +47,7 @@
<li><a href="#Trimming-space">Trimming space</a></li> <li><a href="#Trimming-space">Trimming space</a></li>
</ul> </ul>
</li> </li>
<li><a href="#Controling-the-output">Controling the output</a> <li><a href="#Controlling-the-output">Controlling the output</a>
<ul> <ul>
<li><a href="#Saving-output-into-files">Saving output into files</a></li> <li><a href="#Saving-output-into-files">Saving output into files</a></li>
</ul> </ul>
@ -117,7 +117,7 @@
<p>Install the newest version with:</p> <p>Install the newest version with:</p>
<pre><code> (wget -O - pi.dk/3 || curl pi.dk/3/) | bash</code></pre> <pre><code> (wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash</code></pre>
<p>This will also install the newest version of the tutorial:</p> <p>This will also install the newest version of the tutorial:</p>
@ -165,6 +165,14 @@
<pre><code> perl -e &#39;printf &quot;f1\tf2\nA\tB\nC\tD\n&quot;&#39; &gt; tsv-file.tsv</code></pre> <pre><code> perl -e &#39;printf &quot;f1\tf2\nA\tB\nC\tD\n&quot;&#39; &gt; tsv-file.tsv</code></pre>
</dd>
<dt id="num128">num128</dt>
<dd>
<p>The file can be generated by:</p>
<pre><code> perl -e &#39;for(1..128){print &quot;$_\n&quot;}&#39; &gt; num128</code></pre>
</dd> </dd>
<dt id="num30000">num30000</dt> <dt id="num30000">num30000</dt>
<dd> <dd>
@ -386,7 +394,7 @@
<p>The command can be a script, a binary or a Bash function if the function is exported using &#39;export -f&#39;:</p> <p>The command can be a script, a binary or a Bash function if the function is exported using &#39;export -f&#39;:</p>
<pre><code> # Only works in Bash and only if $SHELL=.../bash <pre><code> # Only works in Bash
my_func() { my_func() {
echo in my_func $1 echo in my_func $1
} }
@ -465,7 +473,7 @@
<pre><code> parallel -j 2 echo {%} ::: A B C</code></pre> <pre><code> parallel -j 2 echo {%} ::: A B C</code></pre>
<p>Output (the order may be different):</p> <p>Output (the order may be different and 1 and 2 may be swapped):</p>
<pre><code> 1 <pre><code> 1
2 2
@ -527,7 +535,7 @@
<pre><code> parallel -j2 --slotreplace ,, echo ,, ::: A B C</code></pre> <pre><code> parallel -j2 --slotreplace ,, echo ,, ::: A B C</code></pre>
<p>Output (the order may be different):</p> <p>Output (the order may be different and 1 and 2 may be swapped):</p>
<pre><code> 1 <pre><code> 1
2 2
@ -664,7 +672,7 @@
<pre><code> cat num30000 | parallel --xargs echo | wc -l</code></pre> <pre><code> cat num30000 | parallel --xargs echo | wc -l</code></pre>
<p>Output:</p> <p>Output (if you run this under Bash on GNU/Linux):</p>
<pre><code> 2</code></pre> <pre><code> 2</code></pre>
@ -686,13 +694,13 @@
<pre><code> cat num30000 | parallel --jobs 4 -m echo | wc -l</code></pre> <pre><code> cat num30000 | parallel --jobs 4 -m echo | wc -l</code></pre>
<p>Output:</p> <p>Output (if you run this under Bash on GNU/Linux):</p>
<pre><code> 5</code></pre> <pre><code> 5</code></pre>
<p>This is even more visible when running 4 jobs with 10 arguments. The 10 arguments are being spread over 4 jobs:</p> <p>This is even more visible when running 4 jobs with 10 arguments. The 10 arguments are being spread over 4 jobs:</p>
<pre><code> parallel --jobs 4 -m echo ::: {1..10}</code></pre> <pre><code> parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10</code></pre>
<p>Output:</p> <p>Output:</p>
@ -839,7 +847,7 @@
<pre><code> pre-A-post</code></pre> <pre><code> pre-A-post</code></pre>
<h1 id="Controling-the-output">Controling the output</h1> <h1 id="Controlling-the-output">Controlling the output</h1>
<p>The output can prefixed with the argument:</p> <p>The output can prefixed with the argument:</p>
@ -1020,28 +1028,28 @@
<p>The number of concurrent jobs is given with --jobs/-j:</p> <p>The number of concurrent jobs is given with --jobs/-j:</p>
<pre><code> /usr/bin/time parallel -N0 -j64 sleep 1 ::: {1..128}</code></pre> <pre><code> /usr/bin/time parallel -N0 -j64 sleep 1 :::: num128</code></pre>
<p>With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run - depending on how fast your machine is.</p> <p>With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run - depending on how fast your machine is.</p>
<p>By default --jobs is the same as the number of CPU cores. So this:</p> <p>By default --jobs is the same as the number of CPU cores. So this:</p>
<pre><code> /usr/bin/time parallel -N0 sleep 1 ::: {1..128}</code></pre> <pre><code> /usr/bin/time parallel -N0 sleep 1 :::: num128</code></pre>
<p>should take twice the time of running 2 jobs per CPU core:</p> <p>should take twice the time of running 2 jobs per CPU core:</p>
<pre><code> /usr/bin/time parallel -N0 --jobs 200% sleep 1 ::: {1..128}</code></pre> <pre><code> /usr/bin/time parallel -N0 --jobs 200% sleep 1 :::: num128</code></pre>
<p>--jobs 0 will run as many jobs in parallel as possible:</p> <p>--jobs 0 will run as many jobs in parallel as possible:</p>
<pre><code> /usr/bin/time parallel -N0 --jobs 0 sleep 1 ::: {1..128}</code></pre> <pre><code> /usr/bin/time parallel -N0 --jobs 0 sleep 1 :::: num128</code></pre>
<p>which should take 1-7 seconds depending on how fast your machine is.</p> <p>which should take 1-7 seconds depending on how fast your machine is.</p>
<p>--jobs can read from a file which is re-read when a job finishes:</p> <p>--jobs can read from a file which is re-read when a job finishes:</p>
<pre><code> echo 50% &gt; my_jobs <pre><code> echo 50% &gt; my_jobs
/usr/bin/time parallel -N0 --jobs my_jobs sleep 1 ::: {1..128} &amp; /usr/bin/time parallel -N0 --jobs my_jobs sleep 1 :::: num128 &amp;
sleep 1 sleep 1
echo 0 &gt; my_jobs echo 0 &gt; my_jobs
wait</code></pre> wait</code></pre>
@ -1050,7 +1058,7 @@
<p>Instead of basing the percentage on the number of CPU cores GNU Parallel can base it on the number of CPUs:</p> <p>Instead of basing the percentage on the number of CPU cores GNU Parallel can base it on the number of CPUs:</p>
<pre><code> parallel --use-cpus-instead-of-cores -N0 sleep 1 ::: {1..128}</code></pre> <pre><code> parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num128</code></pre>
<h2 id="Interactivity">Interactivity</h2> <h2 id="Interactivity">Interactivity</h2>

34
src/parallel_tutorial.pod
View file

@ -19,7 +19,7 @@ To run this tutorial you must have the following:
Install the newest version with: Install the newest version with:
(wget -O - pi.dk/3 || curl pi.dk/3/) | bash (wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
This will also install the newest version of the tutorial: This will also install the newest version of the tutorial:
@ -58,6 +58,12 @@ The file can be generated by:
perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' > tsv-file.tsv perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' > tsv-file.tsv
=item num128
The file can be generated by:
perl -e 'for(1..128){print "$_\n"}' > num128
=item num30000 =item num30000
The file can be generated by: The file can be generated by:
@ -285,7 +291,7 @@ Output (the order may be different):
The command can be a script, a binary or a Bash function if the function is The command can be a script, a binary or a Bash function if the function is
exported using 'export -f': exported using 'export -f':
# Only works in Bash and only if $SHELL=.../bash # Only works in Bash
my_func() { my_func() {
echo in my_func $1 echo in my_func $1
} }
@ -366,7 +372,7 @@ number of jobs to run in parallel):
parallel -j 2 echo {%} ::: A B C parallel -j 2 echo {%} ::: A B C
Output (the order may be different): Output (the order may be different and 1 and 2 may be swapped):
1 1
2 2
@ -428,7 +434,7 @@ The replacement string {%} can be changed with --slotreplace:
parallel -j2 --slotreplace ,, echo ,, ::: A B C parallel -j2 --slotreplace ,, echo ,, ::: A B C
Output (the order may be different): Output (the order may be different and 1 and 2 may be swapped):
1 1
2 2
@ -578,7 +584,7 @@ single line:
cat num30000 | parallel --xargs echo | wc -l cat num30000 | parallel --xargs echo | wc -l
Output: Output (if you run this under Bash on GNU/Linux):
2 2
@ -607,14 +613,14 @@ total of 5 jobs:
cat num30000 | parallel --jobs 4 -m echo | wc -l cat num30000 | parallel --jobs 4 -m echo | wc -l
Output: Output (if you run this under Bash on GNU/Linux):
5 5
This is even more visible when running 4 jobs with 10 arguments. The This is even more visible when running 4 jobs with 10 arguments. The
10 arguments are being spread over 4 jobs: 10 arguments are being spread over 4 jobs:
parallel --jobs 4 -m echo ::: {1..10} parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10
Output: Output:
@ -763,7 +769,7 @@ Output:
pre-A-post pre-A-post
=head1 Controling the output =head1 Controlling the output
The output can prefixed with the argument: The output can prefixed with the argument:
@ -948,29 +954,29 @@ The directories are named after the variables and their values.
The number of concurrent jobs is given with --jobs/-j: The number of concurrent jobs is given with --jobs/-j:
/usr/bin/time parallel -N0 -j64 sleep 1 ::: {1..128} /usr/bin/time parallel -N0 -j64 sleep 1 :::: num128
With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run - With 64 jobs in parallel the 128 sleeps will take 2-8 seconds to run -
depending on how fast your machine is. depending on how fast your machine is.
By default --jobs is the same as the number of CPU cores. So this: By default --jobs is the same as the number of CPU cores. So this:
/usr/bin/time parallel -N0 sleep 1 ::: {1..128} /usr/bin/time parallel -N0 sleep 1 :::: num128
should take twice the time of running 2 jobs per CPU core: should take twice the time of running 2 jobs per CPU core:
/usr/bin/time parallel -N0 --jobs 200% sleep 1 ::: {1..128} /usr/bin/time parallel -N0 --jobs 200% sleep 1 :::: num128
--jobs 0 will run as many jobs in parallel as possible: --jobs 0 will run as many jobs in parallel as possible:
/usr/bin/time parallel -N0 --jobs 0 sleep 1 ::: {1..128} /usr/bin/time parallel -N0 --jobs 0 sleep 1 :::: num128
which should take 1-7 seconds depending on how fast your machine is. which should take 1-7 seconds depending on how fast your machine is.
--jobs can read from a file which is re-read when a job finishes: --jobs can read from a file which is re-read when a job finishes:
echo 50% > my_jobs echo 50% > my_jobs
/usr/bin/time parallel -N0 --jobs my_jobs sleep 1 ::: {1..128} & /usr/bin/time parallel -N0 --jobs my_jobs sleep 1 :::: num128 &
sleep 1 sleep 1
echo 0 > my_jobs echo 0 > my_jobs
wait wait
@ -982,7 +988,7 @@ parallel.
Instead of basing the percentage on the number of CPU cores Instead of basing the percentage on the number of CPU cores
GNU Parallel can base it on the number of CPUs: GNU Parallel can base it on the number of CPUs:
parallel --use-cpus-instead-of-cores -N0 sleep 1 ::: {1..128} parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num128
=head2 Interactivity =head2 Interactivity

7
testsuite/tests-to-run/parallel-freebsd.sh
View file

@ -46,7 +46,7 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
echo Non-shellshock-hardened to non-shellshock-hardened; echo Non-shellshock-hardened to non-shellshock-hardened;
funky() { echo Function $1; }; funky() { echo Function $1; };
export -f funky; export -f funky;
parallel --env funky -S localhost funky ::: non-shellshock-hardened' PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened'
bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
echo Non-shellshock-hardened to shellshock-hardened; echo Non-shellshock-hardened to shellshock-hardened;
@ -54,6 +54,11 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
export -f funky; export -f funky;
parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
echo '### Test --load (must give 1=true)'
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 &
parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 |
parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}'
EOF EOF
VBoxManage controlvm FreeBSD71 savestate VBoxManage controlvm FreeBSD71 savestate

8
testsuite/wanted-results/parallel-freebsd
View file

@ -54,11 +54,17 @@ works
with with
/usr/bin/perl -w ./shebang-wrap-opt options /usr/bin/perl -w ./shebang-wrap-opt options
options options
bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; echo Non-shellshock-hardened to non-shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S localhost funky ::: non-shellshock-hardened' bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; echo Non-shellshock-hardened to non-shellshock-hardened; funky() { echo Function $1; }; export -f funky; PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened'
bug #43358: shellshock breaks exporting functions using --env _ bug #43358: shellshock breaks exporting functions using --env _
Non-shellshock-hardened to non-shellshock-hardened Non-shellshock-hardened to non-shellshock-hardened
Function non-shellshock-hardened Function non-shellshock-hardened
parallel: Warning: Shell functions may not be supported in bash
bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; echo Non-shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; echo Non-shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
bug #43358: shellshock breaks exporting functions using --env _ bug #43358: shellshock breaks exporting functions using --env _
Non-shellshock-hardened to shellshock-hardened Non-shellshock-hardened to shellshock-hardened
Function shellshock-hardened Function shellshock-hardened
echo '### Test --load (must give 1=true)'
### Test --load (must give 1=true)
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 & parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 | parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}'
1