mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 14:07:55 +00:00
parallel: Fixed bug #47558: rc: line 1: syntax error near '='.
parallel: qqx() runs sh -c 2>/dev/null. testsuite: Test for bug #47644: Wrong slot number replacement when resuming.
This commit is contained in:
Ole Tange
parent
2aea2879c9
commit
61dca99762
|
|
@ -258,6 +258,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
|
|||
|
||||
* GNU Parallel was cited in: An Operational Radiometric Landsat Preprocessing Framework for Large-Area Time Series Applications https://www.uni-trier.de/fileadmin/fb6/prof/FER/Publikationen/frantz_et_al_ieee-tgrs-2016-post-print.pdf
|
||||
|
||||
* A basic demo of how GNU Parallel can speed up execution of commands https://www.youtube.com/watch?v=kl8LO2jcvMc
|
||||
|
||||
* Downloading a list of URLs http://blog.gypsydave5.com/2016/02/04/xargs-and-curl/
|
||||
|
||||
* qbatch uses GNU Parallel: https://pypi.python.org/pypi/qbatch/1.0rc2
|
||||
|
|
|
|||
28
src/env_parallel.csh
Executable file
28
src/env_parallel.csh
Executable file
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/csh
|
||||
|
||||
# This file must be sourced in csh:
|
||||
#
|
||||
# source `which env_parallel.csh`
|
||||
#
|
||||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2016
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, see <http://www.gnu.org/licenses/>
|
||||
# or write to the Free Software Foundation, Inc., 51 Franklin St,
|
||||
# Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
|
||||
71
src/parallel
71
src/parallel
|
|
@ -3563,7 +3563,8 @@ sub usage {
|
|||
"",
|
||||
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
|
||||
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
|
||||
"");
|
||||
"",
|
||||
"",);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -3615,13 +3616,13 @@ sub status_no_nl {
|
|||
sub warning {
|
||||
my @w = @_;
|
||||
my $prog = $Global::progname || "parallel";
|
||||
status(map { ($prog, ": Warning: ", $_, "\n"); } @w);
|
||||
status_no_nl(map { ($prog, ": Warning: ", $_, "\n"); } @w);
|
||||
}
|
||||
|
||||
sub error {
|
||||
my @w = @_;
|
||||
my $prog = $Global::progname || "parallel";
|
||||
status(map { ($prog, ": Error: ", $_, "\n"); } @w);
|
||||
status(map { ($prog.": Error: ". $_); } @w);
|
||||
}
|
||||
|
||||
sub die_bug {
|
||||
|
|
@ -3711,7 +3712,7 @@ sub citation {
|
|||
"GNU parallel is indirectly financed through citations, so if users",
|
||||
"do not know they should cite then you are making it harder to finance",
|
||||
"development. However, if you pay 10000 EUR, you should feel free to",
|
||||
"use --will-cite.",
|
||||
"use '--will-cite' in scripts.",
|
||||
"");
|
||||
last;
|
||||
}
|
||||
|
|
@ -3798,9 +3799,7 @@ sub qqx {
|
|||
if($Global::debug) {
|
||||
return qx{ @_ && true };
|
||||
} else {
|
||||
local *STDERR;
|
||||
open (STDERR, ">", "/dev/null");
|
||||
return qx{ @_ };
|
||||
return qx{ ( @_ ) 2>/dev/null };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4580,7 +4579,7 @@ sub swap_activity {
|
|||
my $file = $self->{'swap_activity_file'};
|
||||
my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".swp");
|
||||
::debug("swap", "\n", $swap_activity, "\n");
|
||||
::qqx("($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile) &");
|
||||
::qqx("($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile &)");
|
||||
}
|
||||
return $self->{'swap_activity'};
|
||||
}
|
||||
|
|
@ -4868,10 +4867,10 @@ sub loadavg {
|
|||
}
|
||||
# As the command can take long to run if run remote
|
||||
# save it to a tmp file before moving it to the correct file
|
||||
::debug("load", "Cmd: ", $cmd);
|
||||
::debug("load", "Cmd: ", $cmd,"\n");
|
||||
my $file = $self->{'loadavg_file'};
|
||||
my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa");
|
||||
::qqx(" ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & ");
|
||||
::qqx("($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile & )");
|
||||
}
|
||||
return $self->{'loadavg'};
|
||||
}
|
||||
|
|
@ -6860,11 +6859,14 @@ sub wrapped {
|
|||
and
|
||||
length $command > 499) {
|
||||
# csh does not like words longer than 1000 (499 quoted)
|
||||
# bzip2 breaks --sql mysql://...
|
||||
# $command = "perl -e '".base64_zip_eval()."' ".
|
||||
# join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"');
|
||||
$command = "perl -e '".base64_eval()."' ".
|
||||
join" ",string_base64('exec "'.::perl_quote_scalar($command).'"');
|
||||
# join" ",string_zip_base64(
|
||||
# 'exec "'.::perl_quote_scalar($command).'"');
|
||||
$command = "perl -e ".
|
||||
::shell_quote_scalar(base64_eval())." ".
|
||||
join(" ", ::shell_quote(
|
||||
string_base64('exec "'.
|
||||
::perl_quote_scalar($command).'"')));
|
||||
}
|
||||
$self->{'wrapped'} = $command;
|
||||
}
|
||||
|
|
@ -6888,7 +6890,7 @@ sub sshlogin {
|
|||
}
|
||||
|
||||
sub string_base64 {
|
||||
# Base64 encode it into 1000 byte blocks.
|
||||
# Base64 encode strings into 1000 byte blocks.
|
||||
# 1000 bytes is the largest word size csh supports
|
||||
# Input:
|
||||
# @strings = to be encoded
|
||||
|
|
@ -7153,8 +7155,9 @@ sub sshlogin_wrap {
|
|||
# bzip2 breaks --sql mysql://...
|
||||
# $env_command = "perl -e '".base64_zip_eval()."' ".
|
||||
# join" ",string_zip_base64($env_command);
|
||||
$env_command = "perl -e '".base64_eval()."' ".
|
||||
join" ",string_base64($env_command);
|
||||
$env_command = "perl -e ".
|
||||
::shell_quote_scalar(base64_eval())." ".
|
||||
join" ",::shell_quote(string_base64($env_command));
|
||||
$self->{'sshlogin_wrap'} = $env_command;
|
||||
} else {
|
||||
$self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($env_command);
|
||||
|
|
@ -7172,9 +7175,12 @@ sub sshlogin_wrap {
|
|||
}
|
||||
my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
|
||||
my $remote_command = $pwd.$envset.$bashfuncset.
|
||||
'@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script();
|
||||
$quoted_remote_command = "perl -e ".::shell_quote_scalar($remote_command);
|
||||
my $dq_remote_command = ::shell_quote_scalar($quoted_remote_command);
|
||||
'@ARGV="'.::perl_quote_scalar($command).'";'.
|
||||
monitor_parent_sshd_script();
|
||||
$quoted_remote_command = "perl -e ".
|
||||
::shell_quote_scalar($remote_command);
|
||||
my $dq_remote_command =
|
||||
::shell_quote_scalar($quoted_remote_command);
|
||||
if(length $dq_remote_command > 999
|
||||
or
|
||||
not $csh_friendly
|
||||
|
|
@ -7182,11 +7188,9 @@ sub sshlogin_wrap {
|
|||
$command =~ /\n/) {
|
||||
# csh does not deal well with > 1000 chars in one word
|
||||
# csh does not deal well with $ENV with \n
|
||||
# bzip2 breaks --sql mysql://...
|
||||
# $quoted_remote_command = "perl -e \\''".base64_zip_eval()."'\\' "."".
|
||||
# join" ",string_zip_base64($remote_command);
|
||||
$quoted_remote_command = "perl -e \\''".base64_eval()."'\\' ".
|
||||
join" ",string_base64($remote_command);
|
||||
$quoted_remote_command = "perl -e ".
|
||||
::shell_quote_scalar(::shell_quote_scalar(base64_eval()))." ".
|
||||
join" ",::shell_quote(string_base64($remote_command));
|
||||
} else {
|
||||
$quoted_remote_command = $dq_remote_command;
|
||||
}
|
||||
|
|
@ -8173,13 +8177,14 @@ sub set_exitsignal {
|
|||
$Global::halt_exitstatus = $job->exitstatus();
|
||||
}
|
||||
}
|
||||
::debug("halt","Pct: ",$Global::halt_pct," count: ",$Global::halt_count,"\n");
|
||||
::debug("halt","Pct: ",$Global::halt_pct,
|
||||
" count: ",$Global::halt_count,"\n");
|
||||
if($Global::halt_when eq "soon"
|
||||
and scalar(keys %Global::running) > 0) {
|
||||
::status
|
||||
("$Global::progname: Starting no more jobs. ",
|
||||
"Waiting for ", scalar(keys %Global::running),
|
||||
" jobs to finish.\n");
|
||||
("$Global::progname: Starting no more jobs. ".
|
||||
"Waiting for ". (keys %Global::running).
|
||||
" jobs to finish.");
|
||||
$Global::start_no_new_jobs ||= 1;
|
||||
}
|
||||
return($Global::halt_when);
|
||||
|
|
@ -8199,9 +8204,9 @@ sub set_exitsignal {
|
|||
if($Global::halt_when eq "soon"
|
||||
and scalar(keys %Global::running) > 0) {
|
||||
::status
|
||||
("$Global::progname: Starting no more jobs. ",
|
||||
"Waiting for ", scalar(keys %Global::running),
|
||||
" jobs to finish.\n");
|
||||
("$Global::progname: Starting no more jobs. ".
|
||||
"Waiting for ". (keys %Global::running).
|
||||
" jobs to finish.");
|
||||
$Global::start_no_new_jobs ||= 1;
|
||||
}
|
||||
return($Global::halt_when);
|
||||
|
|
@ -9158,7 +9163,7 @@ sub tmux_length {
|
|||
push @out, ::qqx($tmuxcmd);
|
||||
unlink $tmpfile;
|
||||
}
|
||||
::debug("tmux","tmux-length ",@out);
|
||||
::debug("tmux","tmux-out ",@out);
|
||||
chomp @out;
|
||||
# The arguments is given 3 times on the command line
|
||||
# and the wrapping is around 30 chars
|
||||
|
|
|
|||
|
|
@ -464,9 +464,9 @@ B<--will-cite>.
|
|||
If you use B<--will-cite> in scripts to be run by others you are
|
||||
making it harder for others to see the citation notice. The
|
||||
development of GNU B<parallel> is indirectly financed through
|
||||
citations, so if users do not know they should cite then you are
|
||||
citations, so if your users do not know they should cite then you are
|
||||
making it harder to finance development. However, if you pay 10000
|
||||
EUR, you should feel free to use B<--will-cite>.
|
||||
EUR, you should feel free to use B<--will-cite> in scripts.
|
||||
|
||||
|
||||
=item B<--block> I<size>
|
||||
|
|
|
|||
|
|
@ -1276,7 +1276,7 @@
|
|||
|
||||
<p>And a graphic bar can be shown with <b>--bar</b> and <b>zenity</b>:</p>
|
||||
|
||||
<pre><code> seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill)</code></pre>
|
||||
<pre><code> seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill --auto-close)</code></pre>
|
||||
|
||||
<p>A logfile of the jobs completed so far can be generated with <b>--joblog</b>:</p>
|
||||
|
||||
|
|
|
|||
|
|
@ -1208,7 +1208,7 @@ A progress bar can be shown with B<--bar>:
|
|||
|
||||
And a graphic bar can be shown with B<--bar> and B<zenity>:
|
||||
|
||||
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill)
|
||||
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --progress --auto-kill --auto-close)
|
||||
|
||||
A logfile of the jobs completed so far can be generated with B<--joblog>:
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,10 @@ echo '### Do children receive --termseq signals'
|
|||
echo | stdout parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals;
|
||||
sleep 3;
|
||||
|
||||
echo '**'
|
||||
|
||||
echo '### bug #47644: Wrong slot number replacement when resuming'
|
||||
seq 0 20 | parallel -kj 4 --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==10 and exit =}';
|
||||
seq 0 20 | parallel -kj 4 --resume --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==110 and exit =}'
|
||||
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -64,6 +64,21 @@ echo '### csh2'
|
|||
setenv C `seq 300 -2 1|xargs`;
|
||||
parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
|
||||
|
||||
echo '### rc'
|
||||
echo "3 big vars run remotely - length(base64) > 1000"
|
||||
stdout ssh rc@lo 'A=`{seq 200};
|
||||
B=`{seq 200 -1 1};
|
||||
C=`{seq 300 -2 1};
|
||||
parallel -Src@lo --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C'
|
||||
|
||||
echo '### rc2'
|
||||
echo "3 big vars run locally"
|
||||
stdout ssh rc@lo 'A=`{seq 200};
|
||||
B=`{seq 200 -1 1};
|
||||
C=`{seq 300 -2 1};
|
||||
parallel --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C'
|
||||
|
||||
echo '### Test tmux works on different shells'
|
||||
(stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output';
|
||||
(stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | grep -v 'See output';
|
||||
|
|
|
|||
|
|
@ -36,4 +36,6 @@ echo ZZZZ | script -q -f -c /tmp/parallel-script-for-script3 /dev/null
|
|||
sleep 2
|
||||
rm /tmp/parallel-script-for-script3
|
||||
|
||||
stdout parallel --citation < /dev/null
|
||||
|
||||
touch ~/.parallel/will-cite
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export PARALLEL=-k
|
|||
perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/parallel_tutorial.pod |
|
||||
egrep -v 'curl|tty|parallel_tutorial|interactive|example.(com|net)|shellquote|works' |
|
||||
perl -pe 's/username@//;s/user@//;
|
||||
s/zenity/zenity --timeout=12/;
|
||||
s/zenity/zenity --timeout=15/;
|
||||
s:/usr/bin/time:/usr/bin/time -f %e:;
|
||||
s:ignored_vars:ignored_vars|sort:;
|
||||
# Remove \n to join all joblogs into the previous block
|
||||
|
|
|
|||
|
|
@ -34,9 +34,34 @@ bash---pstree
|
|||
**
|
||||
echo '### Do children receive --termseq signals'
|
||||
### Do children receive --termseq signals
|
||||
show_signals() { perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print STDERR \"Got $_\\n\"; }";} while(1){sleep 1}'; }; export -f show_signals; echo | stdout parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals; echo | stdout parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals; sleep 3;
|
||||
show_signals() { perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print STDERR \"Got $_\\n\"; }";} while(1){sleep 1}'; }; export -f show_signals; echo | stdout parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals; echo | stdout parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals; sleep 3; echo '**'
|
||||
Got TERM
|
||||
Got TERM
|
||||
Got TERM
|
||||
Got INT
|
||||
Got TERM
|
||||
**
|
||||
echo '### bug #47644: Wrong slot number replacement when resuming'
|
||||
### bug #47644: Wrong slot number replacement when resuming
|
||||
seq 0 20 | parallel -kj 4 --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==10 and exit =}'; seq 0 20 | parallel -kj 4 --resume --delay .2 --joblog /tmp/parallel-bug-47558 'sleep 1; echo {%} {=$_==110 and exit =}'
|
||||
1 0
|
||||
2 1
|
||||
3 2
|
||||
4 3
|
||||
1 4
|
||||
2 5
|
||||
3 6
|
||||
4 7
|
||||
1 8
|
||||
2 9
|
||||
3 10
|
||||
4 11
|
||||
1 12
|
||||
2 13
|
||||
3 14
|
||||
4 15
|
||||
1 16
|
||||
2 17
|
||||
3 18
|
||||
4 19
|
||||
1 20
|
||||
|
|
|
|||
|
|
@ -59,6 +59,22 @@ echo '### csh2'
|
|||
1 200 692
|
||||
1 200 692
|
||||
1 150 547
|
||||
echo '### rc'
|
||||
### rc
|
||||
echo "3 big vars run remotely - length(base64) > 1000"
|
||||
3 big vars run remotely - length(base64) > 1000
|
||||
stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel -Src@lo --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C'
|
||||
1 200 692
|
||||
1 200 692
|
||||
1 150 547
|
||||
echo '### rc2'
|
||||
### rc2
|
||||
echo "3 big vars run locally"
|
||||
3 big vars run locally
|
||||
stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C'
|
||||
1 200 692
|
||||
1 200 692
|
||||
1 150 547
|
||||
echo '### Test tmux works on different shells'
|
||||
### Test tmux works on different shells
|
||||
(stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output';
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ Environment variables are:
|
|||
stderr
|
||||
|
||||
rsync: connection unexpectedly closed (0 bytes received so far) [sender]
|
||||
rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.1]
|
||||
rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.0]
|
||||
/usr/lib/autossh/autossh: invalid option -- '-'
|
||||
usage: autossh [-V] [-M monitor_port[:echo_port]] [-f] [SSH_OPTIONS]
|
||||
|
||||
|
|
@ -66,7 +66,7 @@ Environment variables are:
|
|||
stderr
|
||||
|
||||
rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
|
||||
rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.1]
|
||||
rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.0]
|
||||
echo '### bug #46520: --basefile cleans up without --cleanup'
|
||||
### bug #46520: --basefile cleans up without --cleanup
|
||||
touch bug_46520; parallel -S parallel@lo --bf bug_46520 ls ::: bug_46520; ssh parallel@lo ls bug_46520; parallel -S parallel@lo --cleanup --bf bug_46520 ls ::: bug_46520; stdout ssh parallel@lo ls bug_46520 # should not exist
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -9,7 +9,7 @@ please cite:
|
|||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
To silence the citation notice: run 'parallel --bibtex'.
|
||||
To silence the citation notice: run 'parallel --citation'.
|
||||
|
||||
echo a
|
||||
a
|
||||
|
|
@ -30,7 +30,36 @@ please cite:
|
|||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
To silence the citation notice: run 'parallel --bibtex'.
|
||||
To silence the citation notice: run 'parallel --citation'.
|
||||
|
||||
10 files to edit
|
||||
[?1049h[?1h=[1;24r[?12;25h[?12l[?25h[27m[m[H[2J[?25l[24;1H"file1" [New File][2;1H[1m[34m~ [3;1H~ [4;1H~ [5;1H~ [6;1H~ [7;1H~ [8;1H~ [9;1H~ [10;1H~ [11;1H~ [12;1H~ [13;1H~ [14;1H~ [15;1H~ [16;1H~ [17;1H~ [18;1H~ [19;1H~ [20;1H~ [21;1H~ [22;1H~ [23;1H~ [1;1H[?12l[?25h[?25l[m[24;1H[1m[37m[41mE173: 9 more files to edit[1;1H[?12l[?25h[24;1H[m[24;1H[K[24;1H[?1l>[?1049l
|
||||
[?1049h[?1h=[1;24r[?12;25h[?12l[?25h[27m[m[H[2J[?25l[24;1H"file1" [New File][2;1H[1m[34m~ [3;1H~ [4;1H~ [5;1H~ [6;1H~ [7;1H~ [8;1H~ [9;1H~ [10;1H~ [11;1H~ [12;1H~ [13;1H~ [14;1H~ [15;1H~ [16;1H~ [17;1H~ [18;1H~ [19;1H~ [20;1H~ [21;1H~ [22;1H~ [23;1H~ [1;1H[?12l[?25h[?25l[m[24;1H[1m[37m[41mE173: 9 more files to edit[1;1H[?12l[?25h[24;1H[m[24;1H[K[24;1H[?1l>[?1049lAcademic tradition requires you to cite works you base your article on.
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite:
|
||||
|
||||
@article{Tange2011a,
|
||||
title = {GNU Parallel - The Command-Line Power Tool},
|
||||
author = {O. Tange},
|
||||
address = {Frederiksberg, Denmark},
|
||||
journal = {;login: The USENIX Magazine},
|
||||
month = {Feb},
|
||||
number = {1},
|
||||
volume = {36},
|
||||
url = {http://www.gnu.org/s/parallel},
|
||||
year = {2011},
|
||||
pages = {42-47},
|
||||
doi = {10.5281/zenodo.16303}
|
||||
}
|
||||
|
||||
(Feel free to use \nocite{Tange2011a})
|
||||
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
If you send a copy of your published article to tange@gnu.org, it will be
|
||||
mentioned in the release notes of next version of GNU Parallel.
|
||||
|
||||
|
||||
|
||||
Type: 'will cite' and press enter.
|
||||
>
|
||||
|
|
@ -456,7 +456,7 @@ echo '### true < 32767-ys.xi'
|
|||
### true < 32767-ys.xi
|
||||
stdout xargs true < 32767-ys.xi
|
||||
stdout parallel -k true < 32767-ys.xi
|
||||
parallel: Error: Command line too long (98306 >= 65535) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ...
|
||||
parallel: Error: Command line too long (98306 >= 65528) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ...
|
||||
echo '### true < 16383-ys.xi'
|
||||
### true < 16383-ys.xi
|
||||
stdout xargs true < 16383-ys.xi
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | par
|
|||
20 179960 1286702
|
||||
echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1
|
||||
### Test -X with 60000 args
|
||||
af6c8c69409b5cbc55aeb95174e23c37 -
|
||||
97715240fa65309902932877d24273db -
|
||||
21 60000 1346682
|
||||
echo '### Test -X with 60000 args and 5 expansions'
|
||||
### Test -X with 60000 args and 5 expansions
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; CHAR=$(cat /tmp/a$$ | wc -c); LINES=$(cat /tmp/a$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/awc$$; rm /tmp/a$$ /tmp/awc$$
|
||||
### Test of xargs -m command lines > 130k
|
||||
cb2184f70b8e9058cae1413426f8a9ae -
|
||||
14bacad229d8b0d32be0a2339c2a6af7 -
|
||||
Chars per line: 63437
|
||||
11 119989 697810
|
||||
echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$
|
||||
### Test of xargs -X command lines > 130k
|
||||
514f33181f346d416a1ada982b43a542 -
|
||||
81c0a85162c989c07f666b827a30ce52 -
|
||||
Chars per line: 62906
|
||||
13 60000 817788
|
||||
echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum
|
||||
### Test of xargs -m command lines > 130k
|
||||
1e763b036fe666d229520dad45610e6c -
|
||||
4c821f9aa0e42e53b4b5ff8cc809490b -
|
||||
echo '### This causes problems if we kill child processes'; # 2>/dev/null to avoid parallel: Warning: Starting 45 processes took > 2 sec.
|
||||
### This causes problems if we kill child processes
|
||||
seq 2 40 | parallel -j 0 seq 1 10 2>/dev/null | sort | md5sum
|
||||
|
|
|
|||
|
|
@ -82,13 +82,13 @@ GNU parallel comes with no warranty.
|
|||
Web site: http://www.gnu.org/software/parallel
|
||||
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite as described in 'parallel --bibtex'.
|
||||
please cite as described in 'parallel --citation'.
|
||||
echo '### bug #39787: --xargs broken'
|
||||
### bug #39787: --xargs broken
|
||||
nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"'
|
||||
65526
|
||||
65526
|
||||
37842
|
||||
65520
|
||||
65520
|
||||
37854
|
||||
echo '### --delay should grow by 3 sec per arg'
|
||||
### --delay should grow by 3 sec per arg
|
||||
stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"'
|
||||
|
|
|
|||
|
|
@ -407,8 +407,8 @@ seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1
|
|||
echo "### BUG: empty lines with --show-limit"
|
||||
### BUG: empty lines with --show-limit
|
||||
echo | $NICEPAR --show-limits
|
||||
Maximal size of command: 131071
|
||||
Maximal used size of command: 65535
|
||||
Maximal size of command: 131048
|
||||
Maximal used size of command: 65528
|
||||
|
||||
Execution of will continue now, and it will try to read its input
|
||||
and run commands; if this is not what you wanted to happen, please
|
||||
|
|
|
|||
|
|
@ -407,13 +407,13 @@ C
|
|||
outdir/1/C/seq
|
||||
outdir/1/C/stderr
|
||||
outdir/1/C/stdout
|
||||
/bin/bash: outdir/1/A/seq: No such file or directory
|
||||
/bin/bash: outdir/1/A/seq: Permission denied
|
||||
/bin/bash: line 1: outdir/1/A/stderr: No such file or directory
|
||||
/bin/bash: line 2: outdir/1/A/stdout: No such file or directory
|
||||
/bin/bash: line 3: outdir/1/B/seq: No such file or directory
|
||||
/bin/bash: line 3: outdir/1/B/seq: Permission denied
|
||||
/bin/bash: line 4: outdir/1/B/stderr: No such file or directory
|
||||
/bin/bash: line 5: outdir/1/B/stdout: No such file or directory
|
||||
/bin/bash: line 6: outdir/1/C/seq: No such file or directory
|
||||
/bin/bash: line 6: outdir/1/C/seq: Permission denied
|
||||
/bin/bash: line 7: outdir/1/C/stderr: No such file or directory
|
||||
/bin/bash: line 8: outdir/1/C/stdout: No such file or directory
|
||||
parallel --header : --results outdir echo ::: f1 A B ::: f2 C D
|
||||
|
|
@ -522,7 +522,7 @@ Computers / CPU cores / Max jobs to run
|
|||
|
||||
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
|
||||
|
||||
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill)
|
||||
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=15 --progress --auto-kill --auto-close)
|
||||
BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0
|
||||
cat /tmp/log;
|
||||
parallel --joblog /tmp/log exit ::: 1 2 3 0
|
||||
|
|
@ -708,6 +708,36 @@ parallel@lo foo bar
|
|||
alias myecho=echo
|
||||
myvar="Joe's var is"
|
||||
env_parallel -S $SERVER1 'myecho $myvar' ::: green
|
||||
env_parallel only works if it is a function. Do the below and restart your shell.
|
||||
|
||||
bash: Put this in /home/tange/.bashrc: . /usr/local/bin/env_parallel.bash
|
||||
E.g. by doing: echo '. /usr/local/bin/env_parallel.bash' >> /home/tange/.bashrc
|
||||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
zsh: Put this in /home/tange/.zshrc: . /usr/local/bin/env_parallel.zsh
|
||||
E.g. by doing: echo '. /usr/local/bin/env_parallel.zsh' >> /home/tange/.zshenv
|
||||
Supports: functions, variables, arrays
|
||||
|
||||
fish: Put this in /home/tange/.config/fish/config.fish:
|
||||
source (which env_parallel.fish)
|
||||
E.g. by doing:
|
||||
echo 'source (which env_parallel.fish)' >> /home/tange/.config/fish/config.fish
|
||||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
ksh: Put this in /home/tange/.kshrc: source /usr/local/bin/env_parallel.ksh
|
||||
E.g. by doing: echo 'source /usr/local/bin/env_parallel.ksh' >> /home/tange/.kshrc
|
||||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
pdksh: Put this in /home/tange/.profile: source /usr/local/bin/env_parallel.pdksh
|
||||
E.g. by doing: echo 'source /usr/local/bin/env_parallel.pdksh' >> /home/tange/.profile
|
||||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
csh: Put this in /home/tange/.cshrc: source /usr/local/bin/env_parallel.csh
|
||||
E.g. by doing: echo 'source /usr/local/bin/env_parallel.csh' >> /home/tange/.cshrc
|
||||
Supports: aliases
|
||||
|
||||
For details: see man env_parallel
|
||||
|
||||
MYVAR='foo bar'
|
||||
export MYVAR
|
||||
parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz
|
||||
|
|
@ -827,7 +857,7 @@ _
|
|||
}
|
||||
export -f my_func3
|
||||
parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file
|
||||
( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64 BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
|
||||
( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; BASE64 BASE64\=;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
|
||||
parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14
|
||||
cat /tmp/log.csv
|
||||
10
|
||||
|
|
@ -1238,7 +1268,7 @@ GNU parallel comes with no warranty.
|
|||
Web site: http://www.gnu.org/software/parallel
|
||||
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite as described in 'parallel --bibtex'.
|
||||
please cite as described in 'parallel --citation'.
|
||||
parallel --minversion VERSION && echo Your version is at least VERSION.
|
||||
VERSION
|
||||
Your version is at least VERSION.
|
||||
|
|
@ -1269,8 +1299,7 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
|||
If you send a copy of your published article to tange@gnu.org, it will be
|
||||
mentioned in the release notes of next version of GNU Parallel.
|
||||
|
||||
parallel --max-line-length-allowed
|
||||
131071
|
||||
|
||||
parallel --number-of-cpus
|
||||
parallel --number-of-cores
|
||||
1
|
||||
|
|
|
|||
Loading…
Reference in a new issue