mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: Rewrite of --tmux. Fixed some csh issues.
This commit is contained in:
Ole Tange
parent
3549c38b0f
commit
a9dc4f3ea4
|
|
@ -208,18 +208,45 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
|
|||
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
|
||||
Jesse Alama <jesse.alama@gmail.com>
|
||||
|
||||
Subject: GNU Parallel 20150222 ('') released
|
||||
Subject: GNU Parallel 20150222 (' (((:~{> Krudttønden') released
|
||||
|
||||
GNU Parallel 20150122 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
GNU Parallel 20150222 (' (((:~{> Krudttønden') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
|
||||
Haiku of the month:
|
||||
|
||||
<<>>
|
||||
xargs' space and quote
|
||||
headache causing behaviour.
|
||||
Use GNU Parallel
|
||||
-- Ole Tange
|
||||
|
||||
New in this release:
|
||||
|
||||
* --tmux has gotten a major overhaul.
|
||||
|
||||
* GNU Parallel was cited in: RaftLib: A C++ Template Library for High Performance Stream Parallel Processing http://www.cs.wustl.edu/~lip/pubs/pmam15_jbeard.pdf
|
||||
|
||||
* GNU Parallel was cited in: Towards Collaborative Exploration and Analysis of Big Data from Mars: A Noachis Terra Case Study http://link.springer.com/chapter/10.1007/978-3-319-13865-7_25
|
||||
|
||||
* GNU Parallel was cited in: Quantifying properties of hot and dense QCD matter through systematic model-to-data comparison http://arxiv.org/pdf/1502.00339.pdf
|
||||
|
||||
* GNU Parallel was cited in: Towards Collaborative Exploration and Analysis of Big Data from Mars: A Noachis Terra Case Study http://link.springer.com/chapter/10.1007/978-3-319-13865-7_25
|
||||
|
||||
* GNU Parallel was cited in: Towards Recommender Engineering Tools and Experiments for Identifying Recommender Differences http://elehack.net/research/thesis/mde-thesis.pdf
|
||||
|
||||
* GNU Parallel was using (unfortunately with wrong citation) in: Performance and Scaling Comparison Study of RDBMS and NoSQL (MongoDB) http://ijact.in/wp-content/uploads/2014/11/COMPUSOFT-311-1270-1275.pdf
|
||||
|
||||
* GNU Parallel was used (unfortunately without citation) in: Parallel Implementation of Big Data Pre-Processing Algorithms for Sentiment Analysis of Social Networking Data http://www.researchmathsci.org/IJFMAart/ijfma-v6n2-7.pdf
|
||||
|
||||
* GNU Parallel was used (unfortunately without citation) in: SpeedSeq: Ultra-fast personal genome analysis and interpretation http://biorxiv.org/content/biorxiv/early/2014/12/05/012179.full.pdf
|
||||
|
||||
* Zip Folders with GNU Parallel http://fazky.github.io/Linux/2015-01-07-GNU-Parallel.html
|
||||
|
||||
* Using GNU Parallel with Freesurfer http://programminginadarkroom.blogspot.dk/2015/02/using-gnu-parallel-with-freesurfer.html
|
||||
|
||||
* GNU Parallel is used in Velociraptor: https://github.com/ericwhyne/Velociraptor
|
||||
|
||||
* Marcus Beach GNU Parallel http://marcusbeach.co/gnu-parallel/
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
|
|
|||
206
src/parallel
206
src/parallel
|
|
@ -132,7 +132,6 @@ if($Global::semaphore) {
|
|||
$sem = acquire_semaphore();
|
||||
}
|
||||
$SIG{TERM} = \&start_no_new_jobs;
|
||||
|
||||
start_more_jobs();
|
||||
if(not $opt::pipepart) {
|
||||
if($opt::pipe) {
|
||||
|
|
@ -954,7 +953,7 @@ sub parse_options {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20150123;
|
||||
$Global::version = 20150201;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -1161,7 +1160,7 @@ sub open_joblog {
|
|||
if(/$joblog_regexp/o) {
|
||||
# This is 30% faster than set_job_already_run($1);
|
||||
vec($Global::job_already_run,($1||0),1) = 1;
|
||||
} elsif(not /\d+\s+[^\s]+\s+([0-9.]+\s+){6}/) {
|
||||
} elsif(not /\d+\s+[^\s]+\s+([-0-9.]+\s+){6}/) {
|
||||
::error("Format of '$opt::joblog' is wrong: $_");
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
|
|
@ -1198,8 +1197,10 @@ sub find_compression_program {
|
|||
# $compress_program = compress program with options
|
||||
# $decompress_program = decompress program with options
|
||||
|
||||
# Search for these. Sorted by speed
|
||||
my @prg = qw(lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2);
|
||||
# Search for these. Sorted by speed on 16 core
|
||||
# parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2
|
||||
# sort -nk4 jl
|
||||
my @prg = qw(lz4 pigz lzop plzip pbzip2 pxz gzip lzma xz bzip2 lzip);
|
||||
for my $p (@prg) {
|
||||
if(which($p)) {
|
||||
return ("$p -c -1","$p -dc");
|
||||
|
|
@ -1395,6 +1396,7 @@ sub read_args_from_command_line {
|
|||
sub cleanup {
|
||||
# Returns: N/A
|
||||
if(@opt::basefile) { cleanup_basefile(); }
|
||||
unlink keys %Global::unlink;
|
||||
}
|
||||
|
||||
sub __QUOTING_ARGUMENTS_FOR_SHELL__ {}
|
||||
|
|
@ -2569,7 +2571,7 @@ sub parallelized_host_filtering {
|
|||
while (my ($host, $sshlogin) = each %Global::host) {
|
||||
if($host eq ":") { next }
|
||||
# The 'true' is used to get the $host out later
|
||||
my $sshcmd = "true $host;" . $sshlogin->sshcommand()." ".$sshlogin->serverlogin();
|
||||
my $sshcmd = "true $host; exec " . $sshlogin->sshcommand()." ".$sshlogin->serverlogin();
|
||||
push(@cores, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cores\n\0");
|
||||
push(@cpus, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cpus\n\0");
|
||||
push(@maxline, $host."\t".$sshcmd." ".$envvar." parallel --max-line-length-allowed\n\0");
|
||||
|
|
@ -2771,6 +2773,7 @@ sub start_no_new_jobs {
|
|||
# Start no more jobs
|
||||
# Returns: N/A
|
||||
$SIG{TERM} = $Global::original_sig{TERM};
|
||||
unlink keys %Global::unlink;
|
||||
::status
|
||||
("$Global::progname: SIGTERM received. No new jobs will be started.\n",
|
||||
"$Global::progname: Waiting for these ", scalar(keys %Global::running),
|
||||
|
|
@ -2908,11 +2911,13 @@ sub usage {
|
|||
"See 'man $Global::progname' for details",
|
||||
"",
|
||||
"Academic tradition requires you to cite works you base your article on.",
|
||||
"When using programs that use GNU Parallel to process data for publication please cite:",
|
||||
"When using programs that use GNU Parallel to process data for publication",
|
||||
"please cite:",
|
||||
"",
|
||||
"O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
|
||||
";login: The USENIX Magazine, February 2011:42-47.",
|
||||
" O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
|
||||
" ;login: The USENIX Magazine, February 2011:42-47.",
|
||||
"",
|
||||
"This helps funding further development; and it won't cost you a cent.",
|
||||
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.\n",
|
||||
"");
|
||||
}
|
||||
|
|
@ -3003,7 +3008,8 @@ sub bibtex {
|
|||
# Returns: N/A
|
||||
print join("\n",
|
||||
"Academic tradition requires you to cite works you base your article on.",
|
||||
"When using programs that use GNU Parallel to process data for publication please cite:",
|
||||
"When using programs that use GNU Parallel to process data for publication",
|
||||
"please cite:",
|
||||
"",
|
||||
"\@article{Tange2011a,",
|
||||
" title = {GNU Parallel - The Command-Line Power Tool},",
|
||||
|
|
@ -3020,8 +3026,7 @@ sub bibtex {
|
|||
"",
|
||||
"(Feel free to use \\nocite{Tange2011a})",
|
||||
"",
|
||||
"This helps funding further development.",
|
||||
"",
|
||||
"This helps funding further development; and it won't cost you a cent.",
|
||||
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
|
||||
"",
|
||||
"If you send a copy of your published article to tange\@gnu.org, it will be",
|
||||
|
|
@ -3388,6 +3393,7 @@ sub multiply_binary_prefix {
|
|||
sub tmpfile {
|
||||
# Create tempfile as $TMPDIR/parXXXXX
|
||||
# Returns:
|
||||
# $filehandle = opened file handle
|
||||
# $filename = file name created
|
||||
return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_);
|
||||
}
|
||||
|
|
@ -5959,6 +5965,15 @@ sub sshlogin_wrap {
|
|||
# Split up --env VAR1,VAR2
|
||||
push @vars, split /,/, $varstring;
|
||||
}
|
||||
for (@vars) {
|
||||
if(-r $_ and not -d) {
|
||||
# Read as environment definition bug #44041
|
||||
# TODO parse this
|
||||
my $fh = ::open_or_exit($_);
|
||||
$Global::envdef = join("",<$fh>);
|
||||
close $fh;
|
||||
}
|
||||
}
|
||||
if(grep { /^_$/ } @vars) {
|
||||
# --env _
|
||||
# Include all vars that are not in a clean environment
|
||||
|
|
@ -6039,7 +6054,7 @@ sub sshlogin_wrap {
|
|||
my $env_command = $envset.$bashfuncset.
|
||||
'@ARGV="'.::perl_quote_scalar($command).'";'.
|
||||
"exec\"$Global::shell\",\"-c\",\(\$bashfunc.\"\@ARGV\"\)\;die\"exec:\$\!\\n\"\;";
|
||||
if(length $env_command > 1000
|
||||
if(length $env_command > 999
|
||||
or
|
||||
not $csh_friendly
|
||||
or
|
||||
|
|
@ -6067,7 +6082,7 @@ sub sshlogin_wrap {
|
|||
my $remote_command = $pwd.$envset.$bashfuncset.
|
||||
'@ARGV="'.::perl_quote_scalar($command).'";'. monitor_parent_sshd_script();
|
||||
$quoted_remote_command = "perl -e ".::shell_quote_scalar($remote_command);
|
||||
if(length $quoted_remote_command > 1000
|
||||
if(length $quoted_remote_command > 999
|
||||
or
|
||||
not $csh_friendly
|
||||
or
|
||||
|
|
@ -6431,40 +6446,84 @@ sub print_dryrun_and_verbose {
|
|||
}
|
||||
}
|
||||
|
||||
sub tmux_wrap {
|
||||
# Wrap command with tmux for session pPID
|
||||
# Input:
|
||||
# $actual_command = the actual command being run (incl ssh wrap)
|
||||
my $self = shift;
|
||||
my $actual_command = shift;
|
||||
# Temporary file name. Used for fifo to communicate exit val
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx");
|
||||
$Global::unlink{$tmpfile}=1;
|
||||
close $fh;
|
||||
unlink $tmpfile;
|
||||
my $visual_command = $self->replaced();
|
||||
my $title = $visual_command;
|
||||
# ; causes problems
|
||||
# ascii 194-245 annoys tmux
|
||||
$title =~ tr/[\011-\016;\302-\365]//d;
|
||||
{
|
||||
my $tmuxsocket;
|
||||
|
||||
my $tmux;
|
||||
if($Global::total_running == 0) {
|
||||
$tmux = "tmux new-session -s p$$ -d -n ".
|
||||
::shell_quote_scalar($title);
|
||||
::status("See output with: tmux attach -t p$$\n");
|
||||
} else {
|
||||
$tmux = "tmux new-window -t p$$ -n ".::shell_quote_scalar($title);
|
||||
sub tmux_wrap {
|
||||
# Wrap command with tmux for session pPID
|
||||
# Input:
|
||||
# $actual_command = the actual command being run (incl ssh wrap)
|
||||
my $self = shift;
|
||||
my $actual_command = shift;
|
||||
# Temporary file name. Used for fifo to communicate exit val
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx");
|
||||
$Global::unlink{$tmpfile}=1;
|
||||
close $fh;
|
||||
unlink $tmpfile;
|
||||
my $visual_command = $self->replaced();
|
||||
my $title = $visual_command;
|
||||
if($visual_command =~ /\0/) {
|
||||
::error("Command line contains NUL. tmux is confused by NUL.\n");
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
# ; causes problems
|
||||
# ascii 194-245 annoys tmux
|
||||
$title =~ tr/[\011-\016;\302-\365]//d;
|
||||
$title = ::shell_quote_scalar($title);
|
||||
|
||||
my $l_act = length($actual_command);
|
||||
my $l_tit = length($title);
|
||||
# The line to run contains a 118 chars extra code + the title 2x
|
||||
my $l_tot = 2 * $l_tit + $l_act + 30;
|
||||
|
||||
while($l_tit < 1000 and
|
||||
(
|
||||
(900 < $l_tot and $l_tot < 1300)
|
||||
or
|
||||
(9250 < $l_tot and $l_tot < 9800)
|
||||
)) {
|
||||
# tmux blocks for certain lengths:
|
||||
# 900 < title + command < 1200
|
||||
# 9250 < title + command < 9800
|
||||
# but only if title < 1000, so expand the title with 75 spaces
|
||||
# The measured lengths are:
|
||||
# 996 < (title + whole command) < 1127
|
||||
# 9331 < (title + whole command) < 9636
|
||||
$title = $title.('\ 'x75);
|
||||
$l_tit = length($title);
|
||||
$l_tot = 2 * $l_tit + $l_act + 30;
|
||||
}
|
||||
|
||||
my $tmux;
|
||||
$ENV{'TMUX'} ||= "tmux";
|
||||
if(not $tmuxsocket) {
|
||||
(undef, $tmuxsocket) = ::tmpfile(SUFFIX => ".tms");
|
||||
$Global::unlink{$tmuxsocket} = 1;
|
||||
unlink $tmuxsocket;
|
||||
::status("See output with: $ENV{'TMUX'} -S $tmuxsocket attach -t p$$\n");
|
||||
}
|
||||
$tmux = $ENV{'TMUX'}." -S $tmuxsocket new-session -s p$$ -d 'sleep .2' >&/dev/null;" .
|
||||
$ENV{'TMUX'}." -S $tmuxsocket new-window -t p$$ -n $title";
|
||||
|
||||
::debug("tmux", "title len:", $l_tit, " act ", $l_act, " max ",
|
||||
$Limits::Command::line_max_len, " tot ",
|
||||
$l_tot, "\n");
|
||||
|
||||
return "mkfifo $tmpfile; $tmux ".
|
||||
# Run in tmux
|
||||
::shell_quote_scalar
|
||||
(
|
||||
"(".$actual_command.');'.
|
||||
# The triple print is needed - otherwise the testsuite fails
|
||||
q[ perl -e 'while($t++<3){ print $ARGV[0],"\n" }' $?h/$status >> ].$tmpfile."&".
|
||||
"echo $title; echo \007Job finished at: `date`;sleep 10"
|
||||
).
|
||||
# Run outside tmux
|
||||
# Read a / separated line: 0h/2 for csh, 2/0 for bash.
|
||||
# If csh the first will be 0h, so use the second as exit value.
|
||||
# Otherwise just use the first value as exit value.
|
||||
q{; exec perl -e '$/="/";$_=<>;$c=<>;unlink $ARGV; /(\d+)h/ and exit($1);exit$c' }.$tmpfile;
|
||||
}
|
||||
return "mkfifo $tmpfile; $tmux ".
|
||||
# Run in tmux
|
||||
::shell_quote_scalar(
|
||||
"(".$actual_command.');(echo $?$status;echo 255) >'.$tmpfile."&".
|
||||
"echo ".::shell_quote_scalar($visual_command).";".
|
||||
"echo \007Job finished at: `date`;sleep 10").
|
||||
# Run outside tmux
|
||||
# Read the first line from the fifo and use that as status code
|
||||
"; exit `perl -ne 'unlink \$ARGV; 1..1 and print' $tmpfile` ";
|
||||
}
|
||||
|
||||
sub is_already_in_results {
|
||||
|
|
@ -6516,7 +6575,7 @@ sub should_be_retried {
|
|||
::debug("run", "Retry ", $self->seq(), "\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
|
|
@ -7019,10 +7078,12 @@ sub populate {
|
|||
::error("Command line too long (",
|
||||
$self->len(), " >= ",
|
||||
$max_len,
|
||||
") at number ",
|
||||
") at input ",
|
||||
$self->{'arg_queue'}->arg_number(),
|
||||
": ".
|
||||
(substr($args,0,50))."...\n");
|
||||
((length $args > 50) ?
|
||||
(substr($args,0,50))."...\n" :
|
||||
$args."\n"));
|
||||
$self->{'arg_queue'}->unget($self->pop());
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
|
|
@ -7764,7 +7825,7 @@ sub max_length {
|
|||
print $fh $cached_limit;
|
||||
close $fh;
|
||||
}
|
||||
$Limits::Command::line_max_len = $cached_limit;
|
||||
$Limits::Command::line_max_len = tmux_length($cached_limit);
|
||||
if($opt::max_chars) {
|
||||
if($opt::max_chars <= $cached_limit) {
|
||||
$Limits::Command::line_max_len = $opt::max_chars;
|
||||
|
|
@ -7822,6 +7883,48 @@ sub is_acceptable_command_line_length {
|
|||
return not $?;
|
||||
}
|
||||
|
||||
sub tmux_length {
|
||||
# If $opt::tmux set, find the limit for tmux
|
||||
# tmux 1.8 has a 2kB limit
|
||||
# tmux 1.9 has a 16kB limit
|
||||
# Input:
|
||||
# $len = maximal command line length
|
||||
# Returns:
|
||||
# $tmux_len = maximal length runable in tmux
|
||||
my $len = shift;
|
||||
if($opt::tmux) {
|
||||
$ENV{'TMUX'} ||= "tmux";
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb");
|
||||
$Global::unlink{$tmpfile}=1;
|
||||
close $fh;
|
||||
unlink $tmpfile;
|
||||
my $b2020 = "x"x2020;
|
||||
my $b16320 = "x"x16320;
|
||||
my $b100000 = "x"x100000;
|
||||
my $blen = "x"x$len;
|
||||
my @out = qx{
|
||||
sleep .2;
|
||||
echo 1;
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 2020$b2020 2>/dev/null && echo 2020;
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 16320$b16320 2>/dev/null && echo 16320;
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 100000$b100000 2>/dev/null && echo 100000;
|
||||
},
|
||||
# $len can be > 131072 which will confuse bash, so run it on its own
|
||||
qx{
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo $len$blen 2>/dev/null && echo $len;
|
||||
};
|
||||
unlink $tmpfile;
|
||||
::debug("tmux","tmux-length ",@out);
|
||||
chomp @out;
|
||||
# The arguments is given 3 times on the command line
|
||||
# and the wrapping is around 50 chars
|
||||
my $tmux_len = (::max(@out));
|
||||
$len = ::min($len,int($tmux_len/4-28));
|
||||
::debug("tmux","tmux-length ",$len);
|
||||
}
|
||||
return $len;
|
||||
}
|
||||
|
||||
|
||||
package RecordQueue;
|
||||
|
||||
|
|
@ -8608,4 +8711,5 @@ sub mkdir_or_die {
|
|||
|
||||
# Keep perl -w happy
|
||||
$opt::ctrlc = $opt::x = $Semaphore::timeout = $Semaphore::wait =
|
||||
$opt::ignored_option = $Job::file_descriptor_warning_printed = 0;
|
||||
$opt::ignored_option = $Job::file_descriptor_warning_printed =
|
||||
$Global::envdef = 0;
|
||||
|
|
|
|||
|
|
@ -3647,7 +3647,7 @@ B<xargs> deals badly with special characters (such as space, ' and
|
|||
mkdir -p "My brother's 12\" records"
|
||||
ls | xargs rmdir
|
||||
|
||||
You can specify B<-0> or B<-d "\n">, but many input generators are not
|
||||
You can specify B<-0>, but many input generators are not
|
||||
optimized for using B<NUL> as separator but are optimized for
|
||||
B<newline> as separator. E.g B<head>, B<tail>, B<awk>, B<ls>, B<echo>,
|
||||
B<sed>, B<tar -v>, B<perl> (B<-0> and \0 instead of \n), B<locate>
|
||||
|
|
@ -4356,15 +4356,15 @@ When using GNU B<parallel> for a publication please cite:
|
|||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login:
|
||||
The USENIX Magazine, February 2011:42-47.
|
||||
|
||||
Alternatively you can get GNU Parallel without this requirement by
|
||||
paying 10000 EUR.
|
||||
This helps funding further development; and it won't cost you a cent.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2010,2011,2012,2013,2014 Ole Tange, http://ole.tange.dk
|
||||
and Free Software Foundation, Inc.
|
||||
Copyright (C) 2010,2011,2012,2013,2014,2015 Ole Tange,
|
||||
http://ole.tange.dk and Free Software Foundation, Inc.
|
||||
|
||||
Parts of the manual concerning B<xargs> compatibility is inspired by
|
||||
the manual of B<xargs> from GNU findutils 4.4.2.
|
||||
|
|
@ -4372,8 +4372,8 @@ the manual of B<xargs> from GNU findutils 4.4.2.
|
|||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Free Software Foundation,
|
||||
Inc.
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
|
||||
Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
|
|
|||
|
|
@ -83,6 +83,38 @@ version is >= 3.1.0 then B<--protocol 30> is added to force newer
|
|||
B<rsync>s to talk to version 2.5.7.
|
||||
|
||||
|
||||
=head2 Compression
|
||||
|
||||
B<--compress> compresses the data in the temporary files. This is a
|
||||
bit tricky because there should be no files to clean up if GNU
|
||||
B<parallel> is killed by a power outage.
|
||||
|
||||
GNU B<parallel> first selects a compress program. If the user has not
|
||||
selected one, the first of these that are in $PATH is used: B<lzop
|
||||
pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2>. They are sorted by
|
||||
speed on a 8 core machine.
|
||||
|
||||
Schematically the setup is as follows:
|
||||
|
||||
command started by parallel | compress > tmpfile
|
||||
cattail tmpfile | uncompress | parallel
|
||||
|
||||
The setup is duplicated for both standard output (stdout) and standard
|
||||
error (stderr).
|
||||
|
||||
GNU B<parallel> pipes output from the command run into the compress
|
||||
program which saves to a tmpfile. GNU B<parallel> records the pid of
|
||||
the compress program. At the same time a small perl script (called
|
||||
B<cattail> above) is started: It basically does B<cat> followed by
|
||||
B<tail -f>, but it also removes the tmpfile as soon as the first byte
|
||||
is read, and it continously checks if the pid of the compress program
|
||||
is dead. If the compress program is dead, B<cattail> reads the rest of
|
||||
tmpfile and exits.
|
||||
|
||||
As most compress programs write out a header when they start, the
|
||||
tmpfile in practice is unlinked after around 40 ms.
|
||||
|
||||
|
||||
=head2 Wrapping
|
||||
|
||||
The command given by the user can be wrapped in multiple
|
||||
|
|
@ -166,12 +198,20 @@ shell is B<csh> (which cannot hide stderr).
|
|||
|
||||
=item --tmux
|
||||
|
||||
mkfifo I<tmpfile>; tmux new-session -s pI<PID> -d -n <<shell quoted input>> \( <<shell quoted input>> \)\;\(echo\ \$\?\$status\;echo\ 255\)\ \>I<tmpfile>\&echo\ <<shell double quoted input>>\;echo\ \Job\ finished\ at:\ \`date\`\;sleep\ 10; exit `perl -ne 'unlink $ARGV; 1..1 and print' I<tmpfile>`
|
||||
|
||||
The input is used as the name of the windows in B<tmux>. To get the
|
||||
exit value out from B<tmux> a fifo is used. This fifo is being opened
|
||||
by perl and the first value read is used as exit value. Works in
|
||||
B<csh>.
|
||||
mkfifo I<tmpfile>; tmux new-session -s pI<PID> -d -n <<shell quoted input>> \(<<shell quoted input>>\)\;\ perl\ -e\ \'while\(\$t++\<3\)\{\ print\ \$ARGV\[0\],\"\\n\"\ \}\'\ \$\?h/\$status/255\ \>\>\ I<tmpfile>\&echo\ <<shell double quoted input>>\;echo\ \Job\ finished\ at:\ \`date\`\;sleep\ 10; exec perl -e '$/="/";$_=<>;$c=<>;unlink $ARGV; /(\d+)h/ and exit($1);exit$c' I<tmpfile>
|
||||
|
||||
The input is used as the name of the windows in B<tmux>. When the job
|
||||
inside B<tmux> finishes, the exit value is printed to a fifo. This
|
||||
fifo is opened by perl outside B<tmux>, and perl then removes the fifo
|
||||
(but keeping it open). Perl blocks until the first value is read from
|
||||
the fifo, and this value is used as exit value.
|
||||
|
||||
To make it compatible with B<csh> and B<bash> the exit value is
|
||||
printed as: $?h/$status/255 and this is parsed by perl.
|
||||
|
||||
There is a bug that makes it necessary to print the exit value 3
|
||||
times. Works in B<csh>.
|
||||
|
||||
=back
|
||||
|
||||
|
|
@ -249,10 +289,11 @@ stderr. The wrapper looks like this:
|
|||
|
||||
=head2 Transferring of variables and functions
|
||||
|
||||
Transferring of variables and functions is done by running a Perl
|
||||
script before running the actual command. The Perl script sets
|
||||
$ENV{variable} to the correct value before exec'ing the a shell that
|
||||
runs the function definition followed by the actual command.
|
||||
Transferring of variables and functions given by B<-env> is done by
|
||||
running a Perl script remotely that calls the actual command. The Perl
|
||||
script sets $ENV{variable} to the correct value before exec'ing the a
|
||||
shell that runs the function definition followed by the actual
|
||||
command.
|
||||
|
||||
B<env_parallel> (mentioned in the man page) copies the full current
|
||||
environment into the environment variable
|
||||
|
|
@ -317,7 +358,7 @@ B<bash> it will use B<bash>. It does this by looking at the
|
|||
this shell; otherwise look at the parent of this (grand*)parent. If
|
||||
none of the (grand*)parents are shells, then $SHELL is used.
|
||||
|
||||
This will do the right thing in most cases. If called from:
|
||||
This will do the right thing if called from:
|
||||
|
||||
=over 2
|
||||
|
||||
|
|
@ -331,11 +372,11 @@ a shell script
|
|||
|
||||
=item *
|
||||
|
||||
a Perl script in `` or using B<system> if called as a single string
|
||||
a Perl script in `` or using B<system> if called as a single string.
|
||||
|
||||
=back
|
||||
|
||||
But there are situations where it will fail:
|
||||
While these cover most cases, there are situations where it will fail:
|
||||
|
||||
#!/usr/bin/perl
|
||||
|
||||
|
|
@ -343,7 +384,8 @@ But there are situations where it will fail:
|
|||
|
||||
Here it depends on which shell is used to call the Perl script. If the
|
||||
Perl script is called from B<tcsh> it will work just fine, but if it
|
||||
is called from B<bash> it will fail.
|
||||
is called from B<bash> it will fail, because the command B<setenv> is
|
||||
not known to B<bash>.
|
||||
|
||||
|
||||
=head2 Quoting
|
||||
|
|
@ -352,8 +394,9 @@ Quoting is kept simple: Use \ for all special chars and ' for
|
|||
newline. Whether a char is special depends on the shell and the
|
||||
context. Luckily quoting a bit too many does not break things.
|
||||
|
||||
It is fast, but had the distinct disadvantage that if at string needs
|
||||
to be quoted multiple times, the \'s double every time.
|
||||
It is fast, but had the distinct disadvantage that if a string needs
|
||||
to be quoted multiple times, the \'s double every time - increasing
|
||||
the string length exponentially.
|
||||
|
||||
|
||||
=head2 --pipepart vs. --pipe
|
||||
|
|
@ -366,8 +409,8 @@ With B<--pipe> GNU B<parallel> reads the blocks from standard input
|
|||
so every block is being processed by GNU B<parallel> itself. This is
|
||||
the reason why B<--pipe> maxes out at around 100 MB/sec.
|
||||
|
||||
B<--pipepart> on the other hand first identifies at which byte
|
||||
position blocks starts and how long they are. It does that by seeking
|
||||
B<--pipepart>, on the other hand, first identifies at which byte
|
||||
positions blocks start and how long they are. It does that by seeking
|
||||
into the file by the size of a block and then reading until it meets
|
||||
end of a block. The seeking explains why GNU B<parallel> does not know
|
||||
the line number and why B<-L/-l> and B<-N> do not work.
|
||||
|
|
@ -391,7 +434,7 @@ It delivers 1 GB/s per core.
|
|||
|
||||
Instead of the script B<dd> was tried, but many versions of B<dd> do
|
||||
not support reading from one byte to another and might cause partial
|
||||
data:
|
||||
data. See this for a surprising example:
|
||||
|
||||
yes | dd bs=1024k count=10 | wc
|
||||
|
||||
|
|
@ -424,8 +467,8 @@ are no visible files on it.
|
|||
|
||||
GNU B<parallel> buffers on disk. If the disk is full data may be
|
||||
lost. To check if the disk is full GNU B<parallel> writes a 8193 byte
|
||||
file when a job finishes. If this file is written succesfully, it is
|
||||
removed immediately. If it is not written succesfully, the disk is
|
||||
file when a job finishes. If this file is written successfully, it is
|
||||
removed immediately. If it is not written successfully, the disk is
|
||||
full. The size 8193 was chosen because 8192 gave wrong result on some
|
||||
file systems, whereas 8193 did the correct thing on all tested
|
||||
filesystems.
|
||||
|
|
@ -433,7 +476,7 @@ filesystems.
|
|||
|
||||
=head2 Perl replacement strings, {= =}, and --rpl
|
||||
|
||||
The shorthands for replacement strings makes a command look more
|
||||
The shorthands for replacement strings make a command look more
|
||||
cryptic. Different users will need different replacement
|
||||
strings. Instead of inventing more shorthands you get more more
|
||||
flexible replacement strings if they can be programmed by the user.
|
||||
|
|
@ -468,9 +511,9 @@ dependent on how long a given test runs (e.g. more than 10 secs is a
|
|||
pass, but less is a fail). It parallelizes most tests, but it is easy
|
||||
to force a test to run as the single test (which may be important for
|
||||
timing issues). It deals reasonably well with tests that fail
|
||||
intermittently. It detects which tests that failed and pushes these to
|
||||
the top, so when running the test suite again, the tests that failed
|
||||
most recently are run first.
|
||||
intermittently. It detects which tests failed and pushes these to the
|
||||
top, so when running the test suite again, the tests that failed most
|
||||
recently are run first.
|
||||
|
||||
If GNU B<parallel> should adopt a real testing framework then those
|
||||
elements would be important.
|
||||
|
|
@ -486,7 +529,7 @@ sometimes. One of the harder problems was to make a machine start
|
|||
swapping without forcing it to its knees.
|
||||
|
||||
|
||||
=head2 Median runtime
|
||||
=head2 Median run time
|
||||
|
||||
Using a percentage for B<--timeout> causes GNU B<parallel> to compute
|
||||
the median run time of a job. The median is a better indicator of the
|
||||
|
|
@ -523,10 +566,10 @@ Unfortunately it is not always possible to predict the root cause of the error.
|
|||
|
||||
=head2 Computation of load
|
||||
|
||||
Contrary to the obvious --load does not use load average. This is due
|
||||
to load average rising too slowly. Instead it uses B<ps> to list the
|
||||
number of jobs in running or blocked state (state D, O or R). This
|
||||
gives an instat load.
|
||||
Contrary to the obvious B<--load> does not use load average. This is
|
||||
due to load average rising too slowly. Instead it uses B<ps> to list
|
||||
the number of jobs in running or blocked state (state D, O or R). This
|
||||
gives an instant load.
|
||||
|
||||
As remote calculation of load can be slow, a process is spawned to run
|
||||
B<ps> and put the result in a file, which is then used next time.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
stdout parallel -i -s28 -0 true from \{\} to x{}y < items-0.xi |egrep -v 'exit|Command|\.\.\.'
|
||||
stdout parallel -i -s28 -0 true from \{\} to x{}y < items-0.xi | egrep -v 'exit|Command|\.\.\.' | grep .
|
||||
|
||||
# Fejler
|
||||
#stdout parallel -Di -s28 -0 true from \{\} to x{}y < items-0.xi > /dev/null
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### Test if we can deal with output > 4 GB'
|
||||
## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
|
||||
echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
|
||||
echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | nice md5sum
|
||||
|
||||
echo '**'
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ echo '### Special char file and dir transfer return and cleanup'
|
|||
};
|
||||
export -f mytouch;
|
||||
parallel --env mytouch -Sparallel@lo --transfer
|
||||
--return {=s:/f:/g:=}
|
||||
--return {=s:/f:/g:=}
|
||||
mytouch
|
||||
::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
|
||||
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
|
||||
|
|
|
|||
|
|
@ -24,4 +24,34 @@ echo '### csh'
|
|||
setenv C `seq 300 -2 1|xargs`;
|
||||
parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
|
||||
|
||||
echo '### Test tmux works on different shells'
|
||||
parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?
|
||||
parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?
|
||||
|
||||
export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux ';
|
||||
ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status';
|
||||
ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status';
|
||||
ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?';
|
||||
ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?';
|
||||
ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status';
|
||||
ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status'
|
||||
|
||||
echo '### TODO This fails - word too long'
|
||||
export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux ';
|
||||
stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output';
|
||||
stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'
|
||||
|
||||
echo '### works'
|
||||
parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
echo '### These blocked due to length'
|
||||
parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Scsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Stcsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Szsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Scsh@lo --tmux echo ::: 111111111111111111111111111111111111111111111111111111111
|
||||
|
||||
|
||||
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -1,2 +1,69 @@
|
|||
#!/bin/bash
|
||||
|
||||
par_tmux_filter() {
|
||||
perl -pe 's/par......tms/parXXXXX.tms/;s/ p\d+/pID/;'
|
||||
}
|
||||
export -f par_tmux_filter
|
||||
|
||||
par_tmux() {
|
||||
(stdout parallel --timeout 3 --tmux --delay .3 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter
|
||||
}
|
||||
export -f par_tmux
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### tmux1.9'
|
||||
seq 000 100 | TMUX=tmux1.9 par_tmux
|
||||
seq 100 200 | TMUX=tmux1.9 par_tmux
|
||||
seq 200 300 | TMUX=tmux1.9 par_tmux
|
||||
seq 300 400 | TMUX=tmux1.9 par_tmux
|
||||
seq 400 500 | TMUX=tmux1.9 par_tmux
|
||||
seq 500 600 | TMUX=tmux1.9 par_tmux
|
||||
seq 600 700 | TMUX=tmux1.9 par_tmux
|
||||
seq 700 800 | TMUX=tmux1.9 par_tmux
|
||||
seq 800 900 | TMUX=tmux1.9 par_tmux
|
||||
seq 900 1000 | TMUX=tmux1.9 par_tmux
|
||||
seq 1000 1100 | TMUX=tmux1.9 par_tmux
|
||||
seq 1100 1200 | TMUX=tmux1.9 par_tmux
|
||||
seq 1200 1300 | TMUX=tmux1.9 par_tmux
|
||||
seq 1300 1400 | TMUX=tmux1.9 par_tmux
|
||||
seq 1400 1500 | TMUX=tmux1.9 par_tmux
|
||||
seq 1500 1600 | TMUX=tmux1.9 par_tmux
|
||||
seq 1600 1700 | TMUX=tmux1.9 par_tmux
|
||||
seq 1700 1800 | TMUX=tmux1.9 par_tmux
|
||||
seq 1800 1900 | TMUX=tmux1.9 par_tmux
|
||||
seq 1900 2000 | TMUX=tmux1.9 par_tmux
|
||||
seq 2000 2021 | TMUX=tmux1.9 par_tmux
|
||||
echo '### tmux1.9 fails'
|
||||
echo 2022 | TMUX=tmux1.9 par_tmux
|
||||
|
||||
echo '### tmux1.8'
|
||||
seq 1 50 | TMUX=tmux1.8 par_tmux
|
||||
seq 51 100 | TMUX=tmux1.8 par_tmux
|
||||
seq 101 150 | TMUX=tmux1.8 par_tmux
|
||||
seq 151 200 | TMUX=tmux1.8 par_tmux
|
||||
seq 201 233 | TMUX=tmux1.8 par_tmux
|
||||
echo '### tmux1.8 fails'
|
||||
echo 234 | TMUX=tmux1.8 par_tmux
|
||||
echo 235 | TMUX=tmux1.8 par_tmux
|
||||
echo 236 | TMUX=tmux1.8 par_tmux
|
||||
|
||||
echo '### tmux1.8 0..255 ascii'
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' |
|
||||
TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
|
||||
|
||||
echo '### tmux1.9 0..255 ascii'
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' |
|
||||
TMUX=tmux1.9 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
|
||||
|
||||
echo '### Test output ascii'
|
||||
rm -f /tmp/paralocal7*;
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter;
|
||||
sort /tmp/paralocal7* | md5sum
|
||||
|
||||
echo '### Test critical lengths. Must not block'
|
||||
seq 70 130 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
|
||||
seq 70 130 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
|
||||
seq 280 425 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin'
|
|||
|
||||
echo '### Test --load locally - should take >10s'
|
||||
echo '# This will run 10 processes in parallel for 10s';
|
||||
seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||
seq 10 | parallel --nice 19 --timeout 13 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
sleep 2; stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||
|
||||
echo '### Test --load remote'
|
||||
ssh parallel@$SERVER2 'seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null"' &
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
echo '### Test if we can deal with output > 4 GB'
|
||||
### Test if we can deal with output > 4 GB
|
||||
## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
|
||||
echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
|
||||
echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | nice md5sum
|
||||
46a318993dfc8e2afd71ff2bc6f605f1 -
|
||||
echo '**'
|
||||
**
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ echo '### trailing space in sshlogin'
|
|||
sshlogin trailing space
|
||||
echo '### Special char file and dir transfer return and cleanup'
|
||||
### Special char file and dir transfer return and cleanup
|
||||
cd /tmp; mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; ssh parallel@lo rm -rf d'*'/; mytouch() { cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; }; export -f mytouch; parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
|
||||
cd /tmp; mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; ssh parallel@lo rm -rf d'*'/; mytouch() { cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; }; export -f mytouch; parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
|
||||
local
|
||||
remote OK
|
||||
echo '### Uniq {=perlexpr=} in return - not used in command'
|
||||
|
|
|
|||
|
|
@ -18,3 +18,41 @@ echo '### csh'
|
|||
1 200 692
|
||||
1 200 692
|
||||
1 150 547
|
||||
echo '### Test tmux works on different shells'
|
||||
### Test tmux works on different shells
|
||||
parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?
|
||||
0
|
||||
parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?
|
||||
4
|
||||
export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status'; ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status'; ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?'; ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?'; ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status'; ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status'
|
||||
0
|
||||
4
|
||||
0
|
||||
4
|
||||
0
|
||||
4
|
||||
echo '### TODO This fails - word too long'
|
||||
### TODO This fails - word too long
|
||||
export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'
|
||||
Word too long.
|
||||
Word too long.
|
||||
Word too long.
|
||||
Word too long.
|
||||
4
|
||||
Word too long.
|
||||
Word too long.
|
||||
Word too long.
|
||||
Word too long.
|
||||
4
|
||||
echo '### works'
|
||||
### works
|
||||
parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
parallel: Error: Command line too long (402 >= 238) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...
|
||||
echo '### These blocked due to length'
|
||||
### These blocked due to length
|
||||
parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Scsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Stcsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Szsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@
|
||||
parallel -Scsh@lo --tmux echo ::: 111111111111111111111111111111111111111111111111111111111
|
||||
|
|
|
|||
|
|
@ -366,7 +366,7 @@ echo '### Test --seqreplace and line too long'
|
|||
seq 1 1000 | stdout parallel -j1 -s 210 -k --seqreplace I echo IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII \|wc | uniq -c
|
||||
9 1 1 101
|
||||
90 1 1 201
|
||||
1 parallel: Error: Command line too long (309 >= 210) at number 0: 100...
|
||||
1 parallel: Error: Command line too long (309 >= 210) at input 0: 100
|
||||
echo '### bug #37042: -J foo is taken from the whole command line - not just the part before the command'
|
||||
### bug #37042: -J foo is taken from the whole command line - not just the part before the command
|
||||
echo '--tagstring foo' > ~/.parallel/bug_37042_profile; parallel -J bug_37042_profile echo ::: tag_with_foo; parallel --tagstring a -J bug_37042_profile echo ::: tag_with_a; parallel --tagstring a echo -J bug_37042_profile ::: print_-J_bug_37042_profile
|
||||
|
|
|
|||
|
|
@ -86,8 +86,8 @@ stdout xargs -i -s26 -0 echo from \{\} to x{}y < items-0.xi
|
|||
xargs: argument list too long
|
||||
from one to xoney
|
||||
stdout parallel -k -i -s26 -0 echo from \{\} to x{}y < items-0.xi
|
||||
parallel: Error: Command line too long (42 >= 26) at number 0:
|
||||
...
|
||||
parallel: Error: Command line too long (42 >= 26) at input 0:
|
||||
|
||||
echo '### -l -0 echo < ldata-0.xi'
|
||||
### -l -0 echo < ldata-0.xi
|
||||
stdout xargs -l -0 echo < ldata-0.xi
|
||||
|
|
@ -998,7 +998,7 @@ echo '### -s6 echo < files.xi'
|
|||
stdout xargs -s6 echo < files.xi
|
||||
xargs: argument line too long
|
||||
stdout parallel -k -X -s6 echo < files.xi
|
||||
parallel: Error: Command line too long (27 >= 6) at number -7: /src/gnu/autoconf-1.11...
|
||||
parallel: Error: Command line too long (27 >= 6) at input -7: /src/gnu/autoconf-1.11
|
||||
echo '### -iARG -s86 echo ARG is xARGx < files.xi'
|
||||
### -iARG -s86 echo ARG is xARGx < files.xi
|
||||
stdout xargs -iARG -s86 echo ARG is xARGx < files.xi
|
||||
|
|
@ -1209,7 +1209,7 @@ stdout xargs -i -s26 echo from \{\} to x{}y < items.xi
|
|||
xargs: argument list too long
|
||||
from dumb to xdumby
|
||||
stdout parallel -k -i -s26 echo from \{\} to x{}y < items.xi
|
||||
parallel: Error: Command line too long (36 >= 26) at number 0:
...
|
||||
parallel: Error: Command line too long (36 >= 26) at input 0:
|
||||
echo '### -i__ echo FIRST __ IS OK < quotes.xi'
|
||||
### -i__ echo FIRST __ IS OK < quotes.xi
|
||||
stdout xargs -i__ echo FIRST __ IS OK < quotes.xi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
echo '### tmux1.9'
|
||||
### tmux1.9
|
||||
seq 000 100 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 100 200 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 200 300 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 300 400 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 400 500 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 500 600 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 600 700 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 700 800 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 800 900 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 900 1000 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1000 1100 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1100 1200 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1200 1300 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1300 1400 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1400 1500 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1500 1600 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1600 1700 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1700 1800 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1800 1900 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 1900 2000 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 2000 2021 | TMUX=tmux1.9 par_tmux
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
echo '### tmux1.9 fails'
|
||||
### tmux1.9 fails
|
||||
echo 2022 | TMUX=tmux1.9 par_tmux
|
||||
parallel: Error: Command line too long (4053 >= 4052) at input 0: 2022
|
||||
255
|
||||
echo '### tmux1.8'
|
||||
### tmux1.8
|
||||
seq 1 50 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 51 100 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 101 150 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 151 200 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
seq 201 233 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
echo '### tmux1.8 fails'
|
||||
### tmux1.8 fails
|
||||
echo 234 | TMUX=tmux1.8 par_tmux
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
command too long
|
||||
1
|
||||
echo 235 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (478 >= 477) at input 0: 235
|
||||
255
|
||||
echo 236 | TMUX=tmux1.8 par_tmux
|
||||
parallel: Error: Command line too long (480 >= 477) at input 0: 236
|
||||
255
|
||||
echo '### tmux1.8 0..255 ascii'
|
||||
### tmux1.8 0..255 ascii
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
echo '### tmux1.9 0..255 ascii'
|
||||
### tmux1.9 0..255 ascii
|
||||
perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | TMUX=tmux1.9 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
0
|
||||
echo '### Test output ascii'
|
||||
### Test output ascii
|
||||
rm -f /tmp/paralocal7*; perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; sort /tmp/paralocal7* | md5sum
|
||||
See output with: tmux -S /tmp/parXXXXX.tms attach -tpID
|
||||
a7ee232967c8aab2edf227169e8cdce0 -
|
||||
echo '### Test critical lengths. Must not block'
|
||||
### Test critical lengths. Must not block
|
||||
seq 70 130 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
seq 70 130 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
seq 280 425 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.8 -S /tmp/parXXXXX.tms attach -tpID
|
||||
seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
See output with: tmux1.9 -S /tmp/parXXXXX.tms attach -tpID
|
||||
|
|
@ -207,11 +207,13 @@ With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
|
|||
See 'man parallel' for details
|
||||
|
||||
Academic tradition requires you to cite works you base your article on.
|
||||
When using programs that use GNU Parallel to process data for publication please cite:
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite:
|
||||
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
|
||||
This helps funding further development; and it won't cost you a cent.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed.
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ e
|
|||
echo '### test too long args'
|
||||
### test too long args
|
||||
perl -e 'print "z"x1000000' | parallel echo 2>&1
|
||||
parallel: Error: Command line too long (1000005 >= 131071) at number 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
parallel: Error: Command line too long (1000005 >= 131071) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
perl -e 'print "z"x1000000' | xargs echo 2>&1
|
||||
xargs: argument line too long
|
||||
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo
|
||||
|
|
@ -161,7 +161,7 @@ xargs: argument line too long
|
|||
5 6
|
||||
7 8
|
||||
9 10
|
||||
parallel: Error: Command line too long (1000007 >= 10) at number 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout xargs -s 10 echo
|
||||
1 2
|
||||
3 4
|
||||
|
|
@ -175,7 +175,7 @@ xargs: argument line too long
|
|||
5 6
|
||||
7 8
|
||||
9 10
|
||||
parallel: Error: Command line too long (1000007 >= 10) at number 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
|
||||
echo '### Test -x'
|
||||
### Test -x
|
||||
(seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo
|
||||
|
|
@ -184,14 +184,14 @@ echo '### Test -x'
|
|||
5 6
|
||||
7 8
|
||||
9 10
|
||||
parallel: Error: Command line too long (10 >= 10) at number 0: 12345...
|
||||
parallel: Error: Command line too long (10 >= 10) at input 0: 12345
|
||||
(seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -kX -s 10 -x echo
|
||||
1 2
|
||||
3 4
|
||||
5 6
|
||||
7 8
|
||||
9 10
|
||||
parallel: Error: Command line too long (10 >= 10) at number 0: 12345...
|
||||
parallel: Error: Command line too long (10 >= 10) at input 0: 12345
|
||||
(seq 1 10; echo 12345; seq 12 15) | stdout xargs -s 10 -x echo
|
||||
1 2
|
||||
3 4
|
||||
|
|
@ -444,9 +444,9 @@ echo '### Test --help and -h: Help output (just check we get the same amount of
|
|||
echo Output from -h and --help
|
||||
Output from -h and --help
|
||||
parallel -h | wc -l
|
||||
35
|
||||
37
|
||||
parallel --help | wc -l
|
||||
35
|
||||
37
|
||||
echo '### Test --version: Version output (just check we get the same amount of lines)'
|
||||
### Test --version: Version output (just check we get the same amount of lines)
|
||||
parallel --version | wc -l
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin'
|
|||
seq 1 150 | stdout nice parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null"
|
||||
echo '### Test --load locally - should take >10s'
|
||||
### Test --load locally - should take >10s
|
||||
echo '# This will run 10 processes in parallel for 10s'; seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
echo '# This will run 10 processes in parallel for 10s'; seq 10 | parallel --nice 19 --timeout 13 -j0 -N0 "gzip < /dev/zero > /dev/null" &
|
||||
# This will run 10 processes in parallel for 10s
|
||||
stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||
sleep 2; stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||
OK
|
||||
echo '### Test --load remote'
|
||||
### Test --load remote
|
||||
|
|
|
|||
Loading…
Reference in a new issue