parallel: Initial version of --tmux.

This commit is contained in:
Ole Tange 2014-07-16 11:07:41 +02:00
parent 1cf521971f
commit 1a1c0ebe0f
9 changed files with 73 additions and 13 deletions

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
*~ *~
*.pdf
Makefile Makefile
config.h config.h
config.log config.log

24
NEWS
View file

@ -8,19 +8,29 @@
* {%} works as job slot. * {%} works as job slot.
* GNU Parallel was cited in: Energy Efficient, High-speed Communication in WSNs https://gupea.ub.gu.se/bitstream/2077/35801/1/gupea_2077_35801_1.pdf * GNU Parallel was cited in: Energy Efficient, High-speed
Communication in WSNs
https://gupea.ub.gu.se/bitstream/2077/35801/1/gupea_2077_35801_1.pdf
* GNU Parallel was cited in: Ferroelectric contributions to anomalous hysteresis in hybrid perovskite solar cells http://arxiv.org/pdf/1405.5810.pdf * GNU Parallel was cited in: Ferroelectric contributions to anomalous
hysteresis in hybrid perovskite solar cells
http://arxiv.org/pdf/1405.5810.pdf
* Processes Paralleling to Speed up Computing and Tasks Execution in Linux http://kukuruku.co/hub/nix/processes-paralleling-to-speed-up-computing-and-tasks-execution-in-linux * Processes Paralleling to Speed up Computing and Tasks Execution in
Linux
http://kukuruku.co/hub/nix/processes-paralleling-to-speed-up-computing-and-tasks-execution-in-linux
* Moving / Copying lots of s3 files quickly using gnu parallel http://blog.aclarke.eu/moving-copying-lots-of-s3-files-quickly-using-gnu-parallel/ * Moving / Copying lots of s3 files quickly using gnu parallel
http://blog.aclarke.eu/moving-copying-lots-of-s3-files-quickly-using-gnu-parallel/
* Speeding up grep log queries with GNU Parallel http://www.tripwire.com/state-of-security/incident-detection/speeding-grep-queries-gnu-parallel/ * Speeding up grep log queries with GNU Parallel
http://www.tripwire.com/state-of-security/incident-detection/speeding-grep-queries-gnu-parallel/
* Ubuntu - Gnu parallel - It's awesome http://kasunweranga.blogspot.dk/2014/06/ubuntu-gnu-parallel-its-awesome.html * Ubuntu - Gnu parallel - It's awesome
http://kasunweranga.blogspot.dk/2014/06/ubuntu-gnu-parallel-its-awesome.html
* Export quicklooks with vector overlay using TuiView http://spectraldifferences.wordpress.com/2014/03/08/export-quicklooks-with-vector-overlay-using-tuiview/ * Export quicklooks with vector overlay using TuiView
http://spectraldifferences.wordpress.com/2014/03/08/export-quicklooks-with-vector-overlay-using-tuiview/
* GNU Parallel指南 http://my.oschina.net/enyo/blog/271612 * GNU Parallel指南 http://my.oschina.net/enyo/blog/271612

View file

@ -237,7 +237,7 @@ New in this release:
* Positional perl expressions (similar to {2}) are given as {=2 perl expression=} where 2 is the position. * Positional perl expressions (similar to {2}) are given as {=2 perl expression=} where 2 is the position.
* Replacement strings can be defined using --rpl. E.g. parallel --rpl {.gz} 's/\.gz$//' echo {.gz} ::: *.gz * Replacement strings can be defined using --rpl. E.g. parallel --rpl '{.gz} s/\.gz$//' echo {.gz} ::: *.gz
* The parenthesis around {= perl expression =} can be changed with --parens. * The parenthesis around {= perl expression =} can be changed with --parens.
@ -251,6 +251,10 @@ New in this release:
* Webcast at 2014-08-20 covering GNU Parallel: Data Science at the Command Line http://www.oreilly.com/pub/e/3115 * Webcast at 2014-08-20 covering GNU Parallel: Data Science at the Command Line http://www.oreilly.com/pub/e/3115
* Shell command composition and dispatch http://lukeluo.blogspot.dk/2014/07/linux-virtual-console6-shell-command.html
* Parallelising plink (or anything else) the easy way http://chrisladroue.com/2012/03/parallelising-plink-or-anything-else-the-easy-way/
* Paralelizace běžných činností v konzoli pomocí GNU Parallel http://www.abclinuxu.cz/clanky/paralelizace-beznych-cinnosti-v-konzoli-pomoci-gnu-parallel * Paralelizace běžných činností v konzoli pomocí GNU Parallel http://www.abclinuxu.cz/clanky/paralelizace-beznych-cinnosti-v-konzoli-pomoci-gnu-parallel
* [原] Ubuntu 下使用 parallel 命令的注意事项 http://blog.ailms.me/2014/06/28/ubuntu-with-parallel.html * [原] Ubuntu 下使用 parallel 命令的注意事项 http://blog.ailms.me/2014/06/28/ubuntu-with-parallel.html

Binary file not shown.

View file

@ -106,7 +106,6 @@ if($opt::header and not $opt::pipe) {
} }
} }
if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
# Parallel check all hosts are up. Remove hosts that are down # Parallel check all hosts are up. Remove hosts that are down
filter_hosts(); filter_hosts();
@ -567,6 +566,7 @@ sub options_hash {
"g" => \$opt::retired, "g" => \$opt::retired,
"ungroup|u" => \$opt::u, "ungroup|u" => \$opt::u,
"linebuffer|linebuffered|line-buffer|line-buffered" => \$opt::linebuffer, "linebuffer|linebuffered|line-buffer|line-buffered" => \$opt::linebuffer,
"tmux" => \$opt::tmux,
"null|0" => \$opt::0, "null|0" => \$opt::0,
"quote|q" => \$opt::q, "quote|q" => \$opt::q,
# Replacement strings # Replacement strings
@ -2352,6 +2352,8 @@ sub save_original_signal_handler {
# Remember the original signal handler # Remember the original signal handler
# Returns: N/A # Returns: N/A
$SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X $SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X
$SIG{INT} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; } exit -1 };
$SIG{TERM} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; } exit -1 };
%Global::original_sig = %SIG; %Global::original_sig = %SIG;
$SIG{TERM} = sub {}; # Dummy until jobs really start $SIG{TERM} = sub {}; # Dummy until jobs really start
} }
@ -4910,6 +4912,7 @@ sub sshlogin_wrap {
# Propagating CTRL-C to kill remote jobs requires # Propagating CTRL-C to kill remote jobs requires
# remote jobs to be run with a terminal. # remote jobs to be run with a terminal.
$ssh_options = "-tt -oLogLevel=quiet"; $ssh_options = "-tt -oLogLevel=quiet";
# $ssh_options = "";
# tty - check if we have a tty. # tty - check if we have a tty.
# stty: # stty:
# -onlcr - make output 8-bit clean # -onlcr - make output 8-bit clean
@ -4937,7 +4940,7 @@ sub sshlogin_wrap {
$p->mask(STDOUT, POLLHUP); $p->mask(STDOUT, POLLHUP);
$pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"} $pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"}
$p->poll; $p->poll;
kill SIGHUP, -$pid unless $done; kill SIGHUP, -${pid} unless $done;
wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8) wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8)
} . "' "; } . "' ";
$signal_script =~ s/\s+/ /g; $signal_script =~ s/\s+/ /g;
@ -5216,6 +5219,9 @@ sub start {
test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true; test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true;
(cat $tmpfile; rm $tmpfile; cat - ) | } . (cat $tmpfile; rm $tmpfile; cat - ) | } .
"($command);"; "($command);";
if($opt::tmux) {
$command = tmux_wrap($command);
}
# The eval is needed to catch exception from open3 # The eval is needed to catch exception from open3
eval { eval {
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) || $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
@ -5229,6 +5235,9 @@ sub start {
# locally - otherwise CTRL-C does not work for other jobs Bug#36585) # locally - otherwise CTRL-C does not work for other jobs Bug#36585)
*IN = *STDIN; *IN = *STDIN;
# The eval is needed to catch exception from open3 # The eval is needed to catch exception from open3
if($opt::tmux) {
$command = tmux_wrap($command);
}
eval { eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) || $pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-a"); ::die_bug("open3-a");
@ -5242,6 +5251,9 @@ sub start {
# Give /dev/tty to the command if no one else is using it # Give /dev/tty to the command if no one else is using it
*IN = $devtty_fh; *IN = $devtty_fh;
# The eval is needed to catch exception from open3 # The eval is needed to catch exception from open3
if($opt::tmux) {
$command = tmux_wrap($command);
}
eval { eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) || $pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-/dev/tty"); ::die_bug("open3-/dev/tty");
@ -5250,6 +5262,9 @@ sub start {
1; 1;
}; };
} else { } else {
if($opt::tmux) {
$command = tmux_wrap($command);
}
eval { eval {
$pid = ::open3(::gensym, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) || $pid = ::open3(::gensym, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-gensym"); ::die_bug("open3-gensym");
@ -5276,6 +5291,24 @@ sub start {
} }
} }
sub tmux_wrap {
my $command = shift;
# Start tmux seesion as pPID
my ($fh, $tmpfile) = ::tempfile(SUFFIX => ".tmx");
close $fh;
unlink $tmpfile;
my $tmux;
if($Global::total_running == 0) {
$tmux = "tmux new-session -s p$$ -d";
print $Global::original_stderr "See output with: tmux attach -t p$$\n";
} else {
$tmux = "tmux new-window -t p$$";
}
return "mkfifo $tmpfile; $tmux ".
::shell_quote_scalar("(".$command.');echo $?$status >'.$tmpfile).
"; ( exit `cat < $tmpfile;rm $tmpfile` )";
}
sub is_already_in_results { sub is_already_in_results {
# Do we already have results for this job? # Do we already have results for this job?
# Returns: # Returns:

Binary file not shown.

View file

@ -1624,6 +1624,12 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
setting $TMPDIR. setting $TMPDIR.
=item B<--tmux> (alpha testing)
Use B<tmux> for output. Start a B<tmux> session and run each job in a
window in that session. No other output will be produced.
=item B<--timeout> I<val> =item B<--timeout> I<val>
Time out for command. If the command runs for longer than I<val> Time out for command. If the command runs for longer than I<val>

View file

@ -1097,7 +1097,7 @@ The file must be a physical (seekable) file and must be given using @strong{-a}
@item * @item *
@anchor{* 1} @anchor{* 1}
Record counting (@strong{-N}) and line counting (@strong{-L}) do not work. Record counting (@strong{-N}) and line counting (@strong{-L}/@strong{-l}) do not work.
@end table @end table
@ -1806,6 +1806,12 @@ into temporary files in /tmp. By setting @strong{--tmpdir} you can use a
different dir for the files. Setting @strong{--tmpdir} is equivalent to different dir for the files. Setting @strong{--tmpdir} is equivalent to
setting $TMPDIR. setting $TMPDIR.
@item @strong{--tmux} (alpha testing)
@anchor{@strong{--tmux} (alpha testing)}
Use @strong{tmux} for output. Start a @strong{tmux} session and run each job in a
window in that session. No other output will be produced.
@item @strong{--timeout} @emph{val} @item @strong{--timeout} @emph{val}
@anchor{@strong{--timeout} @emph{val}} @anchor{@strong{--timeout} @emph{val}}