From c7cf5b17fd116c7b2e0cbe54aff3a758fc7fe7a1 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 19 Jul 2014 08:53:46 +0200 Subject: [PATCH] parallel: --halt 10% allows up to 10% of jobs to fail (though min. 3). --- doc/release_new_version | 8 ++++++-- src/parallel | 15 +++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 27d44ae1..b78b6f4a 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -237,11 +237,13 @@ New in this release: * Positional perl expressions (similar to {2}) are given as {=2 perl expression=} where 2 is the position. +* One small backwards incompatability: {1}_{2} will replace {2} with the empty string if there is only one argument. Previously {2} would have been left untouched. + * Replacement strings can be defined using --rpl. E.g. parallel --rpl '{.gz} s/\.gz$//' echo {.gz} ::: *.gz * The parenthesis around {= perl expression =} can be changed with --parens. -* --tmux will direct the output to a tmux session instead of files. +* --tmux will direct the output to a tmux session instead of files. Each running jobs will be in its own window. * GNU Parallel was cited in: bammds: A tool for assessing the ancestry of low depth whole genome data using multidimensional scaling (MDS) http://bioinformatics.oxfordjournals.org/content/early/2014/06/28/bioinformatics.btu410.abstract @@ -251,7 +253,7 @@ New in this release: * GNU Parallel was cited in: Web-scale Content Reuse Detection www.isi.edu/~johnh/PAPERS/Ardi14a.pdf -* Webcast at 2014-08-20 covering GNU Parallel: Data Science at the Command Line http://www.oreilly.com/pub/e/3115 +* Tell your friends to sign up for the Webcast at 2014-08-20 covering GNU Parallel: Data Science at the Command Line http://www.oreilly.com/pub/e/3115 * GNU Parallel all the things! http://longwayaround.org.uk/notes/gnu-parallel-all-the-things/ @@ -265,6 +267,8 @@ New in this release: * [原] Ubuntu 下使用 parallel 命令的注意事项 http://blog.ailms.me/2014/06/28/ubuntu-with-parallel.html +* 简单的并行, 可使用GNU parallel http://jerkwin.github.io/2014/06/10/GNU-parallel/ + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel b/src/parallel index 7cad7ca3..c46d1dc3 100755 --- a/src/parallel +++ b/src/parallel @@ -617,7 +617,7 @@ sub options_hash { "compress" => \$opt::compress, "tty" => \$opt::tty, "T" => \$opt::retired, - "halt-on-error|halt=i" => \$opt::halt_on_error, + "halt-on-error|halt=s" => \$opt::halt_on_error, "H=i" => \$opt::retired, "retries=i" => \$opt::retries, "dry-run|dryrun" => \$opt::dryrun, @@ -720,7 +720,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20140714; + $Global::version = 20140719; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -834,6 +834,7 @@ sub parse_options { if(defined $opt::fg) { $Global::semaphore = 1; } if(defined $opt::bg) { $Global::semaphore = 1; } if(defined $opt::wait) { $Global::semaphore = 1; } + if(defined $opt::halt_on_error and $opt::halt_on_error=~/%/) { $opt::halt_on_error /= 100; } if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) { ::error("--timeout must be seconds or percentage\n"); wait_and_exit(255); @@ -2434,9 +2435,15 @@ sub reaper { if($job->exitstatus()) { # The jobs had a exit status <> 0, so error $Global::exitstatus++; + $Global::total_failed++; if($opt::halt_on_error) { - if($opt::halt_on_error == 1) { - # If halt on error == 1 we should gracefully exit + if($opt::halt_on_error == 1 + or + ($opt::halt_on_error < 1 and $Global::total_failed > 1 + and + $Global::total_failed / $Global::total_started > $opt::halt_on_error)) { + # If halt on error == 1 or --halt 10% + # we should gracefully exit print $Global::original_stderr ("$Global::progname: Starting no more jobs. ", "Waiting for ", scalar(keys %Global::running),