From 54200832e43313198053167a6f85926b90d14ef4 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 16 May 2017 01:27:11 +0200 Subject: [PATCH] parallel: --timeout can now use s=second,m=minute,h=hour,d=day. parallel: --dr is now alias for --dry-run. --- doc/release_new_version | 14 ++++++++++++++ src/parallel | 14 +++++++++++--- src/parallel.pod | 11 ++++++++--- testsuite/tests-to-run/parallel-local-100s.sh | 8 ++++++++ testsuite/wanted-results/parallel-local-100s | 4 ++++ 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index bd8ebe28..f78ae7ea 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -214,10 +214,24 @@ New in this release: http://meta.askubuntu.com/a/16750/22307 http://meta.serverfault.com/a/9040/45704 +http://journal.frontiersin.org/article/10.3389/fncom.2017.00021/full#B18 + +https://www.cyberciti.biz/faq/how-to-run-command-or-code-in-parallel-in-bash-shell-under-linux-or-unix/ + https://github.com/jupiter126/Create_Speech_Dataset http://pubs.rsc.org/-/content/articlelanding/2017/cp/c7cp01662j#!divAbstract +http://www.hpc.lsu.edu/training/weekly-materials/2017-Spring/gnuparallel-Feb2017.pdf + +https://joeray.me/gnu-parallel-how-to-list-millions-objects.html + +http://amedee.me/gnu-parallel-is-my-new-toaster/index.html + +http://arjon.es/2017/05/11/which-is-the-best-tool-for-copying-a-large-directory-tree-locally/ + +https://github.com/faroit/dsdtools + ftp://frapftp.fire.ca.gov/outgoing/transfer/PUC_Map1/Utility%20Fire%20map%201%20final%20review%20and%20development%20report21616.pdf https://dzone.com/articles/batch-convert-images-from-png-to-jpeg diff --git a/src/parallel b/src/parallel index 0f1befb1..110e1f47 100755 --- a/src/parallel +++ b/src/parallel @@ -977,7 +977,7 @@ sub options_hash { "tty" => \$opt::tty, "T" => \$opt::retired, "H=i" => \$opt::retired, - "dry-run|dryrun" => \$opt::dryrun, + "dry-run|dryrun|dr" => \$opt::dryrun, "progress" => \$opt::progress, "eta" => \$opt::eta, "bar" => \$opt::bar, @@ -1317,7 +1317,8 @@ sub parse_options { } sub check_invalid_option_combinations { - if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) { + if(defined $opt::timeout and + $opt::timeout !~ /^\d+(\.\d+)?%?$|^(\d+(\.\d+)?[dhms])+$/i) { ::error("--timeout must be seconds or percentage."); wait_and_exit(255); } @@ -1370,7 +1371,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170423; + $Global::version = 20170512; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -10571,6 +10572,13 @@ sub new { $pct = $1/100; $delta_time = 1_000_000; } + if($delta_time =~ /[dhms]/i) { + $delta_time =~ s/s/*1+/gi; + $delta_time =~ s/m/*60+/gi; + $delta_time =~ s/h/*3600+/gi; + $delta_time =~ s/d/*86400+/gi; + $delta_time = eval $delta_time."0"; + } return bless { 'queue' => [], 'delta_time' => $delta_time, diff --git a/src/parallel.pod b/src/parallel.pod index 790b566a..a9161237 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2290,15 +2290,20 @@ Useful if you want to monitor the progress of less than 100 concurrent jobs. -=item B<--timeout> I +=item B<--timeout> I -Time out for command. If the command runs for longer than I +Time out for command. If the command runs for longer than I seconds it will get killed as per B<--termseq>. -If I is followed by a % then the timeout will dynamically be +If I is followed by a % then the timeout will dynamically be computed as a percentage of the median average runtime of successful jobs. Only values > 100% will make sense. +I is normally in seconds, but can be floats postfixed with +s, m, h, or d which would multiply the float by 1, 60, 3600, or +86400. Thus these are equivalent: B<--timeout 100000> and B<--timeout +1d3h46.6m4s>. + =item B<--verbose> diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index b3dd7c09..1d627bbf 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -74,6 +74,14 @@ par_mem_leak() { fi } +par_timeout() { + echo "### test --timeout" + stdout time -f %e parallel --timeout 1s sleep ::: 10 | + perl -ne '1 < $_ and $_ < 10 and print "OK\n"' + stdout time -f %e parallel --timeout 1m sleep ::: 100 | + perl -ne '10 < $_ and $_ < 100 and print "OK\n"' +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 126ce0e1..be15e88a 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -19,3 +19,7 @@ par_retries_unreachable echo 1 par_retries_unreachable 1 par_retries_unreachable echo 2 par_retries_unreachable 2 +par_timeout 2>&1 +par_timeout ### test --timeout +par_timeout OK +par_timeout OK