From 5c89af948e9c8a2036aac701626b2d8d515ad751 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 21 Feb 2013 23:36:14 +0100 Subject: [PATCH] Code cleanup + man page updates. --- NEWS | 34 ++++++++++++++++++++++++++++++++++ doc/release_new_version | 2 ++ src/parallel | 3 +-- src/parallel.pod | 29 +++++++++++++++-------------- src/parallel.texi | 40 +++++++++++++++++++++++++++------------- src/sem.texi | 4 ++-- 6 files changed, 81 insertions(+), 31 deletions(-) diff --git a/NEWS b/NEWS index 5303421f..abf9774b 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,37 @@ +20130222 + +* --resume works with --pipe. + +* --resume-failed will go through --joblog, redo the failed jobs and + then continue like --resume. + +* Negative positional arguments count from the end: {-1} means the + last argument, {-2} the second to last. + +* NetBSD CPU detection. + +* --blocksize increases exponentially if it smaller than a full + record. + +* Processing n-line records (--pipe -L n) is now much faster. + +* --tollef is obsoleted after discussion on the mailing list. + +* GNU Parallel is the highest rated tool on: + http://www.biostars.org/show/tools/?sort=votes&since=all%20time + +* GNU Parallel was loved during FSFE's #ilovefs campaign. + http://fsfe.org/news/2013/news-20130212-01.en.html + +* Using GNU Parallel with s3cmd (Japanese). + http://blog.suz-lab.com/2013/02/s3cmd-gnu-paralells3.html + +* Intro to GNU Parallel (Chinese). + http://guiquanz.github.com/2013/02/12/gnu-parallel-intro/ + +* Bug fixes and man page updates. + + 20130122 * --sshdelay Delay starting next ssh by secs seconds. GNU parallel diff --git a/doc/release_new_version b/doc/release_new_version index a88ff8b5..6d30be3e 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -193,6 +193,8 @@ New in this release: * Processing n-line records (--pipe -L n) is now much faster. +* --tollef is obsoleted after discussion on the mailing list. + * GNU Parallel is the highest rated tool on: http://www.biostars.org/show/tools/?sort=votes&since=all%20time diff --git a/src/parallel b/src/parallel index 17ebdcbf..70a97080 100755 --- a/src/parallel +++ b/src/parallel @@ -937,7 +937,7 @@ sub open_joblog { $joblog_regexp='^(\d+)'; } while(<$joblog_fh>) { - if(/$joblog_regexp/) { + if(/$joblog_regexp/o) { # This is 30% faster than set_job_already_run($1); vec($Global::job_already_run,($1||0),1) = 1; } elsif(not /\d+\t[^\t]+\t([0-9.]+\t){6}/) { @@ -947,7 +947,6 @@ sub open_joblog { } close $joblog_fh; } - # print unpack("b*", $Global::job_already_run)."\n"; } if($append) { # Append to joblog diff --git a/src/parallel.pod b/src/parallel.pod index 93dc4c16..3790309a 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -147,17 +147,18 @@ The replacement string B<{#}> can be changed with B<--seqreplace>. To understand replacement strings see B<{}>. -=item B<{>IB<}> +=item B<{>IB<}> (alpha testing) Argument from input source I or the I'th argument. This positional replacement string will be replaced by the input from input source I (when used with B<-a> or B<::::>) or with the I'th -argument (when used with B<-N>). +argument (when used with B<-N>). If I is negative it refers to the +I'th last argument. To understand replacement strings see B<{}>. -=item B<{>I.B<}> +=item B<{>I.B<}> (alpha testing) Argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}> and B<{.}>. @@ -170,7 +171,7 @@ extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/B<}> +=item B<{>I/B<}> (alpha testing) Basename of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{/}>. @@ -183,7 +184,7 @@ directory (if any) removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I//B<}> +=item B<{>I//B<}> (alpha testing) Dirname of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{//}>. @@ -195,7 +196,7 @@ the I'th argument (when used with B<-N>). See B(1). To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/.B<}> +=item B<{>I/.B<}> (alpha testing) Basename of argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}>, B<{/}>, and @@ -344,9 +345,9 @@ Implies B<--semaphore>. Print the BibTeX entry for GNU B. -=item B<--block> I +=item B<--block> I (alpha testing) -=item B<--block-size> I +=item B<--block-size> I (alpha testing) Size of block in bytes. The size can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply the size with 1024, 1048576, @@ -566,7 +567,7 @@ specified, and for B<-I>{} otherwise. This option is deprecated; use B<-I> instead. -=item B<--joblog> I +=item B<--joblog> I (alpha testing) Logfile for executed jobs. Save a list of the executed jobs to I in the following TAB separated format: sequence number, @@ -664,7 +665,7 @@ to see the difference: =item B<-L> I -When used with B<--pipe>: Read records of I (beta testing). +When used with B<--pipe>: Read records of I (alpha testing). When used otherwise: Use at most I nonblank input lines per command line. Trailing blanks cause an input line to be logically @@ -787,9 +788,9 @@ Instead of printing the output to stdout (standard output) the output of each job is saved in a file and the filename is then printed. -=item B<--pipe> +=item B<--pipe> (alpha testing) -=item B<--spreadstdin> +=item B<--spreadstdin> (alpha testing) Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -1032,7 +1033,7 @@ will generate the files: See also B<--files>, B<--header>, B<--joblog>. -=item B<--resume> +=item B<--resume> (alpha testing) Resumes from the last unfinished job. By reading B<--joblog> GNU B will figure out the last unfinished job and continue from @@ -1376,7 +1377,7 @@ seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms later, followed by SIGKILL 200 ms later. -=item B<--tollef> +=item B<--tollef> (obsolete - will be retired 20140222) Make GNU B behave more like Tollef's parallel command. It activates B<-u>, B<-q>, and B<--arg-sep -->. It also causes B<-l> to diff --git a/src/parallel.texi b/src/parallel.texi index b31b2be2..4b469d82 100644 --- a/src/parallel.texi +++ b/src/parallel.texi @@ -506,8 +506,8 @@ See also: @strong{--bg}, @strong{man sem} Implies @strong{--semaphore}. -@item @strong{--filter-hosts} (beta testing) -@anchor{@strong{--filter-hosts} (beta testing)} +@item @strong{--filter-hosts} (alpha testing) +@anchor{@strong{--filter-hosts} (alpha testing)} Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -1115,7 +1115,20 @@ there. As GNU @strong{parallel} only looks at the sequence numbers in @strong{--joblog} then the input, the command, and @strong{--joblog} all have to remain unchanged; otherwise GNU @strong{parallel} may run wrong commands. -See also: @strong{--joblog}. +See also: @strong{--joblog}, @strong{--resume-failed}. + +@item @strong{--resume-failed} (alpha testing) +@anchor{@strong{--resume-failed} (alpha testing)} + +Retry all failed and resume from the last unfinished job. By reading +@strong{--joblog} GNU @strong{parallel} will figure out the failed jobs and run +those again. After that it will resume last unfinished job and +continue from there. As GNU @strong{parallel} only looks at the sequence +numbers in @strong{--joblog} then the input, the command, and @strong{--joblog} +all have to remain unchanged; otherwise GNU @strong{parallel} may run wrong +commands. + +See also: @strong{--joblog}, @strong{--resume}. @item @strong{--retries} @emph{n} @anchor{@strong{--retries} @emph{n}} @@ -1263,8 +1276,8 @@ Like this: @strong{--shebang} must be set as the first option. -@item @strong{--shebang-wrap} (alpha testing) -@anchor{@strong{--shebang-wrap} (alpha testing)} +@item @strong{--shebang-wrap} (beta testing) +@anchor{@strong{--shebang-wrap} (beta testing)} GNU @strong{parallel} can parallelize scripts by wrapping the shebang line. If the program can be run like this: @@ -1612,11 +1625,11 @@ Use @strong{-v} @strong{-v} to print the wrapping ssh command when running remot Print the version GNU @strong{parallel} and exit. -@item @strong{--workdir} @emph{mydir} (beta testing) -@anchor{@strong{--workdir} @emph{mydir} (beta testing)} +@item @strong{--workdir} @emph{mydir} +@anchor{@strong{--workdir} @emph{mydir}} -@item @strong{--wd} @emph{mydir} (beta testing) -@anchor{@strong{--wd} @emph{mydir} (beta testing)} +@item @strong{--wd} @emph{mydir} +@anchor{@strong{--wd} @emph{mydir}} Files transferred using @strong{--transfer} and @strong{--return} will be relative to @emph{mydir} on remote computers, and the command will be executed in @@ -3672,10 +3685,11 @@ Your bug report should always include: released version you should specify why you believe the problem is not fixed in that version. -@item A complete example that others can run that shows the problem. A -combination of @strong{seq}, @strong{cat}, @strong{echo}, and @strong{sleep} can reproduce -most errors. If your example requires large files, see if you can make -them by something like @strong{seq 1000000} > @strong{file}. +@item A complete example that others can run that shows the problem. This +should preferably be small and simple. A combination of @strong{seq}, +@strong{cat}, @strong{echo}, and @strong{sleep} can reproduce most errors. If your +example requires large files, see if you can make them by something +like @strong{seq 1000000} > @strong{file}. @item The output of your example. If your problem is not easily reproduced by others, the output might help them figure out the problem. diff --git a/src/sem.texi b/src/sem.texi index 4a0bea82..5a156047 100644 --- a/src/sem.texi +++ b/src/sem.texi @@ -214,13 +214,13 @@ Report bugs to . @chapter AUTHOR @anchor{AUTHOR} -Copyright (C) 2010,2011,2012 Ole Tange, http://ole.tange.dk and Free +Copyright (C) 2010,2011,2012,2013 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. @chapter LICENSE @anchor{LICENSE} -Copyright (C) 2010,2011,2012 Free Software Foundation, Inc. +Copyright (C) 2010,2011,2012,2013 Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by