From a14fabad362af4a0040e6723af56c02753b2b1e2 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 7 Apr 2011 21:54:02 +0200 Subject: [PATCH] parallel: --joblog contains exitstatus --- src/parallel | 8 +++++--- src/parallel.pod | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/parallel b/src/parallel index fa4905b0..cc393a4f 100755 --- a/src/parallel +++ b/src/parallel @@ -565,7 +565,7 @@ sub parse_options { } else { print $Global::joblog join("\t", "Seq", "Host", "Starttime", "Runtime", - "Send", "Receive", "Command" + "Send", "Receive", "Status", "Command" ). "\n"; } } @@ -3037,8 +3037,10 @@ sub print { } printf $Global::joblog join("\t", $self->seq(), $self->sshlogin()->string(), - $self->starttime(), $self->runtime(), $self->transfersize(), $self->returnsize(), $cmd - ). "\n"; + $self->starttime(), $self->runtime(), + $self->transfersize(), $self->returnsize(), + $self->exitstatus(), $cmd + ). "\n"; flush $Global::joblog; } diff --git a/src/parallel.pod b/src/parallel.pod index da8b6bee..8a1cf174 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -4,6 +4,7 @@ parallel - build and execute shell command lines from standard input in parallel + =head1 SYNOPSIS B [options] [I [arguments]] < list_of_arguments @@ -16,6 +17,7 @@ B --semaphore [options] I B<#!/usr/bin/parallel> --shebang [options] [I [arguments]] + =head1 DESCRIPTION GNU B is a shell tool for executing jobs in parallel using one @@ -50,6 +52,7 @@ http://www.youtube.com/watch?v=OpaiGYxkSuQ or at http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and http://tinyogg.com/watch/YQuXd/ + =head1 OPTIONS =over 9 @@ -421,7 +424,8 @@ use B<-I> instead. Logfile for executed jobs. Saved a list of the executed jobs to I in the following TAB separated format: sequence number, sshlogin, start time as seconds since epoch, run time in seconds, -bytes in files transfered, bytes in files returned, command run. +bytes in files transfered, bytes in files returned, exit status, +and command run. To convert the times into ISO-8601 strict do: @@ -1133,6 +1137,7 @@ For this to work B<--shebang> or B<-Y> must be set as the first option. =back + =head1 EXAMPLE: Working as xargs -n1. Argument appending GNU B can work similar to B. @@ -1244,6 +1249,7 @@ Other special shell charaters (such as * ; $ > < | >> <<) also needs to be put in quotes, as they may otherwise be interpreted by the shell and not given to GNU B. + =head1 EXAMPLE: Composed commands A job can consist of several commands. This will print the number of @@ -1271,6 +1277,7 @@ symlinks are empty files. B + =head1 EXAMPLE: Removing file extension when processing files When processing files removing the file extension using B<{.}> is @@ -1293,6 +1300,7 @@ Put all converted in the same directory: B + =head1 EXAMPLE: Removing two file extensions when processing files and calling GNU Parallel from itself @@ -1302,6 +1310,7 @@ foo) you can do: B + =head1 EXAMPLE: Download 10 images for each of the past 30 days Let us assume a website stores images like: @@ -1318,6 +1327,19 @@ Based on this we can let GNU B generate 10 Bs per day: I B<| parallel -I {o} seq -w 10 "|" parallel wget http://www.example.com/path/to/{o}_{}.jpg> + +=head1 EXAMPLE: Process files from a tar file while unpacking + +If the files to be processed are in a tar file then unpacking one file +and processing it immediately may be faster than first unpacking all +files. + +B + +The Perl one-liner is needed to avoid race condition. + + =head1 EXAMPLE: Rewriting a for-loop and a while-read-loop for-loops like this: @@ -1421,6 +1443,7 @@ combined in the correct order. B > B + =head1 EXAMPLE: Parallel grep B greps recursively through directories. On multicore CPUs @@ -1554,6 +1577,7 @@ the special short hand I<-S ..> can be used: find logs/ -name '*.gz' | parallel -S .. \ --trc {.}.bz2 "zcat {} | bzip2 -9 >{.}.bz2" + =head1 EXAMPLE: Distributing work to local and remote computers Convert *.mp3 to *.ogg running one process per CPU core on local computer and server2: @@ -1561,6 +1585,7 @@ Convert *.mp3 to *.ogg running one process per CPU core on local computer and se parallel --trc {.}.ogg -S server2,: \ 'mpg321 -w - {} | oggenc -q0 - -o {.}.ogg' ::: *.mp3 + =head1 EXAMPLE: Use multiple inputs in one command Copy files like foo.es.ext to foo.ext: @@ -1750,6 +1775,7 @@ printed when job 11 is started. =back + =head1 EXAMPLE: GNU Parallel as dir processor If you have a dir in which users drop files that needs to be processed @@ -1771,6 +1797,10 @@ computers: B +If the files to be processed are in a tar file then unpacking one file +and processing it immediately may be faster than first unpacking all +files. Set up the dir processor as above and unpack into the dir. + =head1 QUOTING @@ -1973,6 +2003,7 @@ must be one argument. =back + =head1 DEFAULT PROFILE (CONFIG FILE) The file ~/.parallel/config (formerly known as .parallelrc) will be @@ -1984,6 +2015,7 @@ Options on the command line takes precedence over the environment variable $PARALLEL which takes precedence over the file ~/.parallel/config. + =head1 PROFILE FILES If B<--profile> set, GNU B will read the profile from that file instead of @@ -2281,8 +2313,8 @@ The argument replace string ($ITEM) cannot be changed. Arguments must be quoted - thus arguments containing special characters (space '"&!*) may cause problems. More than one argument is not supported. File names containing newlines are not processed correctly. When reading -input from a file null cannot be used terminator. B needs to -read the whole input file before starting any jobs. +input from a file null cannot be used as a terminator. B needs +to read the whole input file before starting any jobs. Output and status information is stored in ppss_dir and thus requires cleanup when completed. If the dir is not removed before running @@ -2796,16 +2828,18 @@ license terms of this work. A copy of the full license is included in the file as cc-by-sa.txt. + =head1 DEPENDENCIES GNU B uses Perl, and the Perl modules Getopt::Long, IPC::Open3, Symbol, IO::File, POSIX, and File::Temp. For remote usage -it also uses Rsync with Ssh. +it also uses rsync with ssh. =head1 SEE ALSO -B(1), B(1), B(1), B(1), B(1), -B(1), B(1), B(1), B(1) +B(1), B(1), B(1), B(1), B(1), +B(1), B(1), B(1), B(1), B(1), +B(1), =cut