From 7bab1b45bcd27564618b84d0a12ffedf36d4fcc0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 21 Nov 2019 23:46:15 +0100 Subject: [PATCH] Documentation update. --- doc/release_new_version | 26 +++++--------------------- src/parallel | 11 +++++++---- src/parallel.pod | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 76ecd710..beb501fd 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -209,9 +209,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20191022 ('Driving IT') released <<[stable]>> +Subject: GNU Parallel 20191122 ('Berlin/Morales') released <<[stable]>> -GNU Parallel 20191022 ('Driving IT') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20191122 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -221,30 +221,14 @@ See https://www.gnu.org/software/parallel/10-years-anniversary.html Quote of the month: - I've said it before: The command line program GNU Parallel is a godsend. - -- Jo Chr. Oterhals @oterhals@twitter New in this release: -* --tee will use --output-error=warn-nopipe if supported by tee. +* GNU Parallel was presented at Driving IT. Slides: -* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it/tools +https://blog.naderman.de/2019/11/14/restarting-supervisord-processes-in-parallel/ -* UMN Duluth: Job parallelization with task arrays and GNU parallel https://www.msi.umn.edu/tutorials/umn-duluth-job-parallelization-task-arrays-and-gnu-parallel - -* Genome updater uses GNU Parallel https://github.com/pirovc/genome_updater - -* Using GNU-Parallel for bioinformatics https://www.danielecook.com/using-gnu-parallel-for-bioinformatics/ - -* Speeding up PostgreSQL ETL pipeline with the help of GODS https://cfengine.com/company/blog-detail/speeding-up-postgresql-etl-pipeline-with-the-help-of-gods/ - -* Runing linux commands in parallel https://dev.to/voyeg3r/runing-linux-commands-in-parallel-4ff8 - -* Research Computing University of Colorado Boulder contains an intro to GNU Parallel https://readthedocs.org/projects/curc/downloads/pdf/latest/ - -* 如何使用Parallel在Shell中并行执行命令https://www.myfreax.com/gnu-parallel/ - -* 如何测试 Amazon Elastic File System https://aws.amazon.com/cn/blogs/china/how-to-test-drive-amazon-elastic-file-system/ +* https://www.reddit.com/r/DataHoarder/comments/dsgmhc/parallel_archiving_techniques/ * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index c26b30de..ac4ae716 100755 --- a/src/parallel +++ b/src/parallel @@ -1193,7 +1193,7 @@ sub nindex($$) { # until one is empty and a new block is passed if(@robin_queue) { # Rotate queue once so new blocks get a fair chance - # to be given to another block + # to be given to another slot push @robin_queue, shift @robin_queue; } else { # Make a queue to spread the blocks evenly @@ -1950,7 +1950,7 @@ sub parse_options(@) { # If you want GNU Parallel to be maintained in the future keep # this line. citation_notice(); - # Seriously: YOU will be harming free software by removing the + # Seriously: *You* will be harming free software by removing the # notice. You make it harder to justify spending time developing # it. If you *do* remove the line, please email # hallofshame@tange.dk if you want to avoid being put in a hall of @@ -8291,7 +8291,7 @@ sub non_blocking_write($) { if (!defined($rv) && $! == ::EAGAIN()) { # would block - but would have written $something_written = 0; - # avoid triggering auto expanding block + # avoid triggering auto expanding block size $Global::no_autoexpand_block ||= 1; } elsif ($self->{'block_pos'}+$rv != $self->{'block_length'}) { # incomplete write @@ -9174,6 +9174,10 @@ sub remote_cleanup($) { } sub exitstatuswrapper(@) { + # Input: + # @shellcode = shell code to execute + # Returns: + # shell script that returns current status after executing @shellcode if($Global::cshell) { return ('set _EXIT_status=$status; ' . join(" ",@_). @@ -9185,7 +9189,6 @@ sub exitstatuswrapper(@) { } } - sub workdir($) { # Returns: # the workdir on a remote machine diff --git a/src/parallel.pod b/src/parallel.pod index 0775f845..5effa604 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2847,6 +2847,7 @@ Exit if the size (see the B<-s> option) is exceeded. =back + =head1 EXAMPLE: Working as xargs -n1. Argument appending GNU B can work similar to B. @@ -3992,7 +3993,13 @@ then the obvious solution would be this, but it B: It does not work because the command is dequoted by B twice where as GNU B only expects it to be dequoted once. -So instead put this in B<~/.ssh/config>: +You can use a bash function and have GNU B quote the command: + + jumpssh() { ssh -A jumphost ssh $(parallel --shellquote ::: "$@"); } + export -f jumpssh + parallel --ssh jumpssh -S host1 echo ::: this works + +Or you can instead put this in B<~/.ssh/config>: Host host1 host2 host3 ProxyCommand ssh jumphost.domain nc -w 1 %h 22 @@ -4224,6 +4231,25 @@ To run 100 processes simultaneously do: As there is not a I the jobs will be evaluated by the shell. +=head1 EXAMPLE: Call program with FASTA sequence + +FASTA files have the format: + + >Sequence name1 + sequence + sequence continued + >Sequence name2 + sequence + sequence continued + more sequence + +To call B with the sequence as argument run: + + cat file.fasta | + parallel --pipe -N1 --recstart '>' --rrs \ + 'read a; echo Name: "$a"; myprog $(tr -d "\n")' + + =head1 EXAMPLE: Processing a big file using more CPUs To process a big file or some output you can use B<--pipe> to split up @@ -5248,8 +5274,11 @@ A copy of the full license is included in the file as cc-by-sa.txt. =head1 DEPENDENCIES GNU B uses Perl, and the Perl modules Getopt::Long, -IPC::Open3, Symbol, IO::File, POSIX, and File::Temp. For remote usage -it also uses rsync with ssh. +IPC::Open3, Symbol, IO::File, POSIX, and File::Temp. + +For B<--csv> it uses the Perl module Text::CSV. + +For remote usage it uses B with B. =head1 SEE ALSO