parallel: Fixed bug #56873: Auto detect --nice level.

10seconds_install

@@ -24,20 +24,20 @@
 
 run() {
     # tail on openindiana must be /usr/xpg4/bin/tail
-    TAIL=$(echo | tail -n 1 2>/dev/null && echo tail ||
+    tail=$(echo | tail -n 1 2>/dev/null && echo tail ||
 		      (echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail))
     # grep on openindiana must be /usr/xpg4/bin/grep
-    GREP=$(echo | grep -vE . 2>/dev/null && echo grep ||
+    grep=$(echo | grep -vE . 2>/dev/null && echo grep ||
 		      (echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep))
     # FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
-    GET=$(
+    get=$(
 	(lynx -source /dev/null && echo lynx -source) ||
 	    (fetch -o /dev/null file:///bin/sh && echo fetch -o -) ||
 	    (curl -h >/dev/null && echo curl -L) ||
        	    (wget -h >/dev/null && echo wget -qO -) ||
 	    echo 'No lynx, wget, curl, fetch: Please inform parallel@gnu.org what you use for downloading URLs' >&2
        )
-    if test "$GET" = ""; then
+    if test "$get" = ""; then
 	exit 1
     fi
 
@@ -47,14 +47,14 @@ run() {
     fi
 
     LANG=C
-    LATEST=$($GET http://ftpmirror.gnu.org/parallel |
+    latest=$($get http://ftpmirror.gnu.org/parallel |
 		    perl -ne '/.*(parallel-\d{8})/ and print $1."\n"' |
-		    sort | $TAIL -n1)
+		    sort | $tail -n1)
-    if test \! -e $LATEST.tar.bz2; then
+    if test \! -e $latest.tar.bz2; then
 	# Source tar does not exist
-	rm -f $LATEST.tar.bz2 $LATEST.tar.bz2.sig
+       	rm -f $latest.tar.bz2 $latest.tar.bz2.sig
-	$GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2 > $LATEST.tar.bz2
+	$get http://ftpmirror.gnu.org/parallel/$latest.tar.bz2 > $latest.tar.bz2
-	$GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig > $LATEST.tar.bz2.sig
+	$get http://ftpmirror.gnu.org/parallel/$latest.tar.bz2.sig > $latest.tar.bz2.sig
     fi
 
     fetch_keys() {
@@ -90,15 +90,15 @@ run() {
 
     # Check signature - in case ftpmirror.gnu.org is compromised
     if fetch_keys; then
-      	if gpg --with-fingerprint $LATEST.tar.bz2.sig 2>&1 |
+      	if gpg --with-fingerprint $latest.tar.bz2.sig 2>&1 |
-	       $GREP -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC  66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610  7E7B D1AB 4516 8888 8888' ; then
+	       $grep -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC  66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610  7E7B D1AB 4516 8888 8888' ; then
    	    # Source code signed by Ole Tange <ole@tange.dk>
 	    # KeyID FFFFFFF1/88888888
 	    true
 	else
     	    # GnuPG signature failed
 	    echo
-    	    echo "The signature on $LATEST.tar.bz2 is wrong. This may indicate that a criminal has changed the code."
+    	    echo "The signature on $latest.tar.bz2 is wrong. This may indicate that a criminal has changed the code."
 	    echo "THIS IS BAD AND THE CODE WILL NOT BE INSTALLED."
 	    echo
 	    echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods."
@@ -119,21 +119,21 @@ run() {
 	fi
     fi
 
-    bzip2 -dc $LATEST.tar.bz2 | tar xf -
+    bzip2 -dc $latest.tar.bz2 | tar xf -
-    cd $LATEST || exit 2
+    cd $latest || exit 2
     if ./configure && make && make install; then
 	echo
-	echo GNU $LATEST installed globally
+	echo GNU $latest installed globally
     else
 	if ./configure --prefix=$HOME && make && make install; then
   	    echo
-	    echo GNU $LATEST installed in $HOME/bin
+	    echo GNU $latest installed in $HOME/bin
 	else
 	    mkdir -p $HOME/bin/;
        	    chmod 755 src/*;
 	    cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin;
 	    echo
-       	    echo GNU $LATEST copied to $HOME/bin
+       	    echo GNU $latest copied to $HOME/bin
 	fi
 
 	# Is $HOME/bin already in $PATH?

README

@@ -1,7 +1,7 @@
 
-			 GNU Parallel README
+			     GNU Parallel
 
-Please send problems and feedback to bug-parallel@gnu.org.
+		https://www.gnu.org/software/parallel/
 
 
 = Presentation of GNU Parallel =

doc/haikus

@@ -1,5 +1,11 @@
 Quote of the month:
 
+  IMHO, SQLite and GNU Parallel are among the world's great software.
+    -- singe@reddit
+
+  Yeah, GNU parallel is a beast when used accordingly.
+    -- @lsde@twitter
+
   GNU Parallel is awesome
     -- Gaurav Jain @gauravjain49@twitter
   

doc/release_new_version

@@ -209,9 +209,9 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20190822 ('Jesper Svarre') released <<[stable]>>
+Subject: GNU Parallel 20190822 ('Dorian') released <<[stable]>>
 
-GNU Parallel 20190822 ('Jesper Svarre') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20190822 ('Dorian') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -225,21 +225,28 @@ Quote of the month:
 
 New in this release:
 
-* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it#beskrivelse
+* --nice is now inherited by the nice level that GNU Parallel is started at. So 'nice -n10 parallel' will also cause remote jobs to be run at nice level 10.
 
-* Linux Productivity Tools https://www.usenix.org/conference/lisa19/presentation/maheshwari
+* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it/tools
 
-* Improve your reconnaissance performance by using GNU Parallel https://mailchi.mp/3f46ceb46357/improve-your-reconnaissance-performance-by-using-gnu-parallel
+* Greenland Ice Sheet solid ice discharge from 1986 through 2017 https://www.earth-syst-sci-data.net/11/769/2019/essd-11-769-2019.pdf
 
-* Paralel İşler / İşlemler / Programlar Bash’de Nasıl Çalıştırılır? https://kaleinfo.com/2019/07/29/paralel-isler-islemler-programlar-bashde-nasil-calistirilir-2/
+https://dualuse.io/blog/curryfinger/
 
-* Use POLYBAR to change WALLPAPER with GNU Parallel https://www.youtube.com/watch?v=rjYg-b_eiRU
+https://wiki.cac.washington.edu/display/hyakusers/Hyak+parallel-sql
 
-* Finding the Balance Between Speed & Accuracy During an Internet-wide Port Scanning https://captmeelo.com/pentest/2019/07/29/port-scanning.html
+https://www.reddit.com/r/perl/comments/cyo2qc/a_meditation_on_analysing_data_with_perl_in/
 
-* GNU Parallel Ryugu released https://linuxreviews.org/GNU_Parallel_Ryugu_released
+https://www.eventbrite.ca/e/inscription-accelerer-ses-taches-avec-gnu-parallel-et-les-lots-de-taches-ul-71412840729
 
-* CO2017 - CW1 - Operating Systems & Networks - Shell scripting http://www.7daixie.com/2019071646302396.html
+https://curc.readthedocs.io/en/latest/software/GNUParallel.html
+
+https://petelawson.com/post/parallel-in-shell/	
+
+https://www.usenix.org/conference/lisa19/presentation/maheshwari
+
+http://atomicer.cn/2018/01/18/gnu-parallel-nfs-%E5%AE%9E%E7%8E%B0%E5%85%B1%E4%BA%AB%E6%96%87%E4%BB%B6%E7%BD%91%E7%BB%9C%E7%9A%84%E5%B9%B6%E5%8F%91%E8%AE%A1%E7%AE%97%E7%AD%89%E6%93%8D%E4%BD%9C/#more
+Gnu/parallel + Nfs 实现共享文件网络的并发计算等操作 
 
 * Bug fixes and man page updates.
 

src/niceload

@@ -23,7 +23,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20190822;
+$Global::version = 20190823;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -1676,7 +1676,8 @@ sub parse_options(@) {
     if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; }
     $ENV{'PARALLEL_RSYNC_OPTS'} = $opt::rsync_opts ||
 	$ENV{'PARALLEL_RSYNC_OPTS'} || '-rlDzR';
-    $opt::nice ||= 0;
+    # Default: Same nice level as GNU Parallel is started at
+    $opt::nice ||= getpriority(0,0) || 0;
     if(defined $opt::help) { usage(); exit(0); }
     if(defined $opt::embed) { embed(); exit(0); }
     if(defined $opt::sqlandworker) {
@@ -1929,9 +1930,11 @@ sub parse_options(@) {
     # the alternatives instead?
     # See a list in: 'man parallel_alternatives'
     #
-    # If you are an honest person please read the above before
+    # If you want GNU Parallel to be maintained in the future you should not
-    # changing this line.
+    # change this line.
     citation_notice();
+    # Seriously: You will be harming free software by removing the notice.
+    # You make it harder to justify spending time developing it.
 
     parse_halt();
 
@@ -2048,7 +2051,7 @@ sub check_invalid_option_combinations() {
 
 sub init_globals() {
     # Defaults:
-    $Global::version = 20190822;
+    $Global::version = 20190823;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;

src/parallel.pod

@@ -1600,13 +1600,18 @@ Overrides an earlier B<--keep-order> (e.g. if set in
 B<~/.parallel/config>).
 
 
-=item B<--nice> I<niceness>
+=item B<--nice> I<niceness> (alpha testing)
 
-Run the command at this niceness. For simple commands you can just add
+Run the command at this niceness.
-B<nice> in front of the command. But if the command consists of more
+
-sub commands (Like: ls|wc) then prepending B<nice> will not always
+By default GNU B<parallel> will run jobs at the same nice level as GNU
-work. B<--nice> will make sure all sub commands are niced - even on
+B<parallel> is started - both on the local machine and remote servers,
-remote servers.
+so you are unlikely to ever use this option.
+
+Setting B<--nice> will override this nice level. If the nice level is
+smaller than the current nice level, it will only affect remote jobs
+(e.g. current level is 10 and B<--nice 5> will cause local jobs to be
+run at level 10, but remote jobs run at nice level 5).
 
 
 =item B<--interactive>

src/parallel_alternatives.pod

@@ -141,12 +141,13 @@ B<xargs> deals badly with special characters (such as space, \, ' and
   touch 'c:\windows\system32\clfs.sys'
   echo 'c:\windows\system32\clfs.sys' | xargs ls -l
 
-You can specify B<-0>, but many input generators are not
+You can specify B<-0>, but many input generators are not optimized for
-optimized for using B<NUL> as separator but are optimized for
+using B<NUL> as separator but are optimized for B<newline> as
-B<newline> as separator. E.g B<head>, B<tail>, B<awk>, B<ls>, B<echo>,
+separator. E.g. B<awk>, B<ls>, B<echo>, B<tar -v>, B<head> (requires
-B<sed>, B<tar -v>, B<perl> (B<-0> and \0 instead of \n), B<locate>
+using B<-z>), B<tail> (requires using B<-z>), B<sed> (requires using
-(requires using B<-0>), B<find> (requires using B<-print0>), B<grep>
+B<-z>), B<perl> (B<-0> and \0 instead of \n), B<locate> (requires
-(requires user to use B<-z> or B<-Z>), B<sort> (requires using B<-z>).
+using B<-0>), B<find> (requires using B<-print0>), B<grep> (requires
+using B<-z> or B<-Z>), B<sort> (requires using B<-z>).
 
 GNU B<parallel>'s newline separation can be emulated with:
 
@@ -467,7 +468,8 @@ B<dxargs> does not deal well with more simultaneous jobs than SSHD's
 MaxStartups. B<dxargs> is only built for remote run jobs, but does not
 support transferring of files.
 
-https://web.archive.org/web/20120518070250/http://www.semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
+https://web.archive.org/web/20120518070250/http://www.
+semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
 
 
 =head2 DIFFERENCES BETWEEN mdm/middleman AND GNU Parallel
@@ -475,8 +477,8 @@ https://web.archive.org/web/20120518070250/http://www.semicomplete.com/blog/geek
 middleman(mdm) is also a tool for running jobs in parallel.
 
 Here are the shellscripts of
-https://web.archive.org/web/20110728064735/http://mdm.berlios.de/usage.html
+https://web.archive.org/web/20110728064735/http://mdm.
-ported to GNU B<parallel>:
+berlios.de/usage.html ported to GNU B<parallel>:
 
   seq 19 | parallel buffon -o - | sort -n > result
   cat files | parallel cmd
@@ -576,7 +578,8 @@ B</usr/joe>, enter:
   apply 'ln %1 /usr/joe' *
   parallel ln {} /usr/joe ::: *
 
-https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
+https://www-01.ibm.com/support/knowledgecenter/
+ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
 
 
 =head2 DIFFERENCES BETWEEN paexec AND GNU Parallel
@@ -877,8 +880,8 @@ If the above is set up you can then do:
 
   orgalorg -o frontend1 -o frontend2 -p -er /tmp -n \
     'md5sum /tmp/bigfile' -S bigfile
-  par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp \
+  par_emul -S frontend1 -S frontend2 --basefile bigfile \
-    md5sum /tmp/bigfile
+    --workdir /tmp md5sum /tmp/bigfile
 
 B<orgalorg> has a progress indicator for the transferring of a
 file. GNU B<parallel> does not.
@@ -1209,7 +1212,7 @@ processing result with `samtools`:>
   $ threads=25
   $ ls -d raw.cluster.clean.mapping/* \
     | rush -v ref=$ref -v j=$threads -v p='{}/{%}' \
-        'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz > {p}.sam; \
+        'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz >{p}.sam;\
         samtools view -bS {p}.sam > {p}.bam; \
         samtools sort -T {p}.tmp -@ {j} {p}.bam -o {p}.sorted.bam; \
         samtools index {p}.sorted.bam; \
@@ -1877,7 +1880,8 @@ output. It buffers in RAM. The examples from README.md:
   find . -iname '*.jpg' |
     machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {}
   find . -iname '*.jpg' |
-    parallel --timeout 5s --bar mogrify -resize 1200x1200 -filter Lanczos {}
+    parallel --timeout 5s --bar mogrify -resize 1200x1200 \
+      -filter Lanczos {}
 
   find . -iname '*.jpg' -print0 |
     machma --null --  mogrify -resize 1200x1200 -filter Lanczos {}
@@ -1889,23 +1893,70 @@ https://github.com/fd0/machma (Last checked: 2019-06)
 
 =head2 DIFFERENCES BETWEEN interlace AND GNU Parallel
 
+Summary table (see legend above):
+- I2 I3 I4 - - -
+M1 - M3 - - M6
+- O2 O3 - - - - x x
+E1 E2 - - - - -
+- - - - - - - - -
+- -
+
 B<interlace> is built for network analysis to run network tools in parallel.
 
 B<interface> does not buffer output, so output from different jobs mixes.
 
+The overhead for each target is O(n*n), so with 1000 targets it
+becomes very slow with an overhead in the order of 500ms/target.
+
 Using B<prips> most of the examples from
 https://github.com/codingo/Interlace can be run with GNU B<parallel>:
 
+Blocker
+
+  commands.txt:
+    mkdir -p _output_/_target_/scans/
+    _blocker_
+    nmap _target_ -oA _output_/_target_/scans/_target_-nmap
+  interlace -tL ./targets.txt -cL commands.txt -o $output
+
+  parallel -a targets.txt \
+    mkdir -p $output/{}/scans/\; nmap {} -oA $output/{}/scans/{}-nmap
+
+Blocks
+
+  commands.txt:
+    _block:nmap_
+    mkdir -p _target_/output/scans/
+    nmap _target_ -oN _target_/output/scans/_target_-nmap
+    _block:nmap_
+    nikto --host _target_
+  interlace -tL ./targets.txt -cL commands.txt
+
+  _nmap() {
+    mkdir -p $1/output/scans/
+    nmap $1 -oN $1/output/scans/$1-nmap
+  }
+  export -f _nmap
+  parallel ::: _nmap "nikto --host" :::: targets.txt
+
+Run Nikto Over Multiple Sites
+
   interlace -tL ./targets.txt -threads 5 \
     -c "nikto --host _target_ > ./_target_-nikto.txt" -v
-  parallel -a targets.txt -P5 nikto --host {} > ./{}_-nikto.txt
+
+  parallel -a targets.txt -P5 nikto --host {} \> ./{}_-nikto.txt
+
+Run Nikto Over Multiple Sites and Ports
 
   interlace -tL ./targets.txt -threads 5 -c \
     "nikto --host _target_:_port_ > ./_target_-_port_-nikto.txt" \
     -p 80,443 -v
-  parallel -P5 nikto --host {1}:{2} > ./{1}-{2}-nikto.txt \
+
+  parallel -P5 nikto --host {1}:{2} \> ./{1}-{2}-nikto.txt \
     :::: targets.txt ::: 80 443
 
+Run a List of Commands against Target Hosts
+
   commands.txt:
     nikto --host _target_:_port_ > _output_/_target_-nikto.txt
     sslscan _target_:_port_ >  _output_/_target_-sslscan.txt
@@ -1913,63 +1964,77 @@ https://github.com/codingo/Interlace can be run with GNU B<parallel>:
   interlace -t example.com -o ~/Engagements/example/ \
     -cL ./commands.txt -p 80,443
 
-  _nikto() {
+  parallel --results ~/Engagements/example/{2}:{3}{1} {1} {2}:{3} \
-    nikto --host "$1:$2"
+    ::: "nikto --host" sslscan testssl.sh ::: example.com ::: 80 443
-  }
+
-  _sslscan() {
+CIDR notation with an application that doesn't support it
-    sslscan "$1:$2"
-  }
-  _testssl() {
-    testssl.sh "$1:$2"
-  }
-  export -f _nikto
-  export -f _sslscan
-  export -f _testssl
-  parallel --results ~/Engagements/example/{2}:{3}{1} \
-    ::: _nikto _sslscan _testssl ::: example.com ::: 80 443
 
   interlace -t 192.168.12.0/24 -c "vhostscan _target_ \
     -oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50
+
   prips 192.168.12.0/24 |
     parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
 
+Glob notation with an application that doesn't support it
+
   interlace -t 192.168.12.* -c "vhostscan _target_ \
     -oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50
+
   # Glob is not supported in prips
   prips 192.168.12.0/24 |
     parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
 
+Dash (-) notation with an application that doesn't support it
+
   interlace -t 192.168.12.1-15 -c \
     "vhostscan _target_ -oN _output_/_target_-vhosts.txt" \
     -o ~/scans/ -threads 50
+
   # Dash notation is not supported in prips
   prips 192.168.12.1 192.168.12.15 |
     parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
 
+Threading Support for an application that doesn't support it
+
   interlace -tL ./target-list.txt -c \
     "vhostscan -t _target_ -oN _output_/_target_-vhosts.txt" \
     -o ~/scans/ -threads 50
+
   cat ./target-list.txt |
     parallel -P50 vhostscan -t {} -oN ~/scans/{}-vhosts.txt
 
-  ./vhosts-commands.txt -tL ./target-list.txt:
+alternatively
+
+  ./vhosts-commands.txt:
     vhostscan -t $target -oN _output_/_target_-vhosts.txt
   interlace -cL ./vhosts-commands.txt -tL ./target-list.txt \
     -threads 50 -o ~/scans
 
-  ./vhosts-commands.txt -tL ./target-list.txt:
+  ./vhosts-commands.txt:
     vhostscan -t "$1" -oN "$2"
-  parallel -P50 ./vhosts-commands.txt {} ~/scans/{} \
+  parallel -P50 ./vhosts-commands.txt {} ~/scans/{}-vhosts.txt \
     :::: ./target-list.txt
 
+Exclusions
+
   interlace -t 192.168.12.0/24 -e 192.168.12.0/26 -c \
     "vhostscan _target_ -oN _output_/_target_-vhosts.txt" \
     -o ~/scans/ -threads 50
+
   prips 192.168.12.0/24 | grep -xv -Ff <(prips 192.168.12.0/26) |
     parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
 
+Run Nikto Using Multiple Proxies
 
-https://github.com/codingo/Interlace (Last checked: 2019-02)
+   interlace -tL ./targets.txt -pL ./proxies.txt -threads 5 -c \
+     "nikto --host _target_:_port_ -useproxy _proxy_ > \
+      ./_target_-_port_-nikto.txt" -p 80,443 -v
+
+   parallel -j5 \
+     "nikto --host {1}:{2} -useproxy {3} > ./{1}-{2}-nikto.txt" \
+     :::: ./targets.txt ::: 80 443 :::: ./proxies.txt
+
+https://github.com/codingo/Interlace (Last checked: 2019-09)
 
 
 =head2 DIFFERENCES BETWEEN otonvm Parallel AND GNU Parallel

src/sql

@@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20190822;
+    $Global::version = 20190823;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself