parallel: Fixed bug #56873: Auto detect --nice level.

This commit is contained in:
Ole Tange 2019-09-16 23:12:53 +02:00
parent c7851f76e2
commit 512059bf1e
9 changed files with 173 additions and 87 deletions

View file

@ -24,20 +24,20 @@
run() {
# tail on openindiana must be /usr/xpg4/bin/tail
TAIL=$(echo | tail -n 1 2>/dev/null && echo tail ||
tail=$(echo | tail -n 1 2>/dev/null && echo tail ||
(echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail))
# grep on openindiana must be /usr/xpg4/bin/grep
GREP=$(echo | grep -vE . 2>/dev/null && echo grep ||
grep=$(echo | grep -vE . 2>/dev/null && echo grep ||
(echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep))
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
GET=$(
get=$(
(lynx -source /dev/null && echo lynx -source) ||
(fetch -o /dev/null file:///bin/sh && echo fetch -o -) ||
(curl -h >/dev/null && echo curl -L) ||
(wget -h >/dev/null && echo wget -qO -) ||
echo 'No lynx, wget, curl, fetch: Please inform parallel@gnu.org what you use for downloading URLs' >&2
)
if test "$GET" = ""; then
if test "$get" = ""; then
exit 1
fi
@ -47,14 +47,14 @@ run() {
fi
LANG=C
LATEST=$($GET http://ftpmirror.gnu.org/parallel |
latest=$($get http://ftpmirror.gnu.org/parallel |
perl -ne '/.*(parallel-\d{8})/ and print $1."\n"' |
sort | $TAIL -n1)
if test \! -e $LATEST.tar.bz2; then
sort | $tail -n1)
if test \! -e $latest.tar.bz2; then
# Source tar does not exist
rm -f $LATEST.tar.bz2 $LATEST.tar.bz2.sig
$GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2 > $LATEST.tar.bz2
$GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig > $LATEST.tar.bz2.sig
rm -f $latest.tar.bz2 $latest.tar.bz2.sig
$get http://ftpmirror.gnu.org/parallel/$latest.tar.bz2 > $latest.tar.bz2
$get http://ftpmirror.gnu.org/parallel/$latest.tar.bz2.sig > $latest.tar.bz2.sig
fi
fetch_keys() {
@ -90,15 +90,15 @@ run() {
# Check signature - in case ftpmirror.gnu.org is compromised
if fetch_keys; then
if gpg --with-fingerprint $LATEST.tar.bz2.sig 2>&1 |
$GREP -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888' ; then
if gpg --with-fingerprint $latest.tar.bz2.sig 2>&1 |
$grep -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888' ; then
# Source code signed by Ole Tange <ole@tange.dk>
# KeyID FFFFFFF1/88888888
true
else
# GnuPG signature failed
echo
echo "The signature on $LATEST.tar.bz2 is wrong. This may indicate that a criminal has changed the code."
echo "The signature on $latest.tar.bz2 is wrong. This may indicate that a criminal has changed the code."
echo "THIS IS BAD AND THE CODE WILL NOT BE INSTALLED."
echo
echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods."
@ -119,21 +119,21 @@ run() {
fi
fi
bzip2 -dc $LATEST.tar.bz2 | tar xf -
cd $LATEST || exit 2
bzip2 -dc $latest.tar.bz2 | tar xf -
cd $latest || exit 2
if ./configure && make && make install; then
echo
echo GNU $LATEST installed globally
echo GNU $latest installed globally
else
if ./configure --prefix=$HOME && make && make install; then
echo
echo GNU $LATEST installed in $HOME/bin
echo GNU $latest installed in $HOME/bin
else
mkdir -p $HOME/bin/;
chmod 755 src/*;
cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin;
echo
echo GNU $LATEST copied to $HOME/bin
echo GNU $latest copied to $HOME/bin
fi
# Is $HOME/bin already in $PATH?

4
README
View file

@ -1,7 +1,7 @@
GNU Parallel README
GNU Parallel
Please send problems and feedback to bug-parallel@gnu.org.
https://www.gnu.org/software/parallel/
= Presentation of GNU Parallel =

View file

@ -1,5 +1,11 @@
Quote of the month:
IMHO, SQLite and GNU Parallel are among the world's great software.
-- singe@reddit
Yeah, GNU parallel is a beast when used accordingly.
-- @lsde@twitter
GNU Parallel is awesome
-- Gaurav Jain @gauravjain49@twitter

View file

@ -209,9 +209,9 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20190822 ('Jesper Svarre') released <<[stable]>>
Subject: GNU Parallel 20190822 ('Dorian') released <<[stable]>>
GNU Parallel 20190822 ('Jesper Svarre') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20190822 ('Dorian') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -225,21 +225,28 @@ Quote of the month:
New in this release:
* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it#beskrivelse
* --nice is now inherited by the nice level that GNU Parallel is started at. So 'nice -n10 parallel' will also cause remote jobs to be run at nice level 10.
* Linux Productivity Tools https://www.usenix.org/conference/lisa19/presentation/maheshwari
* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it/tools
* Improve your reconnaissance performance by using GNU Parallel https://mailchi.mp/3f46ceb46357/improve-your-reconnaissance-performance-by-using-gnu-parallel
* Greenland Ice Sheet solid ice discharge from 1986 through 2017 https://www.earth-syst-sci-data.net/11/769/2019/essd-11-769-2019.pdf
* Paralel İşler / İşlemler / Programlar Bashde Nasıl Çalıştırılır? https://kaleinfo.com/2019/07/29/paralel-isler-islemler-programlar-bashde-nasil-calistirilir-2/
https://dualuse.io/blog/curryfinger/
* Use POLYBAR to change WALLPAPER with GNU Parallel https://www.youtube.com/watch?v=rjYg-b_eiRU
https://wiki.cac.washington.edu/display/hyakusers/Hyak+parallel-sql
* Finding the Balance Between Speed & Accuracy During an Internet-wide Port Scanning https://captmeelo.com/pentest/2019/07/29/port-scanning.html
https://www.reddit.com/r/perl/comments/cyo2qc/a_meditation_on_analysing_data_with_perl_in/
* GNU Parallel Ryugu released https://linuxreviews.org/GNU_Parallel_Ryugu_released
https://www.eventbrite.ca/e/inscription-accelerer-ses-taches-avec-gnu-parallel-et-les-lots-de-taches-ul-71412840729
* CO2017 - CW1 - Operating Systems & Networks - Shell scripting http://www.7daixie.com/2019071646302396.html
https://curc.readthedocs.io/en/latest/software/GNUParallel.html
https://petelawson.com/post/parallel-in-shell/
https://www.usenix.org/conference/lisa19/presentation/maheshwari
http://atomicer.cn/2018/01/18/gnu-parallel-nfs-%E5%AE%9E%E7%8E%B0%E5%85%B1%E4%BA%AB%E6%96%87%E4%BB%B6%E7%BD%91%E7%BB%9C%E7%9A%84%E5%B9%B6%E5%8F%91%E8%AE%A1%E7%AE%97%E7%AD%89%E6%93%8D%E4%BD%9C/#more
Gnu/parallel + Nfs 实现共享文件网络的并发计算等操作
* Bug fixes and man page updates.

View file

@ -23,7 +23,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20190822;
$Global::version = 20190823;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($opt::version) {

View file

@ -1676,7 +1676,8 @@ sub parse_options(@) {
if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; }
$ENV{'PARALLEL_RSYNC_OPTS'} = $opt::rsync_opts ||
$ENV{'PARALLEL_RSYNC_OPTS'} || '-rlDzR';
$opt::nice ||= 0;
# Default: Same nice level as GNU Parallel is started at
$opt::nice ||= getpriority(0,0) || 0;
if(defined $opt::help) { usage(); exit(0); }
if(defined $opt::embed) { embed(); exit(0); }
if(defined $opt::sqlandworker) {
@ -1929,9 +1930,11 @@ sub parse_options(@) {
# the alternatives instead?
# See a list in: 'man parallel_alternatives'
#
# If you are an honest person please read the above before
# changing this line.
# If you want GNU Parallel to be maintained in the future you should not
# change this line.
citation_notice();
# Seriously: You will be harming free software by removing the notice.
# You make it harder to justify spending time developing it.
parse_halt();
@ -2048,7 +2051,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20190822;
$Global::version = 20190823;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;

View file

@ -1600,13 +1600,18 @@ Overrides an earlier B<--keep-order> (e.g. if set in
B<~/.parallel/config>).
=item B<--nice> I<niceness>
=item B<--nice> I<niceness> (alpha testing)
Run the command at this niceness. For simple commands you can just add
B<nice> in front of the command. But if the command consists of more
sub commands (Like: ls|wc) then prepending B<nice> will not always
work. B<--nice> will make sure all sub commands are niced - even on
remote servers.
Run the command at this niceness.
By default GNU B<parallel> will run jobs at the same nice level as GNU
B<parallel> is started - both on the local machine and remote servers,
so you are unlikely to ever use this option.
Setting B<--nice> will override this nice level. If the nice level is
smaller than the current nice level, it will only affect remote jobs
(e.g. current level is 10 and B<--nice 5> will cause local jobs to be
run at level 10, but remote jobs run at nice level 5).
=item B<--interactive>

View file

@ -141,12 +141,13 @@ B<xargs> deals badly with special characters (such as space, \, ' and
touch 'c:\windows\system32\clfs.sys'
echo 'c:\windows\system32\clfs.sys' | xargs ls -l
You can specify B<-0>, but many input generators are not
optimized for using B<NUL> as separator but are optimized for
B<newline> as separator. E.g B<head>, B<tail>, B<awk>, B<ls>, B<echo>,
B<sed>, B<tar -v>, B<perl> (B<-0> and \0 instead of \n), B<locate>
(requires using B<-0>), B<find> (requires using B<-print0>), B<grep>
(requires user to use B<-z> or B<-Z>), B<sort> (requires using B<-z>).
You can specify B<-0>, but many input generators are not optimized for
using B<NUL> as separator but are optimized for B<newline> as
separator. E.g. B<awk>, B<ls>, B<echo>, B<tar -v>, B<head> (requires
using B<-z>), B<tail> (requires using B<-z>), B<sed> (requires using
B<-z>), B<perl> (B<-0> and \0 instead of \n), B<locate> (requires
using B<-0>), B<find> (requires using B<-print0>), B<grep> (requires
using B<-z> or B<-Z>), B<sort> (requires using B<-z>).
GNU B<parallel>'s newline separation can be emulated with:
@ -467,7 +468,8 @@ B<dxargs> does not deal well with more simultaneous jobs than SSHD's
MaxStartups. B<dxargs> is only built for remote run jobs, but does not
support transferring of files.
https://web.archive.org/web/20120518070250/http://www.semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
https://web.archive.org/web/20120518070250/http://www.
semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
=head2 DIFFERENCES BETWEEN mdm/middleman AND GNU Parallel
@ -475,8 +477,8 @@ https://web.archive.org/web/20120518070250/http://www.semicomplete.com/blog/geek
middleman(mdm) is also a tool for running jobs in parallel.
Here are the shellscripts of
https://web.archive.org/web/20110728064735/http://mdm.berlios.de/usage.html
ported to GNU B<parallel>:
https://web.archive.org/web/20110728064735/http://mdm.
berlios.de/usage.html ported to GNU B<parallel>:
seq 19 | parallel buffon -o - | sort -n > result
cat files | parallel cmd
@ -576,7 +578,8 @@ B</usr/joe>, enter:
apply 'ln %1 /usr/joe' *
parallel ln {} /usr/joe ::: *
https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
https://www-01.ibm.com/support/knowledgecenter/
ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel
@ -877,8 +880,8 @@ If the above is set up you can then do:
orgalorg -o frontend1 -o frontend2 -p -er /tmp -n \
'md5sum /tmp/bigfile' -S bigfile
par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp \
md5sum /tmp/bigfile
par_emul -S frontend1 -S frontend2 --basefile bigfile \
--workdir /tmp md5sum /tmp/bigfile
B<orgalorg> has a progress indicator for the transferring of a
file. GNU B<parallel> does not.
@ -1209,7 +1212,7 @@ processing result with `samtools`:>
$ threads=25
$ ls -d raw.cluster.clean.mapping/* \
| rush -v ref=$ref -v j=$threads -v p='{}/{%}' \
'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz > {p}.sam; \
'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz >{p}.sam;\
samtools view -bS {p}.sam > {p}.bam; \
samtools sort -T {p}.tmp -@ {j} {p}.bam -o {p}.sorted.bam; \
samtools index {p}.sorted.bam; \
@ -1877,7 +1880,8 @@ output. It buffers in RAM. The examples from README.md:
find . -iname '*.jpg' |
machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' |
parallel --timeout 5s --bar mogrify -resize 1200x1200 -filter Lanczos {}
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
find . -iname '*.jpg' -print0 |
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
@ -1889,23 +1893,70 @@ https://github.com/fd0/machma (Last checked: 2019-06)
=head2 DIFFERENCES BETWEEN interlace AND GNU Parallel
Summary table (see legend above):
- I2 I3 I4 - - -
M1 - M3 - - M6
- O2 O3 - - - - x x
E1 E2 - - - - -
- - - - - - - - -
- -
B<interlace> is built for network analysis to run network tools in parallel.
B<interface> does not buffer output, so output from different jobs mixes.
The overhead for each target is O(n*n), so with 1000 targets it
becomes very slow with an overhead in the order of 500ms/target.
Using B<prips> most of the examples from
https://github.com/codingo/Interlace can be run with GNU B<parallel>:
Blocker
commands.txt:
mkdir -p _output_/_target_/scans/
_blocker_
nmap _target_ -oA _output_/_target_/scans/_target_-nmap
interlace -tL ./targets.txt -cL commands.txt -o $output
parallel -a targets.txt \
mkdir -p $output/{}/scans/\; nmap {} -oA $output/{}/scans/{}-nmap
Blocks
commands.txt:
_block:nmap_
mkdir -p _target_/output/scans/
nmap _target_ -oN _target_/output/scans/_target_-nmap
_block:nmap_
nikto --host _target_
interlace -tL ./targets.txt -cL commands.txt
_nmap() {
mkdir -p $1/output/scans/
nmap $1 -oN $1/output/scans/$1-nmap
}
export -f _nmap
parallel ::: _nmap "nikto --host" :::: targets.txt
Run Nikto Over Multiple Sites
interlace -tL ./targets.txt -threads 5 \
-c "nikto --host _target_ > ./_target_-nikto.txt" -v
parallel -a targets.txt -P5 nikto --host {} > ./{}_-nikto.txt
parallel -a targets.txt -P5 nikto --host {} \> ./{}_-nikto.txt
Run Nikto Over Multiple Sites and Ports
interlace -tL ./targets.txt -threads 5 -c \
"nikto --host _target_:_port_ > ./_target_-_port_-nikto.txt" \
-p 80,443 -v
parallel -P5 nikto --host {1}:{2} > ./{1}-{2}-nikto.txt \
parallel -P5 nikto --host {1}:{2} \> ./{1}-{2}-nikto.txt \
:::: targets.txt ::: 80 443
Run a List of Commands against Target Hosts
commands.txt:
nikto --host _target_:_port_ > _output_/_target_-nikto.txt
sslscan _target_:_port_ > _output_/_target_-sslscan.txt
@ -1913,63 +1964,77 @@ https://github.com/codingo/Interlace can be run with GNU B<parallel>:
interlace -t example.com -o ~/Engagements/example/ \
-cL ./commands.txt -p 80,443
_nikto() {
nikto --host "$1:$2"
}
_sslscan() {
sslscan "$1:$2"
}
_testssl() {
testssl.sh "$1:$2"
}
export -f _nikto
export -f _sslscan
export -f _testssl
parallel --results ~/Engagements/example/{2}:{3}{1} \
::: _nikto _sslscan _testssl ::: example.com ::: 80 443
parallel --results ~/Engagements/example/{2}:{3}{1} {1} {2}:{3} \
::: "nikto --host" sslscan testssl.sh ::: example.com ::: 80 443
CIDR notation with an application that doesn't support it
interlace -t 192.168.12.0/24 -c "vhostscan _target_ \
-oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50
prips 192.168.12.0/24 |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Glob notation with an application that doesn't support it
interlace -t 192.168.12.* -c "vhostscan _target_ \
-oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50
# Glob is not supported in prips
prips 192.168.12.0/24 |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Dash (-) notation with an application that doesn't support it
interlace -t 192.168.12.1-15 -c \
"vhostscan _target_ -oN _output_/_target_-vhosts.txt" \
-o ~/scans/ -threads 50
# Dash notation is not supported in prips
prips 192.168.12.1 192.168.12.15 |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Threading Support for an application that doesn't support it
interlace -tL ./target-list.txt -c \
"vhostscan -t _target_ -oN _output_/_target_-vhosts.txt" \
-o ~/scans/ -threads 50
cat ./target-list.txt |
parallel -P50 vhostscan -t {} -oN ~/scans/{}-vhosts.txt
./vhosts-commands.txt -tL ./target-list.txt:
alternatively
./vhosts-commands.txt:
vhostscan -t $target -oN _output_/_target_-vhosts.txt
interlace -cL ./vhosts-commands.txt -tL ./target-list.txt \
-threads 50 -o ~/scans
./vhosts-commands.txt -tL ./target-list.txt:
./vhosts-commands.txt:
vhostscan -t "$1" -oN "$2"
parallel -P50 ./vhosts-commands.txt {} ~/scans/{} \
parallel -P50 ./vhosts-commands.txt {} ~/scans/{}-vhosts.txt \
:::: ./target-list.txt
Exclusions
interlace -t 192.168.12.0/24 -e 192.168.12.0/26 -c \
"vhostscan _target_ -oN _output_/_target_-vhosts.txt" \
-o ~/scans/ -threads 50
prips 192.168.12.0/24 | grep -xv -Ff <(prips 192.168.12.0/26) |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Run Nikto Using Multiple Proxies
https://github.com/codingo/Interlace (Last checked: 2019-02)
interlace -tL ./targets.txt -pL ./proxies.txt -threads 5 -c \
"nikto --host _target_:_port_ -useproxy _proxy_ > \
./_target_-_port_-nikto.txt" -p 80,443 -v
parallel -j5 \
"nikto --host {1}:{2} -useproxy {3} > ./{1}-{2}-nikto.txt" \
:::: ./targets.txt ::: 80 443 :::: ./proxies.txt
https://github.com/codingo/Interlace (Last checked: 2019-09)
=head2 DIFFERENCES BETWEEN otonvm Parallel AND GNU Parallel

View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20190822;
$Global::version = 20190823;
$Global::progname = 'sql';
# This must be done first as this may exec myself