parallel: Fixed bug #56873: Auto detect --nice level.

This commit is contained in:
Ole Tange 2019-09-16 23:12:53 +02:00
parent c7851f76e2
commit 512059bf1e
9 changed files with 173 additions and 87 deletions

View file

@ -24,20 +24,20 @@
run() { run() {
# tail on openindiana must be /usr/xpg4/bin/tail # tail on openindiana must be /usr/xpg4/bin/tail
TAIL=$(echo | tail -n 1 2>/dev/null && echo tail || tail=$(echo | tail -n 1 2>/dev/null && echo tail ||
(echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail)) (echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail))
# grep on openindiana must be /usr/xpg4/bin/grep # grep on openindiana must be /usr/xpg4/bin/grep
GREP=$(echo | grep -vE . 2>/dev/null && echo grep || grep=$(echo | grep -vE . 2>/dev/null && echo grep ||
(echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep)) (echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep))
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget' # FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
GET=$( get=$(
(lynx -source /dev/null && echo lynx -source) || (lynx -source /dev/null && echo lynx -source) ||
(fetch -o /dev/null file:///bin/sh && echo fetch -o -) || (fetch -o /dev/null file:///bin/sh && echo fetch -o -) ||
(curl -h >/dev/null && echo curl -L) || (curl -h >/dev/null && echo curl -L) ||
(wget -h >/dev/null && echo wget -qO -) || (wget -h >/dev/null && echo wget -qO -) ||
echo 'No lynx, wget, curl, fetch: Please inform parallel@gnu.org what you use for downloading URLs' >&2 echo 'No lynx, wget, curl, fetch: Please inform parallel@gnu.org what you use for downloading URLs' >&2
) )
if test "$GET" = ""; then if test "$get" = ""; then
exit 1 exit 1
fi fi
@ -47,27 +47,27 @@ run() {
fi fi
LANG=C LANG=C
LATEST=$($GET http://ftpmirror.gnu.org/parallel | latest=$($get http://ftpmirror.gnu.org/parallel |
perl -ne '/.*(parallel-\d{8})/ and print $1."\n"' | perl -ne '/.*(parallel-\d{8})/ and print $1."\n"' |
sort | $TAIL -n1) sort | $tail -n1)
if test \! -e $LATEST.tar.bz2; then if test \! -e $latest.tar.bz2; then
# Source tar does not exist # Source tar does not exist
rm -f $LATEST.tar.bz2 $LATEST.tar.bz2.sig rm -f $latest.tar.bz2 $latest.tar.bz2.sig
$GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2 > $LATEST.tar.bz2 $get http://ftpmirror.gnu.org/parallel/$latest.tar.bz2 > $latest.tar.bz2
$GET http://ftpmirror.gnu.org/parallel/$LATEST.tar.bz2.sig > $LATEST.tar.bz2.sig $get http://ftpmirror.gnu.org/parallel/$latest.tar.bz2.sig > $latest.tar.bz2.sig
fi fi
fetch_keys() { fetch_keys() {
if gpg -h 2>/dev/null >/dev/null ; then if gpg -h 2>/dev/null >/dev/null ; then
# GnuPG installed # GnuPG installed
# Setup .gnupg/gpg.conf if not already done # Setup .gnupg/gpg.conf if not already done
echo | gpg 2>/dev/null >/dev/null echo | gpg 2>/dev/null >/dev/null
keyserver1=keys.gnupg.net keyserver1=keys.gnupg.net
keyserver2=pool.sks-keyservers.net keyserver2=pool.sks-keyservers.net
if gpg --keyserver $keyserver1 --recv-key 0xFFFFFFF1 || if gpg --keyserver $keyserver1 --recv-key 0xFFFFFFF1 ||
gpg --keyserver $keyserver2 --recv-key 0xFFFFFFF1 ; then gpg --keyserver $keyserver2 --recv-key 0xFFFFFFF1 ; then
if gpg --keyserver $keyserver1 --recv-key 0x88888888 || if gpg --keyserver $keyserver1 --recv-key 0x88888888 ||
gpg --keyserver $keyserver2 --recv-key 0x88888888; then gpg --keyserver $keyserver2 --recv-key 0x88888888; then
# OK # OK
return 0 return 0
else else
@ -82,7 +82,7 @@ run() {
fi fi
else else
# GnuPG not installed # GnuPG not installed
echo echo
echo "GnuPG (gpg) is not installed so the signature cannot be checked." echo "GnuPG (gpg) is not installed so the signature cannot be checked."
return 1 return 1
fi fi
@ -90,19 +90,19 @@ run() {
# Check signature - in case ftpmirror.gnu.org is compromised # Check signature - in case ftpmirror.gnu.org is compromised
if fetch_keys; then if fetch_keys; then
if gpg --with-fingerprint $LATEST.tar.bz2.sig 2>&1 | if gpg --with-fingerprint $latest.tar.bz2.sig 2>&1 |
$GREP -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888' ; then $grep -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888' ; then
# Source code signed by Ole Tange <ole@tange.dk> # Source code signed by Ole Tange <ole@tange.dk>
# KeyID FFFFFFF1/88888888 # KeyID FFFFFFF1/88888888
true true
else else
# GnuPG signature failed # GnuPG signature failed
echo echo
echo "The signature on $LATEST.tar.bz2 is wrong. This may indicate that a criminal has changed the code." echo "The signature on $latest.tar.bz2 is wrong. This may indicate that a criminal has changed the code."
echo "THIS IS BAD AND THE CODE WILL NOT BE INSTALLED." echo "THIS IS BAD AND THE CODE WILL NOT BE INSTALLED."
echo echo
echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods." echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods."
exit 1 exit 1
fi fi
else else
# GnuPG not installed or public keys not downloaded # GnuPG not installed or public keys not downloaded
@ -113,27 +113,27 @@ run() {
if test "$YN" = "n"; then if test "$YN" = "n"; then
# Stop # Stop
exit 2 exit 2
else else
# Continue # Continue
true true
fi fi
fi fi
bzip2 -dc $LATEST.tar.bz2 | tar xf - bzip2 -dc $latest.tar.bz2 | tar xf -
cd $LATEST || exit 2 cd $latest || exit 2
if ./configure && make && make install; then if ./configure && make && make install; then
echo echo
echo GNU $LATEST installed globally echo GNU $latest installed globally
else else
if ./configure --prefix=$HOME && make && make install; then if ./configure --prefix=$HOME && make && make install; then
echo echo
echo GNU $LATEST installed in $HOME/bin echo GNU $latest installed in $HOME/bin
else else
mkdir -p $HOME/bin/; mkdir -p $HOME/bin/;
chmod 755 src/*; chmod 755 src/*;
cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin; cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin;
echo echo
echo GNU $LATEST copied to $HOME/bin echo GNU $latest copied to $HOME/bin
fi fi
# Is $HOME/bin already in $PATH? # Is $HOME/bin already in $PATH?
@ -152,9 +152,9 @@ run() {
true true
else else
# Add $HOME/share/man to $MANPATH for both bash and csh # Add $HOME/share/man to $MANPATH for both bash and csh
echo 'MANPATH=$MANPATH:$HOME/share/man' >> $HOME/.bashrc echo 'MANPATH=$MANPATH:$HOME/share/man' >> $HOME/.bashrc
echo 'setenv MANPATH ${MANPATH}:${HOME}/share/man' >> $HOME/.cshrc echo 'setenv MANPATH ${MANPATH}:${HOME}/share/man' >> $HOME/.cshrc
fi fi
fi fi
} }

4
README
View file

@ -1,7 +1,7 @@
GNU Parallel README GNU Parallel
Please send problems and feedback to bug-parallel@gnu.org. https://www.gnu.org/software/parallel/
= Presentation of GNU Parallel = = Presentation of GNU Parallel =

View file

@ -1,5 +1,11 @@
Quote of the month: Quote of the month:
IMHO, SQLite and GNU Parallel are among the world's great software.
-- singe@reddit
Yeah, GNU parallel is a beast when used accordingly.
-- @lsde@twitter
GNU Parallel is awesome GNU Parallel is awesome
-- Gaurav Jain @gauravjain49@twitter -- Gaurav Jain @gauravjain49@twitter

View file

@ -209,9 +209,9 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20190822 ('Jesper Svarre') released <<[stable]>> Subject: GNU Parallel 20190822 ('Dorian') released <<[stable]>>
GNU Parallel 20190822 ('Jesper Svarre') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20190822 ('Dorian') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -225,21 +225,28 @@ Quote of the month:
New in this release: New in this release:
* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it#beskrivelse * --nice is now inherited by the nice level that GNU Parallel is started at. So 'nice -n10 parallel' will also cause remote jobs to be run at nice level 10.
* Linux Productivity Tools https://www.usenix.org/conference/lisa19/presentation/maheshwari * GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it/tools
* Improve your reconnaissance performance by using GNU Parallel https://mailchi.mp/3f46ceb46357/improve-your-reconnaissance-performance-by-using-gnu-parallel * Greenland Ice Sheet solid ice discharge from 1986 through 2017 https://www.earth-syst-sci-data.net/11/769/2019/essd-11-769-2019.pdf
* Paralel İşler / İşlemler / Programlar Bashde Nasıl Çalıştırılır? https://kaleinfo.com/2019/07/29/paralel-isler-islemler-programlar-bashde-nasil-calistirilir-2/ https://dualuse.io/blog/curryfinger/
* Use POLYBAR to change WALLPAPER with GNU Parallel https://www.youtube.com/watch?v=rjYg-b_eiRU https://wiki.cac.washington.edu/display/hyakusers/Hyak+parallel-sql
* Finding the Balance Between Speed & Accuracy During an Internet-wide Port Scanning https://captmeelo.com/pentest/2019/07/29/port-scanning.html https://www.reddit.com/r/perl/comments/cyo2qc/a_meditation_on_analysing_data_with_perl_in/
* GNU Parallel Ryugu released https://linuxreviews.org/GNU_Parallel_Ryugu_released https://www.eventbrite.ca/e/inscription-accelerer-ses-taches-avec-gnu-parallel-et-les-lots-de-taches-ul-71412840729
* CO2017 - CW1 - Operating Systems & Networks - Shell scripting http://www.7daixie.com/2019071646302396.html https://curc.readthedocs.io/en/latest/software/GNUParallel.html
https://petelawson.com/post/parallel-in-shell/
https://www.usenix.org/conference/lisa19/presentation/maheshwari
http://atomicer.cn/2018/01/18/gnu-parallel-nfs-%E5%AE%9E%E7%8E%B0%E5%85%B1%E4%BA%AB%E6%96%87%E4%BB%B6%E7%BD%91%E7%BB%9C%E7%9A%84%E5%B9%B6%E5%8F%91%E8%AE%A1%E7%AE%97%E7%AD%89%E6%93%8D%E4%BD%9C/#more
Gnu/parallel + Nfs 实现共享文件网络的并发计算等操作
* Bug fixes and man page updates. * Bug fixes and man page updates.

View file

@ -23,7 +23,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20190822; $Global::version = 20190823;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

View file

@ -1676,7 +1676,8 @@ sub parse_options(@) {
if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; } if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; }
$ENV{'PARALLEL_RSYNC_OPTS'} = $opt::rsync_opts || $ENV{'PARALLEL_RSYNC_OPTS'} = $opt::rsync_opts ||
$ENV{'PARALLEL_RSYNC_OPTS'} || '-rlDzR'; $ENV{'PARALLEL_RSYNC_OPTS'} || '-rlDzR';
$opt::nice ||= 0; # Default: Same nice level as GNU Parallel is started at
$opt::nice ||= getpriority(0,0) || 0;
if(defined $opt::help) { usage(); exit(0); } if(defined $opt::help) { usage(); exit(0); }
if(defined $opt::embed) { embed(); exit(0); } if(defined $opt::embed) { embed(); exit(0); }
if(defined $opt::sqlandworker) { if(defined $opt::sqlandworker) {
@ -1929,9 +1930,11 @@ sub parse_options(@) {
# the alternatives instead? # the alternatives instead?
# See a list in: 'man parallel_alternatives' # See a list in: 'man parallel_alternatives'
# #
# If you are an honest person please read the above before # If you want GNU Parallel to be maintained in the future you should not
# changing this line. # change this line.
citation_notice(); citation_notice();
# Seriously: You will be harming free software by removing the notice.
# You make it harder to justify spending time developing it.
parse_halt(); parse_halt();
@ -2048,7 +2051,7 @@ sub check_invalid_option_combinations() {
sub init_globals() { sub init_globals() {
# Defaults: # Defaults:
$Global::version = 20190822; $Global::version = 20190823;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;

View file

@ -1600,13 +1600,18 @@ Overrides an earlier B<--keep-order> (e.g. if set in
B<~/.parallel/config>). B<~/.parallel/config>).
=item B<--nice> I<niceness> =item B<--nice> I<niceness> (alpha testing)
Run the command at this niceness. For simple commands you can just add Run the command at this niceness.
B<nice> in front of the command. But if the command consists of more
sub commands (Like: ls|wc) then prepending B<nice> will not always By default GNU B<parallel> will run jobs at the same nice level as GNU
work. B<--nice> will make sure all sub commands are niced - even on B<parallel> is started - both on the local machine and remote servers,
remote servers. so you are unlikely to ever use this option.
Setting B<--nice> will override this nice level. If the nice level is
smaller than the current nice level, it will only affect remote jobs
(e.g. current level is 10 and B<--nice 5> will cause local jobs to be
run at level 10, but remote jobs run at nice level 5).
=item B<--interactive> =item B<--interactive>

View file

@ -141,12 +141,13 @@ B<xargs> deals badly with special characters (such as space, \, ' and
touch 'c:\windows\system32\clfs.sys' touch 'c:\windows\system32\clfs.sys'
echo 'c:\windows\system32\clfs.sys' | xargs ls -l echo 'c:\windows\system32\clfs.sys' | xargs ls -l
You can specify B<-0>, but many input generators are not You can specify B<-0>, but many input generators are not optimized for
optimized for using B<NUL> as separator but are optimized for using B<NUL> as separator but are optimized for B<newline> as
B<newline> as separator. E.g B<head>, B<tail>, B<awk>, B<ls>, B<echo>, separator. E.g. B<awk>, B<ls>, B<echo>, B<tar -v>, B<head> (requires
B<sed>, B<tar -v>, B<perl> (B<-0> and \0 instead of \n), B<locate> using B<-z>), B<tail> (requires using B<-z>), B<sed> (requires using
(requires using B<-0>), B<find> (requires using B<-print0>), B<grep> B<-z>), B<perl> (B<-0> and \0 instead of \n), B<locate> (requires
(requires user to use B<-z> or B<-Z>), B<sort> (requires using B<-z>). using B<-0>), B<find> (requires using B<-print0>), B<grep> (requires
using B<-z> or B<-Z>), B<sort> (requires using B<-z>).
GNU B<parallel>'s newline separation can be emulated with: GNU B<parallel>'s newline separation can be emulated with:
@ -467,7 +468,8 @@ B<dxargs> does not deal well with more simultaneous jobs than SSHD's
MaxStartups. B<dxargs> is only built for remote run jobs, but does not MaxStartups. B<dxargs> is only built for remote run jobs, but does not
support transferring of files. support transferring of files.
https://web.archive.org/web/20120518070250/http://www.semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01) https://web.archive.org/web/20120518070250/http://www.
semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
=head2 DIFFERENCES BETWEEN mdm/middleman AND GNU Parallel =head2 DIFFERENCES BETWEEN mdm/middleman AND GNU Parallel
@ -475,8 +477,8 @@ https://web.archive.org/web/20120518070250/http://www.semicomplete.com/blog/geek
middleman(mdm) is also a tool for running jobs in parallel. middleman(mdm) is also a tool for running jobs in parallel.
Here are the shellscripts of Here are the shellscripts of
https://web.archive.org/web/20110728064735/http://mdm.berlios.de/usage.html https://web.archive.org/web/20110728064735/http://mdm.
ported to GNU B<parallel>: berlios.de/usage.html ported to GNU B<parallel>:
seq 19 | parallel buffon -o - | sort -n > result seq 19 | parallel buffon -o - | sort -n > result
cat files | parallel cmd cat files | parallel cmd
@ -576,7 +578,8 @@ B</usr/joe>, enter:
apply 'ln %1 /usr/joe' * apply 'ln %1 /usr/joe' *
parallel ln {} /usr/joe ::: * parallel ln {} /usr/joe ::: *
https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01) https://www-01.ibm.com/support/knowledgecenter/
ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel =head2 DIFFERENCES BETWEEN paexec AND GNU Parallel
@ -877,8 +880,8 @@ If the above is set up you can then do:
orgalorg -o frontend1 -o frontend2 -p -er /tmp -n \ orgalorg -o frontend1 -o frontend2 -p -er /tmp -n \
'md5sum /tmp/bigfile' -S bigfile 'md5sum /tmp/bigfile' -S bigfile
par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp \ par_emul -S frontend1 -S frontend2 --basefile bigfile \
md5sum /tmp/bigfile --workdir /tmp md5sum /tmp/bigfile
B<orgalorg> has a progress indicator for the transferring of a B<orgalorg> has a progress indicator for the transferring of a
file. GNU B<parallel> does not. file. GNU B<parallel> does not.
@ -1209,7 +1212,7 @@ processing result with `samtools`:>
$ threads=25 $ threads=25
$ ls -d raw.cluster.clean.mapping/* \ $ ls -d raw.cluster.clean.mapping/* \
| rush -v ref=$ref -v j=$threads -v p='{}/{%}' \ | rush -v ref=$ref -v j=$threads -v p='{}/{%}' \
'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz > {p}.sam; \ 'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz >{p}.sam;\
samtools view -bS {p}.sam > {p}.bam; \ samtools view -bS {p}.sam > {p}.bam; \
samtools sort -T {p}.tmp -@ {j} {p}.bam -o {p}.sorted.bam; \ samtools sort -T {p}.tmp -@ {j} {p}.bam -o {p}.sorted.bam; \
samtools index {p}.sorted.bam; \ samtools index {p}.sorted.bam; \
@ -1875,9 +1878,10 @@ output. It buffers in RAM. The examples from README.md:
parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive' parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive'
find . -iname '*.jpg' | find . -iname '*.jpg' |
machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {} machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' | find . -iname '*.jpg' |
parallel --timeout 5s --bar mogrify -resize 1200x1200 -filter Lanczos {} parallel --timeout 5s --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
find . -iname '*.jpg' -print0 | find . -iname '*.jpg' -print0 |
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {} machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
@ -1889,23 +1893,70 @@ https://github.com/fd0/machma (Last checked: 2019-06)
=head2 DIFFERENCES BETWEEN interlace AND GNU Parallel =head2 DIFFERENCES BETWEEN interlace AND GNU Parallel
Summary table (see legend above):
- I2 I3 I4 - - -
M1 - M3 - - M6
- O2 O3 - - - - x x
E1 E2 - - - - -
- - - - - - - - -
- -
B<interlace> is built for network analysis to run network tools in parallel. B<interlace> is built for network analysis to run network tools in parallel.
B<interface> does not buffer output, so output from different jobs mixes. B<interface> does not buffer output, so output from different jobs mixes.
The overhead for each target is O(n*n), so with 1000 targets it
becomes very slow with an overhead in the order of 500ms/target.
Using B<prips> most of the examples from Using B<prips> most of the examples from
https://github.com/codingo/Interlace can be run with GNU B<parallel>: https://github.com/codingo/Interlace can be run with GNU B<parallel>:
Blocker
commands.txt:
mkdir -p _output_/_target_/scans/
_blocker_
nmap _target_ -oA _output_/_target_/scans/_target_-nmap
interlace -tL ./targets.txt -cL commands.txt -o $output
parallel -a targets.txt \
mkdir -p $output/{}/scans/\; nmap {} -oA $output/{}/scans/{}-nmap
Blocks
commands.txt:
_block:nmap_
mkdir -p _target_/output/scans/
nmap _target_ -oN _target_/output/scans/_target_-nmap
_block:nmap_
nikto --host _target_
interlace -tL ./targets.txt -cL commands.txt
_nmap() {
mkdir -p $1/output/scans/
nmap $1 -oN $1/output/scans/$1-nmap
}
export -f _nmap
parallel ::: _nmap "nikto --host" :::: targets.txt
Run Nikto Over Multiple Sites
interlace -tL ./targets.txt -threads 5 \ interlace -tL ./targets.txt -threads 5 \
-c "nikto --host _target_ > ./_target_-nikto.txt" -v -c "nikto --host _target_ > ./_target_-nikto.txt" -v
parallel -a targets.txt -P5 nikto --host {} > ./{}_-nikto.txt
parallel -a targets.txt -P5 nikto --host {} \> ./{}_-nikto.txt
Run Nikto Over Multiple Sites and Ports
interlace -tL ./targets.txt -threads 5 -c \ interlace -tL ./targets.txt -threads 5 -c \
"nikto --host _target_:_port_ > ./_target_-_port_-nikto.txt" \ "nikto --host _target_:_port_ > ./_target_-_port_-nikto.txt" \
-p 80,443 -v -p 80,443 -v
parallel -P5 nikto --host {1}:{2} > ./{1}-{2}-nikto.txt \
parallel -P5 nikto --host {1}:{2} \> ./{1}-{2}-nikto.txt \
:::: targets.txt ::: 80 443 :::: targets.txt ::: 80 443
Run a List of Commands against Target Hosts
commands.txt: commands.txt:
nikto --host _target_:_port_ > _output_/_target_-nikto.txt nikto --host _target_:_port_ > _output_/_target_-nikto.txt
sslscan _target_:_port_ > _output_/_target_-sslscan.txt sslscan _target_:_port_ > _output_/_target_-sslscan.txt
@ -1913,63 +1964,77 @@ https://github.com/codingo/Interlace can be run with GNU B<parallel>:
interlace -t example.com -o ~/Engagements/example/ \ interlace -t example.com -o ~/Engagements/example/ \
-cL ./commands.txt -p 80,443 -cL ./commands.txt -p 80,443
_nikto() { parallel --results ~/Engagements/example/{2}:{3}{1} {1} {2}:{3} \
nikto --host "$1:$2" ::: "nikto --host" sslscan testssl.sh ::: example.com ::: 80 443
}
_sslscan() { CIDR notation with an application that doesn't support it
sslscan "$1:$2"
}
_testssl() {
testssl.sh "$1:$2"
}
export -f _nikto
export -f _sslscan
export -f _testssl
parallel --results ~/Engagements/example/{2}:{3}{1} \
::: _nikto _sslscan _testssl ::: example.com ::: 80 443
interlace -t 192.168.12.0/24 -c "vhostscan _target_ \ interlace -t 192.168.12.0/24 -c "vhostscan _target_ \
-oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50 -oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50
prips 192.168.12.0/24 | prips 192.168.12.0/24 |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Glob notation with an application that doesn't support it
interlace -t 192.168.12.* -c "vhostscan _target_ \ interlace -t 192.168.12.* -c "vhostscan _target_ \
-oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50 -oN _output_/_target_-vhosts.txt" -o ~/scans/ -threads 50
# Glob is not supported in prips # Glob is not supported in prips
prips 192.168.12.0/24 | prips 192.168.12.0/24 |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Dash (-) notation with an application that doesn't support it
interlace -t 192.168.12.1-15 -c \ interlace -t 192.168.12.1-15 -c \
"vhostscan _target_ -oN _output_/_target_-vhosts.txt" \ "vhostscan _target_ -oN _output_/_target_-vhosts.txt" \
-o ~/scans/ -threads 50 -o ~/scans/ -threads 50
# Dash notation is not supported in prips # Dash notation is not supported in prips
prips 192.168.12.1 192.168.12.15 | prips 192.168.12.1 192.168.12.15 |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Threading Support for an application that doesn't support it
interlace -tL ./target-list.txt -c \ interlace -tL ./target-list.txt -c \
"vhostscan -t _target_ -oN _output_/_target_-vhosts.txt" \ "vhostscan -t _target_ -oN _output_/_target_-vhosts.txt" \
-o ~/scans/ -threads 50 -o ~/scans/ -threads 50
cat ./target-list.txt | cat ./target-list.txt |
parallel -P50 vhostscan -t {} -oN ~/scans/{}-vhosts.txt parallel -P50 vhostscan -t {} -oN ~/scans/{}-vhosts.txt
./vhosts-commands.txt -tL ./target-list.txt: alternatively
./vhosts-commands.txt:
vhostscan -t $target -oN _output_/_target_-vhosts.txt vhostscan -t $target -oN _output_/_target_-vhosts.txt
interlace -cL ./vhosts-commands.txt -tL ./target-list.txt \ interlace -cL ./vhosts-commands.txt -tL ./target-list.txt \
-threads 50 -o ~/scans -threads 50 -o ~/scans
./vhosts-commands.txt -tL ./target-list.txt: ./vhosts-commands.txt:
vhostscan -t "$1" -oN "$2" vhostscan -t "$1" -oN "$2"
parallel -P50 ./vhosts-commands.txt {} ~/scans/{} \ parallel -P50 ./vhosts-commands.txt {} ~/scans/{}-vhosts.txt \
:::: ./target-list.txt :::: ./target-list.txt
Exclusions
interlace -t 192.168.12.0/24 -e 192.168.12.0/26 -c \ interlace -t 192.168.12.0/24 -e 192.168.12.0/26 -c \
"vhostscan _target_ -oN _output_/_target_-vhosts.txt" \ "vhostscan _target_ -oN _output_/_target_-vhosts.txt" \
-o ~/scans/ -threads 50 -o ~/scans/ -threads 50
prips 192.168.12.0/24 | grep -xv -Ff <(prips 192.168.12.0/26) | prips 192.168.12.0/24 | grep -xv -Ff <(prips 192.168.12.0/26) |
parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt parallel -P50 vhostscan {} -oN ~/scans/{}-vhosts.txt
Run Nikto Using Multiple Proxies
https://github.com/codingo/Interlace (Last checked: 2019-02) interlace -tL ./targets.txt -pL ./proxies.txt -threads 5 -c \
"nikto --host _target_:_port_ -useproxy _proxy_ > \
./_target_-_port_-nikto.txt" -p 80,443 -v
parallel -j5 \
"nikto --host {1}:{2} -useproxy {3} > ./{1}-{2}-nikto.txt" \
:::: ./targets.txt ::: 80 443 :::: ./proxies.txt
https://github.com/codingo/Interlace (Last checked: 2019-09)
=head2 DIFFERENCES BETWEEN otonvm Parallel AND GNU Parallel =head2 DIFFERENCES BETWEEN otonvm Parallel AND GNU Parallel

View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20190822; $Global::version = 20190823;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself