tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 07:27:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Faster --group-by. -vv --joblog / --onall --ssh bugfix.

Browse source
This commit is contained in:
Ole Tange 2020-10-13 01:17:14 +02:00
parent 73affdb759
commit 53d688077c
13 changed files with 93 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
10seconds_install
View file

@ -1,26 +1,37 @@
#!/bin/bash #!/bin/bash
# Copyright (C) 2013-2019 Ole Tange and Free Software Foundation, Inc. true <<'EOF'
# <p>
# This program is free software; you can redistribute it and/or modify Copyright (C) 2013-2020 Ole Tange and Free Software Foundation, Inc.
# it under the terms of the GNU General Public License as published by <p>
# the Free Software Foundation; either version 3 of the License, or This program is free software; you can redistribute it and/or
# (at your option) any later version. modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of
# This script downloads the latest version of GNU Parallel, checks the License, or (at your option) any later version.
# the signature and installs it. <p>
# This script downloads the latest version of GNU Parallel, checks
# It first tries to install it globally. the signature and installs it.
# If that fails, it does a personal installation.
# If that fails, it copies to $HOME/bin <p> It first tries to install it globally.
# <br> If that fails, it does a personal installation.
# You can download and run the script directly by: <br> If that fails, it copies to $HOME/bin
# $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
# fetch -o - http://pi.dk/3) > install.sh <p> You can download and run the script directly by:
# $ sha1sum install.sh
# $ md5sum install.sh <p> $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 ||
# $ sha512sum install.sh curl pi.dk/3/ || fetch -o - http://pi.dk/3) > install.sh
# $ bash install.sh <br> $ sha1sum install.sh | grep 12345678
<br> $ md5sum install.sh
<br> $ sha512sum install.sh
<p> Check the sums from https://gnu.org/s/parallel/checksums
<br> Then run:
<p> $ bash install.sh
<br><br><br><br>
EOF
run() { run() {
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget' # FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
@ -82,8 +93,8 @@ run() {
fi fi
else else
# GnuPG not installed # GnuPG not installed
echo echo
echo "GnuPG (gpg) is not installed so the signature cannot be checked." echo "GnuPG (gpg) is not installed so the signature cannot be checked."
return 1 return 1
fi fi
} }
@ -94,7 +105,7 @@ run() {
perl -e 'exit not grep /^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888/, <>'; then perl -e 'exit not grep /^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888/, <>'; then
# Source code signed by Ole Tange <ole@tange.dk> # Source code signed by Ole Tange <ole@tange.dk>
# KeyID FFFFFFF1/88888888 # KeyID FFFFFFF1/88888888
true true
else else
# GnuPG signature failed # GnuPG signature failed
echo echo
@ -103,20 +114,20 @@ run() {
echo echo
echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods." echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods."
exit 1 exit 1
fi fi
else else
# GnuPG not installed or public keys not downloaded # GnuPG not installed or public keys not downloaded
echo "This means that if the code has been changed by criminals, you will not discover that!" echo "This means that if the code has been changed by criminals, you will not discover that!"
echo echo
echo "Continue anyway? (y/n)" echo "Continue anyway? (y/n)"
read YN </dev/tty read YN </dev/tty
if test "$YN" = "n"; then if test "$YN" = "n"; then
# Stop # Stop
exit 2 exit 2
else else
# Continue # Continue
true true
fi fi
fi fi
bzip2 -dc $latest.tar.bz2 | tar xf - bzip2 -dc $latest.tar.bz2 | tar xf -
@ -130,7 +141,7 @@ run() {
echo GNU $latest installed in $HOME/bin echo GNU $latest installed in $HOME/bin
else else
mkdir -p $HOME/bin/; mkdir -p $HOME/bin/;
chmod 755 src/*; chmod 755 src/*;
cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin; cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin;
echo echo
echo GNU $latest copied to $HOME/bin echo GNU $latest copied to $HOME/bin
@ -144,7 +155,7 @@ run() {
# Add $HOME/bin to $PATH for both bash and csh # Add $HOME/bin to $PATH for both bash and csh
echo 'PATH=$PATH:$HOME/bin' >> $HOME/.bashrc echo 'PATH=$PATH:$HOME/bin' >> $HOME/.bashrc
echo 'setenv PATH ${PATH}:${HOME}/bin' >> $HOME/.cshrc echo 'setenv PATH ${PATH}:${HOME}/bin' >> $HOME/.cshrc
fi fi
# Is $HOME/share/man already in $MANPATH? # Is $HOME/share/man already in $MANPATH?
if echo $MANPATH | grep $HOME/share/man >/dev/null; then if echo $MANPATH | grep $HOME/share/man >/dev/null; then

14
README
View file

@ -40,13 +40,13 @@ installation.
$ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \ $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
fetch -o - http://pi.dk/3 ) > install.sh fetch -o - http://pi.dk/3 ) > install.sh
$ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9 $ sha1sum install.sh | grep 67bd7bc7dc20aff99eb8f1266574dadb
12345678 3374ec53 bacb199b 245af2dd a86df6c9 12345678 67bd7bc7 dc20aff9 9eb8f126 6574dadb
$ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca $ md5sum install.sh | grep b7a15cdbb07fb6e11b0338577bc1780f
029a9ac0 6e8b5bc6 052eac57 b2c3c9ca b7a15cdb b07fb6e1 1b033857 7bc1780f
$ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b $ sha512sum install.sh | grep 186000b62b66969d7506ca4f885e0c80e02a22444
40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4 6f25960b d4b90cf6 ba5b76de c1acdf39 f3d24249 72930394 a4164351 93a7668d
60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb 21ff9839 6f920be5 186000b6 2b66969d 7506ca4f 885e0c80 e02a2244 40e8a43f
$ bash install.sh $ bash install.sh
This will literally install faster than reading the rest of this This will literally install faster than reading the rest of this

14
doc/release_new_version
View file

@ -192,34 +192,32 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20200922 ('Ginsburg') released <<[stable]>> Subject: GNU Parallel 20201022 ('CRISPR/Memphis-mumier/TrumpVirus/Belarus/Assange') released <<[stable]>>
GNU Parallel 20200922 ('Ginsburg') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20201022 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
Quote of the month: Quote of the month:
Great tool, gets jobs done fast.
Great tool, gets jobs done fast.
Great tool, gets jobs done fast.
-- Paul F. De La Cruz @pdelacruzcc@twitter
New in this release: New in this release:
* New CPU detection for GNU/Linux.
* Bug fixes and man page updates. * Bug fixes and man page updates.
News about GNU Parallel: News about GNU Parallel:
* Aug 21, 2020 - More Unix tools https://floki.blog/2020/08/more_unix_tools https://cloud.tencent.com/developer/article/1705499
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
GNU Parallel - For people who live life in the parallel lane. GNU Parallel - For people who live life in the parallel lane.
If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include command that uses GNU Parallel if you feel like it.
= About GNU Parallel = = About GNU Parallel =

2
src/niceload
View file

@ -23,7 +23,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20200922; $Global::version = 20200923;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

26
src/parallel
View file

@ -881,16 +881,17 @@ sub group_by_loop($$) {
# Numbered 0..n-1 due to being used by $F[n] # Numbered 0..n-1 due to being used by $F[n]
if($group_by::col) { $group_by::col--; } if($group_by::col) { $group_by::col--; }
my $loop = ::spacefree(0,'{ my $loop = ::spacefree(0,q{
local $_=COLVALUE; BEGIN{ $last = "RECSEP"; }
PERLEXPR; {
if(! defined $last) { $last = $_ } local $_=COLVALUE;
if(($last) ne $_) { PERLEXPR;
print "RECSEP"; if(($last) ne $_) {
$last = $_; print "RECSEP";
$last = $_;
}
} }
}'); });
if(defined $group_by::col) { if(defined $group_by::col) {
$loop =~ s/COLVALUE/\$F[$group_by::col]/g; $loop =~ s/COLVALUE/\$F[$group_by::col]/g;
} else { } else {
@ -2160,7 +2161,7 @@ sub check_invalid_option_combinations() {
sub init_globals() { sub init_globals() {
# Defaults: # Defaults:
$Global::version = 20200922; $Global::version = 20200923;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$::name = "GNU Parallel"; $::name = "GNU Parallel";
$Global::infinity = 2**31; $Global::infinity = 2**31;
@ -4554,6 +4555,7 @@ sub onall($@) {
((defined $opt::retries) ? "--retries ".$opt::retries : ""), ((defined $opt::retries) ? "--retries ".$opt::retries : ""),
((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""), ((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""),
((defined $opt::ungroup) ? "-u" : ""), ((defined $opt::ungroup) ? "-u" : ""),
((defined $opt::ssh) ? "--ssh '".$opt::ssh."'" : ""),
((defined $opt::tee) ? "--tee" : ""), ((defined $opt::tee) ? "--tee" : ""),
((defined $opt::workdir) ? "--wd ".Q($opt::workdir) : ""), ((defined $opt::workdir) ? "--wd ".Q($opt::workdir) : ""),
(@Global::transfer_files ? map { "--tf ".Q($_) } (@Global::transfer_files ? map { "--tf ".Q($_) }
@ -4913,7 +4915,7 @@ sub usage() {
"--recend str Record end separator for --pipe.", "--recend str Record end separator for --pipe.",
"--recstart str Record start separator for --pipe.", "--recstart str Record start separator for --pipe.",
"", "",
"See 'man $Global::progname' for details", "GNU Parallel can do much more. See 'man $Global::progname' for details",
"", "",
"Academic tradition requires you to cite works you base your article on.", "Academic tradition requires you to cite works you base your article on.",
"If you use programs that use GNU Parallel to process data for an article in a", "If you use programs that use GNU Parallel to process data for an article in a",
@ -10330,7 +10332,7 @@ sub print_joblog($) {
$cmd = $self->replaced(); $cmd = $self->replaced();
} else { } else {
# Verbose level > 1: Print the rsync and stuff # Verbose level > 1: Print the rsync and stuff
$cmd = join " ", @{$self->{'commandline'}}; $cmd = $self->wrapped();
} }
# Newlines make it hard to parse the joblog # Newlines make it hard to parse the joblog
$cmd =~ s/\n/\0/g; $cmd =~ s/\n/\0/g;

8
src/parallel.pod
View file

@ -4056,6 +4056,14 @@ output.
If you have a lot of hosts use '-j0' to access more hosts in parallel. If you have a lot of hosts use '-j0' to access more hosts in parallel.
=head1 EXAMPLE: Running 'sudo' on remote computers
Put the password into passwordfile then run:
parallel --ssh 'cat passwordfile | ssh' --nonall \
-S user@server1,user@server2 sudo -S ls -l /root
=head1 EXAMPLE: Using remote computers behind NAT wall =head1 EXAMPLE: Using remote computers behind NAT wall
If the workers are behind a NAT wall, you need some trickery to get to If the workers are behind a NAT wall, you need some trickery to get to

2
src/sql
View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20200922; $Global::version = 20200923;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

2
testsuite/tests-to-run/niceload01.sh
View file

@ -44,7 +44,7 @@ EOF
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --joblog /tmp/jl-`basename $0` -L1 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --joblog /tmp/jl-`basename $0` -L1
echo '### -f and --factor' echo '### -f and --factor'
niceload -H --factor 10 -l6 echo factor 10 finish last niceload -H --factor 30 -l6 echo factor 30 finish last
niceload -H -f 0.01 -l6 echo f 0.1 finish first niceload -H -f 0.01 -l6 echo f 0.1 finish first
EOF EOF

1
testsuite/tests-to-run/parallel-local-10s.sh
View file

@ -546,6 +546,7 @@ par_retries_all_fail() {
par_sockets_cores_threads() { par_sockets_cores_threads() {
echo '### Test --number-of-sockets/cores/threads' echo '### Test --number-of-sockets/cores/threads'
unset PARALLEL_CPUINFO unset PARALLEL_CPUINFO
unset PARALLEL_LSCPU
parallel --number-of-sockets parallel --number-of-sockets
parallel --number-of-cores parallel --number-of-cores
parallel --number-of-threads parallel --number-of-threads

10
testsuite/tests-to-run/parallel-virtualbox1.sh
View file

@ -1,5 +1,10 @@
#!/bin/bash #!/bin/bash
(
cd vagrant/tange/centos3/
vagrant up
)
par_warning_on_centos3() { par_warning_on_centos3() {
echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error" echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error"
testone() { testone() {
@ -20,3 +25,8 @@ export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort | compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
perl -pe 's:/usr/bin:/bin:g;' perl -pe 's:/usr/bin:/bin:g;'
(
cd vagrant/tange/centos3/
vagrant suspend
)

2
testsuite/wanted-results/parallel-local-10s
View file

@ -251,7 +251,7 @@ par_jobs_file --pipe Split stdin (standard input) to multiple jobs.
par_jobs_file --recend str Record end separator for --pipe. par_jobs_file --recend str Record end separator for --pipe.
par_jobs_file --recstart str Record start separator for --pipe. par_jobs_file --recstart str Record start separator for --pipe.
par_jobs_file par_jobs_file
par_jobs_file See 'man parallel' for details par_jobs_file GNU Parallel can do much more. See 'man parallel' for details
par_jobs_file par_jobs_file
par_jobs_file Academic tradition requires you to cite works you base your article on. par_jobs_file Academic tradition requires you to cite works you base your article on.
par_jobs_file If you use programs that use GNU Parallel to process data for an article in a par_jobs_file If you use programs that use GNU Parallel to process data for an article in a

4
testsuite/wanted-results/parallel-local-ssh7
View file

@ -439,7 +439,7 @@ par_tcsh_man --pipe Split stdin (standard input) to multiple jobs. par_
par_tcsh_man --recend str Record end separator for --pipe. par_tcsh_man par_tcsh_man --recend str Record end separator for --pipe. par_tcsh_man
par_tcsh_man --recstart str Record start separator for --pipe. par_tcsh_man par_tcsh_man --recstart str Record start separator for --pipe. par_tcsh_man
par_tcsh_man par_tcsh_man par_tcsh_man par_tcsh_man
par_tcsh_man See 'man parallel' for details par_tcsh_man par_tcsh_man GNU Parallel can do much more. See 'man parallel' for details par_tcsh_man
par_tcsh_man par_tcsh_man par_tcsh_man par_tcsh_man
par_tcsh_man Academic tradition requires you to cite works you base your article on. par_tcsh_man par_tcsh_man Academic tradition requires you to cite works you base your article on. par_tcsh_man
par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a par_tcsh_man par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a par_tcsh_man
@ -1856,9 +1856,9 @@ par_csh_man -X Multiple arguments with context replace
par_csh_man -j n Run n jobs in parallel par_csh_man -j n Run n jobs in parallel
par_csh_man -k Keep same order par_csh_man -k Keep same order
par_csh_man Academic tradition requires you to cite works you base your article on. par_csh_man Academic tradition requires you to cite works you base your article on.
par_csh_man GNU Parallel can do much more. See 'man parallel' for details
par_csh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing. par_csh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
par_csh_man If you use programs that use GNU Parallel to process data for an article in a par_csh_man If you use programs that use GNU Parallel to process data for an article in a
par_csh_man See 'man parallel' for details
par_csh_man This helps funding further development; AND IT WON'T COST YOU A CENT. par_csh_man This helps funding further development; AND IT WON'T COST YOU A CENT.
par_csh_man Unknown option: no-such-option par_csh_man Unknown option: no-such-option
par_csh_man Usage: par_csh_man Usage:

1
testsuite/wanted-results/parallel-polarhome
View file

@ -166,6 +166,7 @@ bin/env_parallel.dash macosx copy macosx bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash redhat copy redhat bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash redhat copy redhat bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash netbsd copy netbsd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash netbsd copy netbsd bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash openbsd copy openbsd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash openbsd copy openbsd bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash debian copy debian bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash minix copy minix bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash minix copy minix bin/env_parallel.dash env_parallel.dash
bin/env_parallel.fish qnx copy qnx bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish qnx copy qnx bin/env_parallel.fish env_parallel.fish
bin/env_parallel.fish pidora copy pidora bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish pidora copy pidora bin/env_parallel.fish env_parallel.fish