mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-10-18 04:49:40 +00:00
parallel: Faster --group-by. -vv --joblog / --onall --ssh bugfix.
This commit is contained in:
Ole Tange
parent
73affdb759
commit
53d688077c
|
|
@ -1,26 +1,37 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (C) 2013-2019 Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This script downloads the latest version of GNU Parallel, checks
|
||||
# the signature and installs it.
|
||||
#
|
||||
# It first tries to install it globally.
|
||||
# If that fails, it does a personal installation.
|
||||
# If that fails, it copies to $HOME/bin
|
||||
#
|
||||
# You can download and run the script directly by:
|
||||
# $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
|
||||
# fetch -o - http://pi.dk/3) > install.sh
|
||||
# $ sha1sum install.sh
|
||||
# $ md5sum install.sh
|
||||
# $ sha512sum install.sh
|
||||
# $ bash install.sh
|
||||
true <<'EOF'
|
||||
<p>
|
||||
Copyright (C) 2013-2020 Ole Tange and Free Software Foundation, Inc.
|
||||
<p>
|
||||
This program is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU General Public License as
|
||||
published by the Free Software Foundation; either version 3 of
|
||||
the License, or (at your option) any later version.
|
||||
<p>
|
||||
This script downloads the latest version of GNU Parallel, checks
|
||||
the signature and installs it.
|
||||
|
||||
<p> It first tries to install it globally.
|
||||
<br> If that fails, it does a personal installation.
|
||||
<br> If that fails, it copies to $HOME/bin
|
||||
|
||||
<p> You can download and run the script directly by:
|
||||
|
||||
<p> $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 ||
|
||||
curl pi.dk/3/ || fetch -o - http://pi.dk/3) > install.sh
|
||||
<br> $ sha1sum install.sh | grep 12345678
|
||||
<br> $ md5sum install.sh
|
||||
<br> $ sha512sum install.sh
|
||||
|
||||
<p> Check the sums from https://gnu.org/s/parallel/checksums
|
||||
<br> Then run:
|
||||
|
||||
<p> $ bash install.sh
|
||||
|
||||
|
||||
<br><br><br><br>
|
||||
EOF
|
||||
|
||||
run() {
|
||||
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
|
||||
|
|
@ -82,8 +93,8 @@ run() {
|
|||
fi
|
||||
else
|
||||
# GnuPG not installed
|
||||
echo
|
||||
echo "GnuPG (gpg) is not installed so the signature cannot be checked."
|
||||
echo
|
||||
echo "GnuPG (gpg) is not installed so the signature cannot be checked."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
|
@ -94,7 +105,7 @@ run() {
|
|||
perl -e 'exit not grep /^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888/, <>'; then
|
||||
# Source code signed by Ole Tange <ole@tange.dk>
|
||||
# KeyID FFFFFFF1/88888888
|
||||
true
|
||||
true
|
||||
else
|
||||
# GnuPG signature failed
|
||||
echo
|
||||
|
|
@ -103,20 +114,20 @@ run() {
|
|||
echo
|
||||
echo "See http://git.savannah.gnu.org/cgit/parallel.git/tree/README for other installation methods."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# GnuPG not installed or public keys not downloaded
|
||||
echo "This means that if the code has been changed by criminals, you will not discover that!"
|
||||
echo
|
||||
echo "Continue anyway? (y/n)"
|
||||
read YN </dev/tty
|
||||
read YN </dev/tty
|
||||
if test "$YN" = "n"; then
|
||||
# Stop
|
||||
exit 2
|
||||
else
|
||||
# Continue
|
||||
true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
bzip2 -dc $latest.tar.bz2 | tar xf -
|
||||
|
|
@ -130,7 +141,7 @@ run() {
|
|||
echo GNU $latest installed in $HOME/bin
|
||||
else
|
||||
mkdir -p $HOME/bin/;
|
||||
chmod 755 src/*;
|
||||
chmod 755 src/*;
|
||||
cp src/parallel src/env_parallel* src/sem src/sql src/niceload src/parcat $HOME/bin;
|
||||
echo
|
||||
echo GNU $latest copied to $HOME/bin
|
||||
|
|
@ -144,7 +155,7 @@ run() {
|
|||
# Add $HOME/bin to $PATH for both bash and csh
|
||||
echo 'PATH=$PATH:$HOME/bin' >> $HOME/.bashrc
|
||||
echo 'setenv PATH ${PATH}:${HOME}/bin' >> $HOME/.cshrc
|
||||
fi
|
||||
fi
|
||||
|
||||
# Is $HOME/share/man already in $MANPATH?
|
||||
if echo $MANPATH | grep $HOME/share/man >/dev/null; then
|
||||
|
|
|
|||
14
README
14
README
|
|
@ -40,13 +40,13 @@ installation.
|
|||
|
||||
$ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
|
||||
fetch -o - http://pi.dk/3 ) > install.sh
|
||||
$ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9
|
||||
12345678 3374ec53 bacb199b 245af2dd a86df6c9
|
||||
$ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca
|
||||
029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
|
||||
$ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b
|
||||
40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
|
||||
60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
|
||||
$ sha1sum install.sh | grep 67bd7bc7dc20aff99eb8f1266574dadb
|
||||
12345678 67bd7bc7 dc20aff9 9eb8f126 6574dadb
|
||||
$ md5sum install.sh | grep b7a15cdbb07fb6e11b0338577bc1780f
|
||||
b7a15cdb b07fb6e1 1b033857 7bc1780f
|
||||
$ sha512sum install.sh | grep 186000b62b66969d7506ca4f885e0c80e02a22444
|
||||
6f25960b d4b90cf6 ba5b76de c1acdf39 f3d24249 72930394 a4164351 93a7668d
|
||||
21ff9839 6f920be5 186000b6 2b66969d 7506ca4f 885e0c80 e02a2244 40e8a43f
|
||||
$ bash install.sh
|
||||
|
||||
This will literally install faster than reading the rest of this
|
||||
|
|
|
|||
|
|
@ -192,34 +192,32 @@ from:tange@gnu.org
|
|||
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
Subject: GNU Parallel 20200922 ('Ginsburg') released <<[stable]>>
|
||||
Subject: GNU Parallel 20201022 ('CRISPR/Memphis-mumier/TrumpVirus/Belarus/Assange') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20200922 ('Ginsburg') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
GNU Parallel 20201022 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
|
||||
<<No new functionality was introduced so this is a good candidate for a stable release.>>
|
||||
|
||||
Quote of the month:
|
||||
|
||||
Great tool, gets jobs done fast.
|
||||
Great tool, gets jobs done fast.
|
||||
Great tool, gets jobs done fast.
|
||||
-- Paul F. De La Cruz @pdelacruzcc@twitter
|
||||
|
||||
New in this release:
|
||||
|
||||
* New CPU detection for GNU/Linux.
|
||||
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
News about GNU Parallel:
|
||||
|
||||
* Aug 21, 2020 - More Unix tools https://floki.blog/2020/08/more_unix_tools
|
||||
https://cloud.tencent.com/developer/article/1705499
|
||||
|
||||
|
||||
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
||||
If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include command that uses GNU Parallel if you feel like it.
|
||||
|
||||
|
||||
= About GNU Parallel =
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20200922;
|
||||
$Global::version = 20200923;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
26
src/parallel
26
src/parallel
|
|
@ -881,16 +881,17 @@ sub group_by_loop($$) {
|
|||
# Numbered 0..n-1 due to being used by $F[n]
|
||||
if($group_by::col) { $group_by::col--; }
|
||||
|
||||
my $loop = ::spacefree(0,'{
|
||||
local $_=COLVALUE;
|
||||
PERLEXPR;
|
||||
if(! defined $last) { $last = $_ }
|
||||
if(($last) ne $_) {
|
||||
print "RECSEP";
|
||||
$last = $_;
|
||||
my $loop = ::spacefree(0,q{
|
||||
BEGIN{ $last = "RECSEP"; }
|
||||
{
|
||||
local $_=COLVALUE;
|
||||
PERLEXPR;
|
||||
if(($last) ne $_) {
|
||||
print "RECSEP";
|
||||
$last = $_;
|
||||
}
|
||||
}
|
||||
}');
|
||||
|
||||
});
|
||||
if(defined $group_by::col) {
|
||||
$loop =~ s/COLVALUE/\$F[$group_by::col]/g;
|
||||
} else {
|
||||
|
|
@ -2160,7 +2161,7 @@ sub check_invalid_option_combinations() {
|
|||
|
||||
sub init_globals() {
|
||||
# Defaults:
|
||||
$Global::version = 20200922;
|
||||
$Global::version = 20200923;
|
||||
$Global::progname = 'parallel';
|
||||
$::name = "GNU Parallel";
|
||||
$Global::infinity = 2**31;
|
||||
|
|
@ -4554,6 +4555,7 @@ sub onall($@) {
|
|||
((defined $opt::retries) ? "--retries ".$opt::retries : ""),
|
||||
((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""),
|
||||
((defined $opt::ungroup) ? "-u" : ""),
|
||||
((defined $opt::ssh) ? "--ssh '".$opt::ssh."'" : ""),
|
||||
((defined $opt::tee) ? "--tee" : ""),
|
||||
((defined $opt::workdir) ? "--wd ".Q($opt::workdir) : ""),
|
||||
(@Global::transfer_files ? map { "--tf ".Q($_) }
|
||||
|
|
@ -4913,7 +4915,7 @@ sub usage() {
|
|||
"--recend str Record end separator for --pipe.",
|
||||
"--recstart str Record start separator for --pipe.",
|
||||
"",
|
||||
"See 'man $Global::progname' for details",
|
||||
"GNU Parallel can do much more. See 'man $Global::progname' for details",
|
||||
"",
|
||||
"Academic tradition requires you to cite works you base your article on.",
|
||||
"If you use programs that use GNU Parallel to process data for an article in a",
|
||||
|
|
@ -10330,7 +10332,7 @@ sub print_joblog($) {
|
|||
$cmd = $self->replaced();
|
||||
} else {
|
||||
# Verbose level > 1: Print the rsync and stuff
|
||||
$cmd = join " ", @{$self->{'commandline'}};
|
||||
$cmd = $self->wrapped();
|
||||
}
|
||||
# Newlines make it hard to parse the joblog
|
||||
$cmd =~ s/\n/\0/g;
|
||||
|
|
|
|||
|
|
@ -4056,6 +4056,14 @@ output.
|
|||
If you have a lot of hosts use '-j0' to access more hosts in parallel.
|
||||
|
||||
|
||||
=head1 EXAMPLE: Running 'sudo' on remote computers
|
||||
|
||||
Put the password into passwordfile then run:
|
||||
|
||||
parallel --ssh 'cat passwordfile | ssh' --nonall \
|
||||
-S user@server1,user@server2 sudo -S ls -l /root
|
||||
|
||||
|
||||
=head1 EXAMPLE: Using remote computers behind NAT wall
|
||||
|
||||
If the workers are behind a NAT wall, you need some trickery to get to
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20200922;
|
||||
$Global::version = 20200923;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ EOF
|
|||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### -f and --factor'
|
||||
niceload -H --factor 10 -l6 echo factor 10 finish last
|
||||
niceload -H --factor 30 -l6 echo factor 30 finish last
|
||||
niceload -H -f 0.01 -l6 echo f 0.1 finish first
|
||||
EOF
|
||||
|
||||
|
|
|
|||
|
|
@ -546,6 +546,7 @@ par_retries_all_fail() {
|
|||
par_sockets_cores_threads() {
|
||||
echo '### Test --number-of-sockets/cores/threads'
|
||||
unset PARALLEL_CPUINFO
|
||||
unset PARALLEL_LSCPU
|
||||
parallel --number-of-sockets
|
||||
parallel --number-of-cores
|
||||
parallel --number-of-threads
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
(
|
||||
cd vagrant/tange/centos3/
|
||||
vagrant up
|
||||
)
|
||||
|
||||
par_warning_on_centos3() {
|
||||
echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error"
|
||||
testone() {
|
||||
|
|
@ -20,3 +25,8 @@ export -f $(compgen -A function | grep par_)
|
|||
compgen -A function | grep par_ | LC_ALL=C sort |
|
||||
parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
|
||||
perl -pe 's:/usr/bin:/bin:g;'
|
||||
|
||||
(
|
||||
cd vagrant/tange/centos3/
|
||||
vagrant suspend
|
||||
)
|
||||
|
|
|
|||
|
|
@ -251,7 +251,7 @@ par_jobs_file --pipe Split stdin (standard input) to multiple jobs.
|
|||
par_jobs_file --recend str Record end separator for --pipe.
|
||||
par_jobs_file --recstart str Record start separator for --pipe.
|
||||
par_jobs_file
|
||||
par_jobs_file See 'man parallel' for details
|
||||
par_jobs_file GNU Parallel can do much more. See 'man parallel' for details
|
||||
par_jobs_file
|
||||
par_jobs_file Academic tradition requires you to cite works you base your article on.
|
||||
par_jobs_file If you use programs that use GNU Parallel to process data for an article in a
|
||||
|
|
|
|||
|
|
@ -439,7 +439,7 @@ par_tcsh_man --pipe Split stdin (standard input) to multiple jobs.
par_
|
|||
par_tcsh_man --recend str Record end separator for --pipe.
par_tcsh_man
|
||||
par_tcsh_man --recstart str Record start separator for --pipe.
par_tcsh_man
|
||||
par_tcsh_man
par_tcsh_man
|
||||
par_tcsh_man See 'man parallel' for details
par_tcsh_man
|
||||
par_tcsh_man GNU Parallel can do much more. See 'man parallel' for details
par_tcsh_man
|
||||
par_tcsh_man
par_tcsh_man
|
||||
par_tcsh_man Academic tradition requires you to cite works you base your article on.
par_tcsh_man
|
||||
par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a
par_tcsh_man
|
||||
|
|
@ -1856,9 +1856,9 @@ par_csh_man -X Multiple arguments with context replace
|
|||
par_csh_man -j n Run n jobs in parallel
|
||||
par_csh_man -k Keep same order
|
||||
par_csh_man Academic tradition requires you to cite works you base your article on.
|
||||
par_csh_man GNU Parallel can do much more. See 'man parallel' for details
|
||||
par_csh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
par_csh_man If you use programs that use GNU Parallel to process data for an article in a
|
||||
par_csh_man See 'man parallel' for details
|
||||
par_csh_man This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
par_csh_man Unknown option: no-such-option
|
||||
par_csh_man Usage:
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ bin/env_parallel.dash macosx copy macosx bin/env_parallel.dash env_parallel.dash
|
|||
bin/env_parallel.dash redhat copy redhat bin/env_parallel.dash env_parallel.dash
|
||||
bin/env_parallel.dash netbsd copy netbsd bin/env_parallel.dash env_parallel.dash
|
||||
bin/env_parallel.dash openbsd copy openbsd bin/env_parallel.dash env_parallel.dash
|
||||
bin/env_parallel.dash debian copy debian bin/env_parallel.dash env_parallel.dash
|
||||
bin/env_parallel.dash minix copy minix bin/env_parallel.dash env_parallel.dash
|
||||
bin/env_parallel.fish qnx copy qnx bin/env_parallel.fish env_parallel.fish
|
||||
bin/env_parallel.fish pidora copy pidora bin/env_parallel.fish env_parallel.fish
|
||||
|
|
|
|||
Loading…
Reference in a new issue