tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-21 21:47:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Faster --group-by. -vv --joblog / --onall --ssh bugfix.

Browse source
This commit is contained in:
Ole Tange 2020-10-13 01:17:14 +02:00
parent 73affdb759
commit 53d688077c
13 changed files with 93 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
10seconds_install
View file

@ -1,26 +1,37 @@
#!/bin/bash
# Copyright (C) 2013-2019 Ole Tange and Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This script downloads the latest version of GNU Parallel, checks
# the signature and installs it.
#
# It first tries to install it globally.
# If that fails, it does a personal installation.
# If that fails, it copies to $HOME/bin
#
# You can download and run the script directly by:
# $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
# fetch -o - http://pi.dk/3) > install.sh
# $ sha1sum install.sh
# $ md5sum install.sh
# $ sha512sum install.sh
# $ bash install.sh
true <<'EOF'
<p>
Copyright (C) 2013-2020 Ole Tange and Free Software Foundation, Inc.
<p>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 3 of
the License, or (at your option) any later version.
<p>
This script downloads the latest version of GNU Parallel, checks
the signature and installs it.
<p> It first tries to install it globally.
<br> If that fails, it does a personal installation.
<br> If that fails, it copies to $HOME/bin
<p> You can download and run the script directly by:
<p> $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 ||
curl pi.dk/3/ || fetch -o - http://pi.dk/3) > install.sh
<br> $ sha1sum install.sh | grep 12345678
<br> $ md5sum install.sh
<br> $ sha512sum install.sh
<p> Check the sums from https://gnu.org/s/parallel/checksums
<br> Then run:
<p> $ bash install.sh
<br><br><br><br>
EOF
run() {
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'

14
README
View file

@ -40,13 +40,13 @@ installation.
$ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
fetch -o - http://pi.dk/3 ) > install.sh
$ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9
12345678 3374ec53 bacb199b 245af2dd a86df6c9
$ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca
029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
$ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b
40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
$ sha1sum install.sh | grep 67bd7bc7dc20aff99eb8f1266574dadb
12345678 67bd7bc7 dc20aff9 9eb8f126 6574dadb
$ md5sum install.sh | grep b7a15cdbb07fb6e11b0338577bc1780f
b7a15cdb b07fb6e1 1b033857 7bc1780f
$ sha512sum install.sh | grep 186000b62b66969d7506ca4f885e0c80e02a22444
6f25960b d4b90cf6 ba5b76de c1acdf39 f3d24249 72930394 a4164351 93a7668d
21ff9839 6f920be5 186000b6 2b66969d 7506ca4f 885e0c80 e02a2244 40e8a43f
$ bash install.sh
This will literally install faster than reading the rest of this

14
doc/release_new_version
View file

@ -192,34 +192,32 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20200922 ('Ginsburg') released <<[stable]>>
Subject: GNU Parallel 20201022 ('CRISPR/Memphis-mumier/TrumpVirus/Belarus/Assange') released <<[stable]>>
GNU Parallel 20200922 ('Ginsburg') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20201022 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
Quote of the month:
Great tool, gets jobs done fast.
Great tool, gets jobs done fast.
Great tool, gets jobs done fast.
-- Paul F. De La Cruz @pdelacruzcc@twitter
New in this release:
* New CPU detection for GNU/Linux.
* Bug fixes and man page updates.
News about GNU Parallel:
* Aug 21, 2020 - More Unix tools https://floki.blog/2020/08/more_unix_tools
https://cloud.tencent.com/developer/article/1705499
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
GNU Parallel - For people who live life in the parallel lane.
If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include command that uses GNU Parallel if you feel like it.
= About GNU Parallel =

2
src/niceload
View file

@ -23,7 +23,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20200922;
$Global::version = 20200923;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($opt::version) {

16
src/parallel
View file

@ -881,16 +881,17 @@ sub group_by_loop($$) {
# Numbered 0..n-1 due to being used by $F[n]
if($group_by::col) { $group_by::col--; }
my $loop = ::spacefree(0,'{
my $loop = ::spacefree(0,q{
BEGIN{ $last = "RECSEP"; }
{
local $_=COLVALUE;
PERLEXPR;
if(! defined $last) { $last = $_ }
if(($last) ne $_) {
print "RECSEP";
$last = $_;
}
}');
}
});
if(defined $group_by::col) {
$loop =~ s/COLVALUE/\$F[$group_by::col]/g;
} else {
@ -2160,7 +2161,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20200922;
$Global::version = 20200923;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -4554,6 +4555,7 @@ sub onall($@) {
((defined $opt::retries) ? "--retries ".$opt::retries : ""),
((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""),
((defined $opt::ungroup) ? "-u" : ""),
((defined $opt::ssh) ? "--ssh '".$opt::ssh."'" : ""),
((defined $opt::tee) ? "--tee" : ""),
((defined $opt::workdir) ? "--wd ".Q($opt::workdir) : ""),
(@Global::transfer_files ? map { "--tf ".Q($_) }
@ -4913,7 +4915,7 @@ sub usage() {
"--recend str Record end separator for --pipe.",
"--recstart str Record start separator for --pipe.",
"",
"See 'man $Global::progname' for details",
"GNU Parallel can do much more. See 'man $Global::progname' for details",
"",
"Academic tradition requires you to cite works you base your article on.",
"If you use programs that use GNU Parallel to process data for an article in a",
@ -10330,7 +10332,7 @@ sub print_joblog($) {
$cmd = $self->replaced();
} else {
# Verbose level > 1: Print the rsync and stuff
$cmd = join " ", @{$self->{'commandline'}};
$cmd = $self->wrapped();
}
# Newlines make it hard to parse the joblog
$cmd =~ s/\n/\0/g;

8
src/parallel.pod
View file

@ -4056,6 +4056,14 @@ output.
If you have a lot of hosts use '-j0' to access more hosts in parallel.
=head1 EXAMPLE: Running 'sudo' on remote computers
Put the password into passwordfile then run:
parallel --ssh 'cat passwordfile | ssh' --nonall \
-S user@server1,user@server2 sudo -S ls -l /root
=head1 EXAMPLE: Using remote computers behind NAT wall
If the workers are behind a NAT wall, you need some trickery to get to

2
src/sql
View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20200922;
$Global::version = 20200923;
$Global::progname = 'sql';
# This must be done first as this may exec myself

2
testsuite/tests-to-run/niceload01.sh
View file

@ -44,7 +44,7 @@ EOF
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --joblog /tmp/jl-`basename $0` -L1
echo '### -f and --factor'
niceload -H --factor 10 -l6 echo factor 10 finish last
niceload -H --factor 30 -l6 echo factor 30 finish last
niceload -H -f 0.01 -l6 echo f 0.1 finish first
EOF

1
testsuite/tests-to-run/parallel-local-10s.sh
View file

@ -546,6 +546,7 @@ par_retries_all_fail() {
par_sockets_cores_threads() {
echo '### Test --number-of-sockets/cores/threads'
unset PARALLEL_CPUINFO
unset PARALLEL_LSCPU
parallel --number-of-sockets
parallel --number-of-cores
parallel --number-of-threads

10
testsuite/tests-to-run/parallel-virtualbox1.sh
View file

@ -1,5 +1,10 @@
#!/bin/bash
(
cd vagrant/tange/centos3/
vagrant up
)
par_warning_on_centos3() {
echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error"
testone() {
@ -20,3 +25,8 @@ export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
perl -pe 's:/usr/bin:/bin:g;'
(
cd vagrant/tange/centos3/
vagrant suspend
)

2
testsuite/wanted-results/parallel-local-10s
View file

@ -251,7 +251,7 @@ par_jobs_file --pipe Split stdin (standard input) to multiple jobs.
par_jobs_file --recend str Record end separator for --pipe.
par_jobs_file --recstart str Record start separator for --pipe.
par_jobs_file
par_jobs_file See 'man parallel' for details
par_jobs_file GNU Parallel can do much more. See 'man parallel' for details
par_jobs_file
par_jobs_file Academic tradition requires you to cite works you base your article on.
par_jobs_file If you use programs that use GNU Parallel to process data for an article in a

4
testsuite/wanted-results/parallel-local-ssh7
View file

@ -439,7 +439,7 @@ par_tcsh_man --pipe Split stdin (standard input) to multiple jobs. par_
par_tcsh_man --recend str Record end separator for --pipe. par_tcsh_man
par_tcsh_man --recstart str Record start separator for --pipe. par_tcsh_man
par_tcsh_man par_tcsh_man
par_tcsh_man See 'man parallel' for details par_tcsh_man
par_tcsh_man GNU Parallel can do much more. See 'man parallel' for details par_tcsh_man
par_tcsh_man par_tcsh_man
par_tcsh_man Academic tradition requires you to cite works you base your article on. par_tcsh_man
par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a par_tcsh_man
@ -1856,9 +1856,9 @@ par_csh_man -X Multiple arguments with context replace
par_csh_man -j n Run n jobs in parallel
par_csh_man -k Keep same order
par_csh_man Academic tradition requires you to cite works you base your article on.
par_csh_man GNU Parallel can do much more. See 'man parallel' for details
par_csh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
par_csh_man If you use programs that use GNU Parallel to process data for an article in a
par_csh_man See 'man parallel' for details
par_csh_man This helps funding further development; AND IT WON'T COST YOU A CENT.
par_csh_man Unknown option: no-such-option
par_csh_man Usage:

1
testsuite/wanted-results/parallel-polarhome
View file

@ -166,6 +166,7 @@ bin/env_parallel.dash macosx copy macosx bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash redhat copy redhat bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash netbsd copy netbsd bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash openbsd copy openbsd bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash debian copy debian bin/env_parallel.dash env_parallel.dash
bin/env_parallel.dash minix copy minix bin/env_parallel.dash env_parallel.dash
bin/env_parallel.fish qnx copy qnx bin/env_parallel.fish env_parallel.fish
bin/env_parallel.fish pidora copy pidora bin/env_parallel.fish env_parallel.fish