parallel: Faster --group-by. -vv --joblog / --onall --ssh bugfix.

10seconds_install

@@ -1,26 +1,37 @@
 #!/bin/bash
 
-# Copyright (C) 2013-2019 Ole Tange and Free Software Foundation, Inc.
+true <<'EOF'
-#
+<p>
-# This program is free software; you can redistribute it and/or modify
+     Copyright (C) 2013-2020 Ole Tange and Free Software Foundation, Inc.
-# it under the terms of the GNU General Public License as published by
+<p>
-# the Free Software Foundation; either version 3 of the License, or
+     This program is free software; you can redistribute it and/or
-# (at your option) any later version.
+     modify it under the terms of the GNU General Public License as
-#
+     published by the Free Software Foundation; either version 3 of
-# This script downloads the latest version of GNU Parallel, checks
+     the License, or (at your option) any later version.
-# the signature and installs it.
+<p>
-#
+     This script downloads the latest version of GNU Parallel, checks
-# It first tries to install it globally.
+     the signature and installs it.
-# If that fails, it does a personal installation.
+
-# If that fails, it copies to $HOME/bin
+<p>  It first tries to install it globally.
-#
+<br> If that fails, it does a personal installation.
-# You can download and run the script directly by:
+<br> If that fails, it copies to $HOME/bin
-#    $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
+
-#       fetch -o - http://pi.dk/3) > install.sh
+<p>  You can download and run the script directly by:
-#    $ sha1sum install.sh
+
-#    $ md5sum install.sh
+<p>  $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 ||
-#    $ sha512sum install.sh
+        curl pi.dk/3/ || fetch -o - http://pi.dk/3) > install.sh
-#    $ bash install.sh
+<br> $ sha1sum install.sh | grep 12345678
+<br> $ md5sum install.sh
+<br> $ sha512sum install.sh
+
+<p>  Check the sums from https://gnu.org/s/parallel/checksums
+<br> Then run:
+
+<p>  $ bash install.sh
+
+
+<br><br><br><br>
+EOF
 
 run() {
     # FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'

README

@@ -40,13 +40,13 @@ installation.
 
     $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
        fetch -o - http://pi.dk/3 ) > install.sh
-    $ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9
+    $ sha1sum install.sh | grep 67bd7bc7dc20aff99eb8f1266574dadb
-    12345678 3374ec53 bacb199b 245af2dd a86df6c9
+    12345678 67bd7bc7 dc20aff9 9eb8f126 6574dadb
-    $ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca
+    $ md5sum install.sh | grep b7a15cdbb07fb6e11b0338577bc1780f
-    029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
+    b7a15cdb b07fb6e1 1b033857 7bc1780f
-    $ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b
+    $ sha512sum install.sh | grep 186000b62b66969d7506ca4f885e0c80e02a22444
-    40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
+    6f25960b d4b90cf6 ba5b76de c1acdf39 f3d24249 72930394 a4164351 93a7668d
-    60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
+    21ff9839 6f920be5 186000b6 2b66969d 7506ca4f 885e0c80 e02a2244 40e8a43f
     $ bash install.sh
 
 This will literally install faster than reading the rest of this

doc/release_new_version

@@ -192,34 +192,32 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20200922 ('Ginsburg') released <<[stable]>>
+Subject: GNU Parallel 20201022 ('CRISPR/Memphis-mumier/TrumpVirus/Belarus/Assange') released <<[stable]>>
 
-GNU Parallel 20200922 ('Ginsburg') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20201022 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
 Quote of the month:
 
-  Great tool, gets jobs done fast.
-  Great tool, gets jobs done fast.
-  Great tool, gets jobs done fast.
-    -- Paul F. De La Cruz @pdelacruzcc@twitter
   
 New in this release:
 
-* New CPU detection for GNU/Linux.
+
 
 * Bug fixes and man page updates.
 
 News about GNU Parallel:
 
-* Aug 21, 2020 - More Unix tools https://floki.blog/2020/08/more_unix_tools
+https://cloud.tencent.com/developer/article/1705499
 
 
 Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
 
 GNU Parallel - For people who live life in the parallel lane.
 
+If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include command that uses GNU Parallel if you feel like it.
+
 
 = About GNU Parallel =
 

src/niceload

@@ -23,7 +23,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20200922;
+$Global::version = 20200923;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -881,16 +881,17 @@ sub group_by_loop($$) {
     # Numbered 0..n-1 due to being used by $F[n]
     if($group_by::col) { $group_by::col--; }
 
-    my $loop = ::spacefree(0,'{
+    my $loop = ::spacefree(0,q{
+	BEGIN{ $last = "RECSEP"; }
+        {
 	    local $_=COLVALUE;
 	    PERLEXPR;
-	if(! defined $last) { $last = $_ }
 	    if(($last) ne $_) {
 		print "RECSEP";
 		$last = $_;
 	    }
-    }');
+	}
-
+			   });
     if(defined $group_by::col) {
 	$loop =~ s/COLVALUE/\$F[$group_by::col]/g;
     } else {
@@ -2160,7 +2161,7 @@ sub check_invalid_option_combinations() {
 
 sub init_globals() {
     # Defaults:
-    $Global::version = 20200922;
+    $Global::version = 20200923;
     $Global::progname = 'parallel';
     $::name = "GNU Parallel";
     $Global::infinity = 2**31;
@@ -4554,6 +4555,7 @@ sub onall($@) {
 	     ((defined $opt::retries) ? "--retries ".$opt::retries : ""),
 	     ((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""),
 	     ((defined $opt::ungroup) ? "-u" : ""),
+	     ((defined $opt::ssh) ? "--ssh '".$opt::ssh."'" : ""),
 	     ((defined $opt::tee) ? "--tee" : ""),
 	     ((defined $opt::workdir) ? "--wd ".Q($opt::workdir) : ""),
 	     (@Global::transfer_files ? map { "--tf ".Q($_) }
@@ -4913,7 +4915,7 @@ sub usage() {
 	 "--recend str    Record end separator for --pipe.",
 	 "--recstart str  Record start separator for --pipe.",
 	 "",
-	 "See 'man $Global::progname' for details",
+	 "GNU Parallel can do much more. See 'man $Global::progname' for details",
 	 "",
 	 "Academic tradition requires you to cite works you base your article on.",
 	 "If you use programs that use GNU Parallel to process data for an article in a",
@@ -10330,7 +10332,7 @@ sub print_joblog($) {
 	$cmd = $self->replaced();
     } else {
 	# Verbose level > 1: Print the rsync and stuff
-	$cmd = join " ", @{$self->{'commandline'}};
+	$cmd = $self->wrapped();
     }
     # Newlines make it hard to parse the joblog
     $cmd =~ s/\n/\0/g;

src/parallel.pod

@@ -4056,6 +4056,14 @@ output.
 If you have a lot of hosts use '-j0' to access more hosts in parallel.
 
 
+=head1 EXAMPLE: Running 'sudo' on remote computers
+
+Put the password into passwordfile then run:
+
+  parallel --ssh 'cat passwordfile | ssh' --nonall \
+    -S user@server1,user@server2 sudo -S ls -l /root
+
+
 =head1 EXAMPLE: Using remote computers behind NAT wall
 
 If the workers are behind a NAT wall, you need some trickery to get to

src/sql

@@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20200922;
+    $Global::version = 20200923;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/niceload01.sh

@@ -44,7 +44,7 @@ EOF
 
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --joblog /tmp/jl-`basename $0` -L1
 echo '### -f and --factor'
-  niceload -H --factor 10 -l6 echo factor 10 finish last
+  niceload -H --factor 30 -l6 echo factor 30 finish last
   niceload -H -f 0.01 -l6 echo f 0.1 finish first
 EOF
 

testsuite/tests-to-run/parallel-local-10s.sh

@@ -546,6 +546,7 @@ par_retries_all_fail() {
 par_sockets_cores_threads() {
     echo '### Test --number-of-sockets/cores/threads'
     unset PARALLEL_CPUINFO
+    unset PARALLEL_LSCPU
     parallel --number-of-sockets
     parallel --number-of-cores
     parallel --number-of-threads

testsuite/tests-to-run/parallel-virtualbox1.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+(
+    cd vagrant/tange/centos3/
+    vagrant up
+)
+
 par_warning_on_centos3() {
     echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error"
     testone() {
@@ -20,3 +25,8 @@ export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | LC_ALL=C sort |
     parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
     perl -pe 's:/usr/bin:/bin:g;'
+
+(
+    cd vagrant/tange/centos3/
+    vagrant suspend
+)

testsuite/wanted-results/parallel-local-10s

@@ -251,7 +251,7 @@ par_jobs_file	--pipe          Split stdin (standard input) to multiple jobs.
 par_jobs_file	--recend str    Record end separator for --pipe.
 par_jobs_file	--recstart str  Record start separator for --pipe.
 par_jobs_file	
-par_jobs_file	See 'man parallel' for details
+par_jobs_file	GNU Parallel can do much more. See 'man parallel' for details
 par_jobs_file	
 par_jobs_file	Academic tradition requires you to cite works you base your article on.
 par_jobs_file	If you use programs that use GNU Parallel to process data for an article in a

testsuite/wanted-results/parallel-local-ssh7

@@ -439,7 +439,7 @@ par_tcsh_man	--pipe          Split stdin (standard input) to multiple jobs.
par_
 par_tcsh_man	--recend str    Record end separator for --pipe.
par_tcsh_man	
 par_tcsh_man	--recstart str  Record start separator for --pipe.
par_tcsh_man	
 par_tcsh_man	
par_tcsh_man	
-par_tcsh_man	See 'man parallel' for details
par_tcsh_man	
+par_tcsh_man	GNU Parallel can do much more. See 'man parallel' for details
par_tcsh_man	
 par_tcsh_man	
par_tcsh_man	
 par_tcsh_man	Academic tradition requires you to cite works you base your article on.
par_tcsh_man	
 par_tcsh_man	If you use programs that use GNU Parallel to process data for an article in a
par_tcsh_man	
@@ -1856,9 +1856,9 @@ par_csh_man	-X              Multiple arguments with context replace
 par_csh_man	-j n            Run n jobs in parallel
 par_csh_man	-k              Keep same order
 par_csh_man	Academic tradition requires you to cite works you base your article on.
+par_csh_man	GNU Parallel can do much more. See 'man parallel' for details
 par_csh_man	If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 par_csh_man	If you use programs that use GNU Parallel to process data for an article in a
-par_csh_man	See 'man parallel' for details
 par_csh_man	This helps funding further development; AND IT WON'T COST YOU A CENT.
 par_csh_man	Unknown option: no-such-option
 par_csh_man	Usage:

testsuite/wanted-results/parallel-polarhome

@@ -166,6 +166,7 @@ bin/env_parallel.dash macosx	copy macosx bin/env_parallel.dash env_parallel.dash
 bin/env_parallel.dash redhat	copy redhat bin/env_parallel.dash env_parallel.dash
 bin/env_parallel.dash netbsd	copy netbsd bin/env_parallel.dash env_parallel.dash
 bin/env_parallel.dash openbsd	copy openbsd bin/env_parallel.dash env_parallel.dash
+bin/env_parallel.dash debian	copy debian bin/env_parallel.dash env_parallel.dash
 bin/env_parallel.dash minix	copy minix bin/env_parallel.dash env_parallel.dash
 bin/env_parallel.fish qnx	copy qnx bin/env_parallel.fish env_parallel.fish
 bin/env_parallel.fish pidora	copy pidora bin/env_parallel.fish env_parallel.fish