parallel: --header : sets named replacement string if input line ends in \r\n'.

NEWS

@@ -7,7 +7,7 @@
 
 * Use -J ./profile to read a profile in current dir.
 
-* Speedup of startup by 40%: Find the parent shell differerently on
+* Speedup of startup by 40%: Find the parent shell differently on
   GNU/Linux, cache information about the CPU and which setpgrp method
   to use to make GNU Parallel start 40% faster.
 

doc/haikus

@@ -1,5 +1,11 @@
 Quote of the month:
 
+With multicore systems everywhere GNU Parallel is a must have tool.
+  -- Neil H. Watson @neil_h_watson
+
+My new favourite command line tool is gnu parallel. After knowing it for a while I just recently came back to it for a refactor of a script and now I've changed like 4 different huge scripts into tiny parallel scripts that do their job literally 3 to 10 times faster and just [mind blown]
+  -- Izzy @angry_izzy
+
 I love so much @GnuParallel to parallelize my SQL requests on @PostgreSQL
   -- @rmaziere_85 Romain
 

doc/release_new_version

@@ -206,9 +206,9 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20190722 ('Ryugu') released <<[stable]>>
+Subject: GNU Parallel 20190822 ('Jesper Svarre') released <<[stable]>>
 
-GNU Parallel 20190722 ('Ryugu') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20190822 ('Jesper Svarre') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -218,28 +218,13 @@ See https://www.gnu.org/software/parallel/10-years-anniversary.html
 
 Quote of the month:
 
-  It is SUPER easy to speed up jobs from the command line w/ GNU parallel.
+  <<>>
-    -- B3n @B3njaminHimes@twitter
 
 New in this release:
 
-* {= uq; =} causes the replacement string to be unquoted. Example: parallel echo '{=uq;=}.jpg' ::: '*'
+https://captmeelo.com/pentest/2019/07/29/port-scanning.html
-
+https://linuxreviews.org/GNU_Parallel_Ryugu_released
-* --tagstring {=...=} is now evaluated for each line with --linebuffer.
+http://www.7daixie.com/2019071646302396.html
-
-* Use -J ./profile to read a profile in current dir.
-
-* Speedup of startup by 40%: Find the parent shell differerently on GNU/Linux, cache information about the CPU and which setpgrp method to use to make GNU Parallel start 40% faster.
-
-* $PARALLEL_SSHLOGIN can be used in the command line.
-
-* Occasionally sane: Simple usage of GNU parallel https://vfoley.xyz/parallel/
-
-* How to run multiple bash scripts in parallel https://techieroop.com/how-to-run-multiple-bash-scripts-in-parallel/
-
-* Announcing Oracle Solaris 11.4 SRU10 https://blogs.oracle.com/solaris/announcing-oracle-solaris-114-sru10
-
-* Руководство по использованию GNU Parallel https://zalinux.ru/?p=2623
 
 * Bug fixes and man page updates.
 
@@ -256,10 +241,30 @@ If you use xargs and tee today you will find GNU Parallel very easy to use as GN
 
 GNU Parallel makes sure output from the commands is the same output as you would get had you run the commands sequentially. This makes it possible to use output from GNU Parallel as input for other programs.
 
+For example you can run this to convert all jpeg files into png and gif files and have a progress bar:
+
+  parallel --bar convert {1} {1.}.{2} ::: *.jpg ::: png gif
+
+Or you can generate big, medium, and small thumbnails of all jpeg files in sub dirs:
+
+  find . -name '*.jpg' |
+    parallel convert -geometry {2} {1} {1//}/thumb{2}_{1/} :::: - ::: 50 100 200
+
+
 You can find more about GNU Parallel at: http://www.gnu.org/s/parallel/
 
 You can install GNU Parallel in just 10 seconds with:
-(wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
+
+    $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
+       fetch -o - http://pi.dk/3 ) > install.sh
+    $ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9
+    12345678 3374ec53 bacb199b 245af2dd a86df6c9
+    $ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca
+    029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
+    $ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b
+    40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
+    60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
+    $ bash install.sh
 
 Watch the intro video on http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1
 

src/niceload

@@ -23,7 +23,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20190722;
+$Global::version = 20190724;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -48,6 +48,7 @@ sub set_input_source_header($$) {
 	for my $fh (@$input_source_fh_ref) {
 	    my $line = <$fh>;
 	    chomp($line);
+	    $line =~ s/\r$//;
 	    ::debug("init", "Delimiter: '$delimiter'");
 	    for my $s (split /$delimiter/o, $line) {
 		::debug("init", "Colname: '$s'");
@@ -1519,7 +1520,7 @@ sub options_hash() {
 	 "link|xapply" => \$opt::link,
 	 "linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource,
 	 # Before changing this line, please read
-         # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
+	 # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
 	 "bibtex|citation" => \$opt::citation,
 	 "wc|willcite|will-cite|nn|nonotice|no-notice" => \$opt::willcite,
 	 # Termination and retries
@@ -2047,7 +2048,7 @@ sub check_invalid_option_combinations() {
 
 sub init_globals() {
     # Defaults:
-    $Global::version = 20190722;
+    $Global::version = 20190724;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -2838,17 +2839,18 @@ sub shell_quote_scalar_default($) {
     #   $string = string to be quoted
     # Returns:
     #   $shell_quoted = string quoted as needed by the shell
-    my $par = $_[0];
+    my $s = $_[0];
-    if($par =~ /[^-_.+a-z0-9\/]/i) {
+    if($s =~ /[^-_.+a-z0-9\/]/i) {
-	$par =~ s/'/'"'"'/g;  # "-quote single quotes
+	$s =~ s/'/'"'"'/g; # "-quote single quotes
-	$par = "'$par'";      # '-quote entire string
+	$s = "'$s'";       # '-quote entire string
-	$par =~ s/^''|''$//g; # Remove unneeded '' at ends
+	$s =~ s/^''//;     # Remove unneeded '' at ends
-	return $par;
+	$s =~ s/''$//;     # (faster than s/^''|''$//g)
-    } elsif ($par eq "") {
+	return $s;
+    } elsif ($s eq "") {
 	return "''";
     } else {
 	# No quoting needed
-	return $par;
+	return $s;
     }
 }
 

src/sql

@@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20190722;
+    $Global::version = 20190724;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -780,6 +780,12 @@ par_bin() {
     # paste <(seq 10000000) <(seq 10000000 -1 1) | parallel --pipe --colsep '\t' --bin 2 wc
 }
 
+par_cr_newline_header() {
+    echo '### --header : should set named replacement string if input line ends in \r\n'
+    printf "foo\r\nbar\r\n" |
+	parallel --colsep , --header : echo {foo}
+}
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | LC_ALL=C sort |
     parallel --timeout 30 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1'

testsuite/tests-to-run/parallel-local-ssh9.sh

@@ -166,14 +166,13 @@ par_no_route_to_host() {
     }
     export -f raw
 
-    # Random hosts that there is no route to
+    # Random hosts
     findhosts() {
 	ip='$(($RANDOM%256)).$(($RANDOM%256)).$(($RANDOM%256)).$(($RANDOM%256))'
-	stdout parallel --timeout 2 -j0 ssh -o PasswordAuthentication=no $ip echo ::: {1..10000} |
+	seq 10000 | parallel -N0 echo $ip 
-	    perl -ne 's/ssh:.* host (\d+\.\d+\.\d+\.\d+) .* No route .*/$1/ and print; $|=1'
     }
 
-    # Retry if the hosts really fails this fast
+    # See if the hosts fail fast
     filterhosts() {
 	stdout parallel --timeout 2 -j5 ssh -o PasswordAuthentication=no {} echo |
 	    perl -ne 's/ssh:.* host (\d+\.\d+\.\d+\.\d+) .* No route .*/$1/ and print; $|=1'
@@ -183,12 +182,12 @@ par_no_route_to_host() {
 	# Cache a list of hosts that fail fast with 'No route'
 	# Filter the list 4 times to make sure to get good hosts
 	renice 10 -p $$ >/dev/null
-	findhosts | filterhosts | filterhosts |
+	findhosts | filterhosts | filterhosts | filterhosts |
 	    filterhosts | filterhosts | head > /tmp/filtered.$$
 	mv /tmp/filtered.$$ /tmp/filtered.hosts
     ) &
     (
-	# We just need one to complete
+	# We just need one of each to complete
 	stdout parallel --halt now,done=1 -j0 raw :::: /tmp/filtered.hosts
 	stdout parallel --halt now,done=1 -j0 via_parallel :::: /tmp/filtered.hosts
     ) | perl -pe 's/(\d+\.\d+\.\d+\.\d+)/i.p.n.r/' | puniq

testsuite/wanted-results/parallel-local-0.3s

@@ -77,6 +77,8 @@ par_blocking_redir	stdout
 par_colsep_0	bug --colsep 0
 par_colsep_0	OK
 par_colsep_0	OK
+par_cr_newline_header	### --header : should set named replacement string if input line ends in \r\n
+par_cr_newline_header	bar
 par_csv	col1"x3"-new
 par_csv	line col2-new2
 par_csv	line col3-col 4