diff --git a/NEWS b/NEWS index 7c1bb7ae..96aec213 100644 --- a/NEWS +++ b/NEWS @@ -7,7 +7,7 @@ * Use -J ./profile to read a profile in current dir. -* Speedup of startup by 40%: Find the parent shell differerently on +* Speedup of startup by 40%: Find the parent shell differently on GNU/Linux, cache information about the CPU and which setpgrp method to use to make GNU Parallel start 40% faster. diff --git a/doc/haikus b/doc/haikus index 627f5db6..3dacecff 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,5 +1,11 @@ Quote of the month: +With multicore systems everywhere GNU Parallel is a must have tool. + -- Neil H. Watson @neil_h_watson + +My new favourite command line tool is gnu parallel. After knowing it for a while I just recently came back to it for a refactor of a script and now I've changed like 4 different huge scripts into tiny parallel scripts that do their job literally 3 to 10 times faster and just [mind blown] + -- Izzy @angry_izzy + I love so much @GnuParallel to parallelize my SQL requests on @PostgreSQL -- @rmaziere_85 Romain diff --git a/doc/release_new_version b/doc/release_new_version index e9c37130..0f412834 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -206,9 +206,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20190722 ('Ryugu') released <<[stable]>> +Subject: GNU Parallel 20190822 ('Jesper Svarre') released <<[stable]>> -GNU Parallel 20190722 ('Ryugu') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20190822 ('Jesper Svarre') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -218,28 +218,13 @@ See https://www.gnu.org/software/parallel/10-years-anniversary.html Quote of the month: - It is SUPER easy to speed up jobs from the command line w/ GNU parallel. - -- B3n @B3njaminHimes@twitter + <<>> New in this release: -* {= uq; =} causes the replacement string to be unquoted. Example: parallel echo '{=uq;=}.jpg' ::: '*' - -* --tagstring {=...=} is now evaluated for each line with --linebuffer. - -* Use -J ./profile to read a profile in current dir. - -* Speedup of startup by 40%: Find the parent shell differerently on GNU/Linux, cache information about the CPU and which setpgrp method to use to make GNU Parallel start 40% faster. - -* $PARALLEL_SSHLOGIN can be used in the command line. - -* Occasionally sane: Simple usage of GNU parallel https://vfoley.xyz/parallel/ - -* How to run multiple bash scripts in parallel https://techieroop.com/how-to-run-multiple-bash-scripts-in-parallel/ - -* Announcing Oracle Solaris 11.4 SRU10 https://blogs.oracle.com/solaris/announcing-oracle-solaris-114-sru10 - -* Руководство по использованию GNU Parallel https://zalinux.ru/?p=2623 +https://captmeelo.com/pentest/2019/07/29/port-scanning.html +https://linuxreviews.org/GNU_Parallel_Ryugu_released +http://www.7daixie.com/2019071646302396.html * Bug fixes and man page updates. @@ -256,10 +241,30 @@ If you use xargs and tee today you will find GNU Parallel very easy to use as GN GNU Parallel makes sure output from the commands is the same output as you would get had you run the commands sequentially. This makes it possible to use output from GNU Parallel as input for other programs. +For example you can run this to convert all jpeg files into png and gif files and have a progress bar: + + parallel --bar convert {1} {1.}.{2} ::: *.jpg ::: png gif + +Or you can generate big, medium, and small thumbnails of all jpeg files in sub dirs: + + find . -name '*.jpg' | + parallel convert -geometry {2} {1} {1//}/thumb{2}_{1/} :::: - ::: 50 100 200 + + You can find more about GNU Parallel at: http://www.gnu.org/s/parallel/ You can install GNU Parallel in just 10 seconds with: -(wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash + + $ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \ + fetch -o - http://pi.dk/3 ) > install.sh + $ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9 + 12345678 3374ec53 bacb199b 245af2dd a86df6c9 + $ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca + 029a9ac0 6e8b5bc6 052eac57 b2c3c9ca + $ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b + 40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4 + 60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb + $ bash install.sh Watch the intro video on http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 diff --git a/src/niceload b/src/niceload index a818feed..be7f65c7 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20190722; +$Global::version = 20190724; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 2d2d837d..d46384bc 100755 --- a/src/parallel +++ b/src/parallel @@ -48,6 +48,7 @@ sub set_input_source_header($$) { for my $fh (@$input_source_fh_ref) { my $line = <$fh>; chomp($line); + $line =~ s/\r$//; ::debug("init", "Delimiter: '$delimiter'"); for my $s (split /$delimiter/o, $line) { ::debug("init", "Colname: '$s'"); @@ -1519,7 +1520,7 @@ sub options_hash() { "link|xapply" => \$opt::link, "linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource, # Before changing this line, please read - # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice "bibtex|citation" => \$opt::citation, "wc|willcite|will-cite|nn|nonotice|no-notice" => \$opt::willcite, # Termination and retries @@ -2047,7 +2048,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20190722; + $Global::version = 20190724; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -2838,17 +2839,18 @@ sub shell_quote_scalar_default($) { # $string = string to be quoted # Returns: # $shell_quoted = string quoted as needed by the shell - my $par = $_[0]; - if($par =~ /[^-_.+a-z0-9\/]/i) { - $par =~ s/'/'"'"'/g; # "-quote single quotes - $par = "'$par'"; # '-quote entire string - $par =~ s/^''|''$//g; # Remove unneeded '' at ends - return $par; - } elsif ($par eq "") { + my $s = $_[0]; + if($s =~ /[^-_.+a-z0-9\/]/i) { + $s =~ s/'/'"'"'/g; # "-quote single quotes + $s = "'$s'"; # '-quote entire string + $s =~ s/^''//; # Remove unneeded '' at ends + $s =~ s/''$//; # (faster than s/^''|''$//g) + return $s; + } elsif ($s eq "") { return "''"; } else { # No quoting needed - return $par; + return $s; } } diff --git a/src/sql b/src/sql index cda4198f..927b7ee5 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20190722; + $Global::version = 20190724; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 3fd5c280..b98da6df 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -780,6 +780,12 @@ par_bin() { # paste <(seq 10000000) <(seq 10000000 -1 1) | parallel --pipe --colsep '\t' --bin 2 wc } +par_cr_newline_header() { + echo '### --header : should set named replacement string if input line ends in \r\n' + printf "foo\r\nbar\r\n" | + parallel --colsep , --header : echo {foo} +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | parallel --timeout 30 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-ssh9.sh b/testsuite/tests-to-run/parallel-local-ssh9.sh index e21401a5..e424ce62 100644 --- a/testsuite/tests-to-run/parallel-local-ssh9.sh +++ b/testsuite/tests-to-run/parallel-local-ssh9.sh @@ -166,14 +166,13 @@ par_no_route_to_host() { } export -f raw - # Random hosts that there is no route to + # Random hosts findhosts() { ip='$(($RANDOM%256)).$(($RANDOM%256)).$(($RANDOM%256)).$(($RANDOM%256))' - stdout parallel --timeout 2 -j0 ssh -o PasswordAuthentication=no $ip echo ::: {1..10000} | - perl -ne 's/ssh:.* host (\d+\.\d+\.\d+\.\d+) .* No route .*/$1/ and print; $|=1' + seq 10000 | parallel -N0 echo $ip } - # Retry if the hosts really fails this fast + # See if the hosts fail fast filterhosts() { stdout parallel --timeout 2 -j5 ssh -o PasswordAuthentication=no {} echo | perl -ne 's/ssh:.* host (\d+\.\d+\.\d+\.\d+) .* No route .*/$1/ and print; $|=1' @@ -183,12 +182,12 @@ par_no_route_to_host() { # Cache a list of hosts that fail fast with 'No route' # Filter the list 4 times to make sure to get good hosts renice 10 -p $$ >/dev/null - findhosts | filterhosts | filterhosts | + findhosts | filterhosts | filterhosts | filterhosts | filterhosts | filterhosts | head > /tmp/filtered.$$ mv /tmp/filtered.$$ /tmp/filtered.hosts ) & ( - # We just need one to complete + # We just need one of each to complete stdout parallel --halt now,done=1 -j0 raw :::: /tmp/filtered.hosts stdout parallel --halt now,done=1 -j0 via_parallel :::: /tmp/filtered.hosts ) | perl -pe 's/(\d+\.\d+\.\d+\.\d+)/i.p.n.r/' | puniq diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 8f3c86bd..fb83232a 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -77,6 +77,8 @@ par_blocking_redir stdout par_colsep_0 bug --colsep 0 par_colsep_0 OK par_colsep_0 OK +par_cr_newline_header ### --header : should set named replacement string if input line ends in \r\n +par_cr_newline_header bar par_csv col1"x3"-new par_csv line col2-new2 par_csv line col3-col 4