mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
parallel: --header : sets named replacement string if input line ends in \r\n'.
This commit is contained in:
Ole Tange
parent
3b7b988c16
commit
2aabbf5e76
2
NEWS
2
NEWS
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
* Use -J ./profile to read a profile in current dir.
|
||||
|
||||
* Speedup of startup by 40%: Find the parent shell differerently on
|
||||
* Speedup of startup by 40%: Find the parent shell differently on
|
||||
GNU/Linux, cache information about the CPU and which setpgrp method
|
||||
to use to make GNU Parallel start 40% faster.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
Quote of the month:
|
||||
|
||||
With multicore systems everywhere GNU Parallel is a must have tool.
|
||||
-- Neil H. Watson @neil_h_watson
|
||||
|
||||
My new favourite command line tool is gnu parallel. After knowing it for a while I just recently came back to it for a refactor of a script and now I've changed like 4 different huge scripts into tiny parallel scripts that do their job literally 3 to 10 times faster and just [mind blown]
|
||||
-- Izzy @angry_izzy
|
||||
|
||||
I love so much @GnuParallel to parallelize my SQL requests on @PostgreSQL
|
||||
-- @rmaziere_85 Romain
|
||||
|
||||
|
|
|
|||
|
|
@ -206,9 +206,9 @@ from:tange@gnu.org
|
|||
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
Subject: GNU Parallel 20190722 ('Ryugu') released <<[stable]>>
|
||||
Subject: GNU Parallel 20190822 ('Jesper Svarre') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20190722 ('Ryugu') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
GNU Parallel 20190822 ('Jesper Svarre') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
|
||||
<<No new functionality was introduced so this is a good candidate for a stable release.>>
|
||||
|
||||
|
|
@ -218,28 +218,13 @@ See https://www.gnu.org/software/parallel/10-years-anniversary.html
|
|||
|
||||
Quote of the month:
|
||||
|
||||
It is SUPER easy to speed up jobs from the command line w/ GNU parallel.
|
||||
-- B3n @B3njaminHimes@twitter
|
||||
<<>>
|
||||
|
||||
New in this release:
|
||||
|
||||
* {= uq; =} causes the replacement string to be unquoted. Example: parallel echo '{=uq;=}.jpg' ::: '*'
|
||||
|
||||
* --tagstring {=...=} is now evaluated for each line with --linebuffer.
|
||||
|
||||
* Use -J ./profile to read a profile in current dir.
|
||||
|
||||
* Speedup of startup by 40%: Find the parent shell differerently on GNU/Linux, cache information about the CPU and which setpgrp method to use to make GNU Parallel start 40% faster.
|
||||
|
||||
* $PARALLEL_SSHLOGIN can be used in the command line.
|
||||
|
||||
* Occasionally sane: Simple usage of GNU parallel https://vfoley.xyz/parallel/
|
||||
|
||||
* How to run multiple bash scripts in parallel https://techieroop.com/how-to-run-multiple-bash-scripts-in-parallel/
|
||||
|
||||
* Announcing Oracle Solaris 11.4 SRU10 https://blogs.oracle.com/solaris/announcing-oracle-solaris-114-sru10
|
||||
|
||||
* Руководство по использованию GNU Parallel https://zalinux.ru/?p=2623
|
||||
https://captmeelo.com/pentest/2019/07/29/port-scanning.html
|
||||
https://linuxreviews.org/GNU_Parallel_Ryugu_released
|
||||
http://www.7daixie.com/2019071646302396.html
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
|
|
@ -256,10 +241,30 @@ If you use xargs and tee today you will find GNU Parallel very easy to use as GN
|
|||
|
||||
GNU Parallel makes sure output from the commands is the same output as you would get had you run the commands sequentially. This makes it possible to use output from GNU Parallel as input for other programs.
|
||||
|
||||
For example you can run this to convert all jpeg files into png and gif files and have a progress bar:
|
||||
|
||||
parallel --bar convert {1} {1.}.{2} ::: *.jpg ::: png gif
|
||||
|
||||
Or you can generate big, medium, and small thumbnails of all jpeg files in sub dirs:
|
||||
|
||||
find . -name '*.jpg' |
|
||||
parallel convert -geometry {2} {1} {1//}/thumb{2}_{1/} :::: - ::: 50 100 200
|
||||
|
||||
|
||||
You can find more about GNU Parallel at: http://www.gnu.org/s/parallel/
|
||||
|
||||
You can install GNU Parallel in just 10 seconds with:
|
||||
(wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
|
||||
|
||||
$ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
|
||||
fetch -o - http://pi.dk/3 ) > install.sh
|
||||
$ sha1sum install.sh | grep 3374ec53bacb199b245af2dda86df6c9
|
||||
12345678 3374ec53 bacb199b 245af2dd a86df6c9
|
||||
$ md5sum install.sh | grep 029a9ac06e8b5bc6052eac57b2c3c9ca
|
||||
029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
|
||||
$ sha512sum install.sh | grep f517006d9897747bed8a4694b1acba1b
|
||||
40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
|
||||
60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
|
||||
$ bash install.sh
|
||||
|
||||
Watch the intro video on http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20190722;
|
||||
$Global::version = 20190724;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
22
src/parallel
22
src/parallel
|
|
@ -48,6 +48,7 @@ sub set_input_source_header($$) {
|
|||
for my $fh (@$input_source_fh_ref) {
|
||||
my $line = <$fh>;
|
||||
chomp($line);
|
||||
$line =~ s/\r$//;
|
||||
::debug("init", "Delimiter: '$delimiter'");
|
||||
for my $s (split /$delimiter/o, $line) {
|
||||
::debug("init", "Colname: '$s'");
|
||||
|
|
@ -1519,7 +1520,7 @@ sub options_hash() {
|
|||
"link|xapply" => \$opt::link,
|
||||
"linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource,
|
||||
# Before changing this line, please read
|
||||
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
||||
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
||||
"bibtex|citation" => \$opt::citation,
|
||||
"wc|willcite|will-cite|nn|nonotice|no-notice" => \$opt::willcite,
|
||||
# Termination and retries
|
||||
|
|
@ -2047,7 +2048,7 @@ sub check_invalid_option_combinations() {
|
|||
|
||||
sub init_globals() {
|
||||
# Defaults:
|
||||
$Global::version = 20190722;
|
||||
$Global::version = 20190724;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -2838,17 +2839,18 @@ sub shell_quote_scalar_default($) {
|
|||
# $string = string to be quoted
|
||||
# Returns:
|
||||
# $shell_quoted = string quoted as needed by the shell
|
||||
my $par = $_[0];
|
||||
if($par =~ /[^-_.+a-z0-9\/]/i) {
|
||||
$par =~ s/'/'"'"'/g; # "-quote single quotes
|
||||
$par = "'$par'"; # '-quote entire string
|
||||
$par =~ s/^''|''$//g; # Remove unneeded '' at ends
|
||||
return $par;
|
||||
} elsif ($par eq "") {
|
||||
my $s = $_[0];
|
||||
if($s =~ /[^-_.+a-z0-9\/]/i) {
|
||||
$s =~ s/'/'"'"'/g; # "-quote single quotes
|
||||
$s = "'$s'"; # '-quote entire string
|
||||
$s =~ s/^''//; # Remove unneeded '' at ends
|
||||
$s =~ s/''$//; # (faster than s/^''|''$//g)
|
||||
return $s;
|
||||
} elsif ($s eq "") {
|
||||
return "''";
|
||||
} else {
|
||||
# No quoting needed
|
||||
return $par;
|
||||
return $s;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20190722;
|
||||
$Global::version = 20190724;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -780,6 +780,12 @@ par_bin() {
|
|||
# paste <(seq 10000000) <(seq 10000000 -1 1) | parallel --pipe --colsep '\t' --bin 2 wc
|
||||
}
|
||||
|
||||
par_cr_newline_header() {
|
||||
echo '### --header : should set named replacement string if input line ends in \r\n'
|
||||
printf "foo\r\nbar\r\n" |
|
||||
parallel --colsep , --header : echo {foo}
|
||||
}
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
compgen -A function | grep par_ | LC_ALL=C sort |
|
||||
parallel --timeout 30 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1'
|
||||
|
|
|
|||
|
|
@ -166,14 +166,13 @@ par_no_route_to_host() {
|
|||
}
|
||||
export -f raw
|
||||
|
||||
# Random hosts that there is no route to
|
||||
# Random hosts
|
||||
findhosts() {
|
||||
ip='$(($RANDOM%256)).$(($RANDOM%256)).$(($RANDOM%256)).$(($RANDOM%256))'
|
||||
stdout parallel --timeout 2 -j0 ssh -o PasswordAuthentication=no $ip echo ::: {1..10000} |
|
||||
perl -ne 's/ssh:.* host (\d+\.\d+\.\d+\.\d+) .* No route .*/$1/ and print; $|=1'
|
||||
seq 10000 | parallel -N0 echo $ip
|
||||
}
|
||||
|
||||
# Retry if the hosts really fails this fast
|
||||
# See if the hosts fail fast
|
||||
filterhosts() {
|
||||
stdout parallel --timeout 2 -j5 ssh -o PasswordAuthentication=no {} echo |
|
||||
perl -ne 's/ssh:.* host (\d+\.\d+\.\d+\.\d+) .* No route .*/$1/ and print; $|=1'
|
||||
|
|
@ -183,12 +182,12 @@ par_no_route_to_host() {
|
|||
# Cache a list of hosts that fail fast with 'No route'
|
||||
# Filter the list 4 times to make sure to get good hosts
|
||||
renice 10 -p $$ >/dev/null
|
||||
findhosts | filterhosts | filterhosts |
|
||||
findhosts | filterhosts | filterhosts | filterhosts |
|
||||
filterhosts | filterhosts | head > /tmp/filtered.$$
|
||||
mv /tmp/filtered.$$ /tmp/filtered.hosts
|
||||
) &
|
||||
(
|
||||
# We just need one to complete
|
||||
# We just need one of each to complete
|
||||
stdout parallel --halt now,done=1 -j0 raw :::: /tmp/filtered.hosts
|
||||
stdout parallel --halt now,done=1 -j0 via_parallel :::: /tmp/filtered.hosts
|
||||
) | perl -pe 's/(\d+\.\d+\.\d+\.\d+)/i.p.n.r/' | puniq
|
||||
|
|
|
|||
|
|
@ -77,6 +77,8 @@ par_blocking_redir stdout
|
|||
par_colsep_0 bug --colsep 0
|
||||
par_colsep_0 OK
|
||||
par_colsep_0 OK
|
||||
par_cr_newline_header ### --header : should set named replacement string if input line ends in \r\n
|
||||
par_cr_newline_header bar
|
||||
par_csv col1"x3"-new
|
||||
par_csv line col2-new2
|
||||
par_csv line col3-col 4
|
||||
|
|
|
|||
Loading…
Reference in a new issue