parset: support for associative arrays in bash, ksh, zsh.

This commit is contained in:
Ole Tange 2021-07-11 16:59:27 +02:00
parent 373e5cccc9
commit 71d008a5a4
22 changed files with 335 additions and 341 deletions

View file

@ -74,25 +74,21 @@ run() {
# OK
return 0
else
keyserver1=keys.gnupg.net
keyserver2=pool.sks-keyservers.net
if gpg --keyserver "$keyserver1" --recv-key 0xFFFFFFF1 ||
gpg --keyserver "$keyserver2" --recv-key 0xFFFFFFF1 ; then
if gpg --keyserver "$keyserver1" --recv-key 0x88888888 ||
gpg --keyserver "$keyserver2" --recv-key 0x88888888; then
keyservers="pgp.surf.nl
keyserver.bazon.ru
agora.cenditel.gob.ve
pgp.benny-baumann.de"
for keyserver in $keyservers ; do
if gpg --keyserver "$keyserver" --recv-key 0xFFFFFFF1 &&
gpg --keyserver "$keyserver" --recv-key 0x88888888 ; then
# OK
return 0
else
fi
done
echo
echo "Cannot fetch keyID 0x88888888, so the signature cannot be checked."
return 1
fi
else
echo
echo "Cannot fetch keyID 0xFFFFFFF1, so the signature cannot be checked."
return 1
fi
fi
else
# GnuPG not installed
echo
@ -104,7 +100,7 @@ run() {
# Check signature - in case ftpmirror.gnu.org is compromised
if fetch_keys; then
if gpg --with-fingerprint "$latest".tar.bz2.sig 2>&1 |
perl -e 'exit not grep /^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888/, <>'; then
perl -e 'exit not grep /^Primary key fingerprint: BE9C B493 81DE 3166 A3BC..66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610..7E7B D1AB 4516 8888 8888/, <>'; then
# Source code signed by Ole Tange <ole@tange.dk>
# KeyID FFFFFFF1/88888888
true

14
README
View file

@ -40,13 +40,13 @@ installation.
$ (wget -O - pi.dk/3 || lynx -source pi.dk/3 || curl pi.dk/3/ || \
fetch -o - http://pi.dk/3 ) > install.sh
$ sha1sum install.sh | grep c82233e7da3166308632ac8c34f850c0
12345678 c82233e7 da316630 8632ac8c 34f850c0
$ md5sum install.sh | grep ae3d7aac5e15cf3dfc87046cfc5918d2
ae3d7aac 5e15cf3d fc87046c fc5918d2
$ sha512sum install.sh | grep dfc00d823137271a6d96225cea9e89f533ff6c81f
9c5198d5 31a3b755 b7910ece 3a42d206 c804694d fc00d823 137271a6 d96225ce
a9e89f53 3ff6c81f f52b298b ef9fb613 2d3f9ccd 0e2c7bd3 c35978b5 79acb5ca
$ sha1sum install.sh | grep 883c667e01eed62f975ad28b6d50e22a
12345678 883c667e 01eed62f 975ad28b 6d50e22a
$ md5sum install.sh | grep cc21b4c943fd03e93ae1ae49e28573c0
cc21b4c9 43fd03e9 3ae1ae49 e28573c0
$ sha512sum install.sh | grep da012ec113b49a54e705f86d51e784ebced224fdf
79945d9d 250b42a4 2067bb00 99da012e c113b49a 54e705f8 6d51e784 ebced224
fdff3f52 ca588d64 e75f6033 61bd543f d631f592 2f87ceb2 ab034149 6df84a35
$ bash install.sh
This will literally install faster than reading the rest of this

View file

@ -90,15 +90,15 @@ https://build.opensuse.org/package/show/home:tange/parallel
lbry://@GnuParallel#4/parallel-20210322.tar.bz2
An easy way to support GNU Parallel is to tip on LBRY.
:
If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include a command that uses GNU Parallel if you feel like it.
Thumbnail: https://www.gnu.org/software/parallel/logo-gray+black10000.png
Tags: gnu parallel software
# An easy way to support GNU Parallel is to tip on LBRY.
#
# :
#
# If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include a command that uses GNU Parallel if you feel like it.
#
# Thumbnail: https://www.gnu.org/software/parallel/logo-gray+black10000.png
#
# Tags: gnu parallel software
file_path="parallel-20210622.tar.bz2"
@ -109,45 +109,13 @@ license="GNU GPLv3 or later"
thumbnail_url=https://www.gnu.org/software/parallel/logo-gray+black10000.png
channel_name="@GnuParallel"
tags_opt='--tag gnu --tag parallel --tag free --tag software'
description="An easy way to support GNU Parallel is to tip on LBRY.
GNU Parallel 20210622 ('Protasevich') [stable] has been released. It is available for download at: lbry://@GnuParallel:4
No new functionality was introduced so this is a good candidate for a stable release.
Please help spreading GNU Parallel by making a testimonial video like Juan Sierra Pons: http://www.elsotanillo.net/wp-content/uploads/GnuParallel_JuanSierraPons.mp4
It does not have to be as detailed as Juan's. It is perfectly fine if you just say your name, and what field you are using GNU Parallel for.
Quote of the month:
GNU Parallel makes my life so much easier.
I'm glad I don't have to implement multi-threaded Python scripts on the regular.
-- Fredrick Brennan @fr_brennan@twitter
New in this release:
* Bug fixes and man page updates.
News about GNU Parallel:
* How to use GNU Parallel https://techtipbits.com/linux/how-to-use-gnu-parallel/
* How to Speed Up Bash Scripts with Multithreading and GNU Parallel https://adamtheautomator.com/how-to-speed-up-bash-scripts-with-multithreading-and-gnu-parallel/
* Use Parallel to split by line https://madflex.de/use-parallel-to-split-by-line/
* Optimizing long batch processes or ETL by using buff/cache properly II (parallelizing network operations) http://www.elsotanillo.net/2021/06/optimizing-long-batch-processes-or-etl-by-using-buff-cache-properly-ii-parallelizing-network-operations/
* Parallelization 3: GNU Parallel https://www.youtube.com/watch?v=Rl06WD60afA
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
GNU Parallel - For people who live life in the parallel lane.
If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include a command that uses GNU Parallel if you feel like it.
$(perl -ne '/About\sGNU\sParallel/ and exit;/It\sis\savailable/..0 and print' doc/release_new_version)
"
echo "$description" | grep '<<.*>>' && echo STOP STOP STOP
echo "$description"
lbrynet publish \
--bid=0.01 \
@ -162,7 +130,7 @@ lbrynet publish \
--thumbnail_url="$thumbnail_url" \
--channel_name="$channel_name" \
--release_time="$release_time" \
# --release_time="$release_time" \
== Update website ==
@ -284,9 +252,9 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20210622 ('Protasevich') released <<[stable]>>
Subject: GNU Parallel 20210722 ('Bill Cosby/Moïse/Derek Chauvin') released <<[stable]>>
GNU Parallel 20210622 ('Protasevich') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4
GNU Parallel 20210722 ('') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4
<<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -296,9 +264,6 @@ It does not have to be as detailed as Juan's. It is perfectly fine if you just s
Quote of the month:
GNU Parallel makes my life so much easier.
I'm glad I don't have to implement multi-threaded Python scripts on the regular.
-- Fredrick Brennan @fr_brennan@twitter
<<>>
New in this release:
@ -307,16 +272,6 @@ New in this release:
News about GNU Parallel:
* How to use GNU Parallel https://techtipbits.com/linux/how-to-use-gnu-parallel/
* How to Speed Up Bash Scripts with Multithreading and GNU Parallel https://adamtheautomator.com/how-to-speed-up-bash-scripts-with-multithreading-and-gnu-parallel/
* Use Parallel to split by line https://madflex.de/use-parallel-to-split-by-line/
* Optimizing long batch processes or ETL by using buff/cache properly II (parallelizing network operations) http://www.elsotanillo.net/2021/06/optimizing-long-batch-processes-or-etl-by-using-buff-cache-properly-ii-parallelizing-network-operations/
* Parallelization 3: GNU Parallel https://www.youtube.com/watch?v=Rl06WD60afA
<<>>

View file

@ -385,7 +385,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -369,13 +369,6 @@ _parset_main() {
# parset "var_a4 var_b4 var_c4" echo ::: {1..3}
# echo $var_c4
_make_TEMP() {
# mktemp does not exist on some OS
perl -e 'use File::Temp qw(tempfile);
$ENV{"TMPDIR"} ||= "/tmp";
print((tempfile(DIR=>$ENV{"TMPDIR"}, TEMPLATE => "parXXXXX"))[1])'
}
_parset_NAME="$1"
if [ "$_parset_NAME" = "" ] ; then
echo parset: Error: No destination variable given. >&2
@ -391,7 +384,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
@ -406,46 +399,18 @@ _parset_main() {
return 255
fi
shift
echo "$_parset_NAME" |
perl -ne 'chomp;for (split /[, ]/) {
# Allow: var_32 var[3]
if(not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/) {
print STDERR "parset: Error: $_ is an invalid variable name.\n";
print STDERR "parset: Error: Variable names must be letter followed by letters or digits.\n";
print STDERR "parset: Error: Usage:\n";
print STDERR "parset: Error: parset varname GNU Parallel options and command\n";
$exitval = 255;
}
}
exit $exitval;
' || return 255
_exit_FILE=`_make_TEMP`
if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then
# $_parset_NAME contains , or space
# Split on , or space to get the names
# shellcheck disable=SC2016,SC2046
eval "$(
# Compute results into files
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
# var1=`cat tmpfile1; rm tmpfile1`
# var2=`cat tmpfile2; rm tmpfile2`
parallel -k --plain -q echo '{2}=`cat {1}; rm {1}`' :::: - :::+ $(
echo "$_parset_NAME" | perl -pe 's/,/ /g'
)
);
"
# Bash: declare -A myassoc=( )
# Zsh: typeset -A myassoc=( )
# Ksh: typeset -A myassoc=( )
if (typeset -p "$_parset_NAME" 2>/dev/null; echo) |
perl -ne 'exit not (/^declare[^=]+-A|^typeset[^=]+-A/)' ; then
# This is an associative array
eval "`$_parset_PARALLEL_PRG -k --parset assoc,"$_parset_NAME" "$@"`"
# The eval returns the function!
else
# $_parset_NAME does not contain , or space
# => $_parset_NAME is the name of the array to put data into
# Supported in: bash zsh ksh mksh
# Arrays do not work in: sh ash dash
eval "$_parset_NAME=( $(
# Compute results into files. Save exit value
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "'
) )"
# This is a normal array or a list of variable names
eval "`$_parset_PARALLEL_PRG -k --parset var,"$_parset_NAME" "$@"`"
# The eval returns the function!
fi
unset _parset_NAME _parset_PARALLEL_PRG _parallel_exit_CODE
# Unset _exit_FILE before return
eval "unset _exit_FILE; return \`cat $_exit_FILE; rm $_exit_FILE\`"
}

View file

@ -385,7 +385,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -69,8 +69,10 @@ env_parallel() {
# sh on UnixWare: readonly TIMEOUT
# ash: readonly var='val'
# ksh: var='val'
s/^(readonly )?([^= ]*)(=.*|)$/$2/ or
# mksh: PIPESTATUS[0]
s/^(readonly )?([^=\[ ]*?)(\[\d+\])?(=.*|)$/$2/ or
# bash: declare -ar BASH_VERSINFO=([0]="4" [1]="4")
# zsh: typeset -r var='val'
s/^\S+\s+\S+\s+(\S[^=]*)(=.*|$)/$1/;
$_ } <>;
$vars = join "|",map { quotemeta $_ } @r;
@ -214,7 +216,7 @@ env_parallel() {
(_names_of_ALIASES;
_names_of_FUNCTIONS;
_names_of_VARIABLES) |
cat > $HOME/.parallel/ignored_vars
cat > "$HOME"/.parallel/ignored_vars
return 0
fi
@ -346,13 +348,6 @@ _parset_main() {
# parset "var_a4 var_b4 var_c4" echo ::: {1..3}
# echo $var_c4
_make_TEMP() {
# mktemp does not exist on some OS
perl -e 'use File::Temp qw(tempfile);
$ENV{"TMPDIR"} ||= "/tmp";
print((tempfile(DIR=>$ENV{"TMPDIR"}, TEMPLATE => "parXXXXX"))[1])'
}
_parset_NAME="$1"
if [ "$_parset_NAME" = "" ] ; then
echo parset: Error: No destination variable given. >&2
@ -368,7 +363,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
@ -383,45 +378,18 @@ _parset_main() {
return 255
fi
shift
echo "$_parset_NAME" |
perl -ne 'chomp;for (split /[, ]/) {
# Allow: var_32 var[3]
if(not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/) {
print STDERR "parset: Error: $_ is an invalid variable name.\n";
print STDERR "parset: Error: Variable names must be letter followed by letters or digits.\n";
print STDERR "parset: Error: Usage:\n";
print STDERR "parset: Error: parset varname GNU Parallel options and command\n";
$exitval = 255;
}
}
exit $exitval;
' || return 255
_exit_FILE=`_make_TEMP`
if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then
# $_parset_NAME contains , or space
# Split on , or space to get the names
eval "$(
# Compute results into files
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
# var1=`cat tmpfile1; rm tmpfile1`
# var2=`cat tmpfile2; rm tmpfile2`
parallel -q echo {2}='`cat {1}; rm {1}`' :::: - :::+ $(
echo "$_parset_NAME" | perl -pe 's/,/ /g'
)
);
"
# Bash: declare -A myassoc=( )
# Zsh: typeset -A myassoc=( )
# Ksh: typeset -A myassoc=( )
if (typeset -p "$_parset_NAME" 2>/dev/null; echo) |
perl -ne 'exit not (/^declare[^=]+-A|^typeset[^=]+-A/)' ; then
# This is an associative array
eval "`$_parset_PARALLEL_PRG -k --parset assoc,"$_parset_NAME" "$@"`"
# The eval returns the function!
else
# $_parset_NAME does not contain , or space
# => $_parset_NAME is the name of the array to put data into
# Supported in: bash zsh ksh mksh
# Arrays do not work in: sh ash dash
eval "$_parset_NAME=( $(
# Compute results into files. Save exit value
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "'
) )"
# This is a normal array or a list of variable names
eval "`$_parset_PARALLEL_PRG -k --parset var,"$_parset_NAME" "$@"`"
# The eval returns the function!
fi
unset _parset_NAME _parset_PARALLEL_PRG _parallel_exit_CODE
# Unset _exit_FILE before return
eval "unset _exit_FILE; return \`cat $_exit_FILE; rm $_exit_FILE\`"
}

View file

@ -74,6 +74,7 @@ env_parallel() {
# mksh: PIPESTATUS[0]
s/^(readonly )?([^=\[ ]*?)(\[\d+\])?(=.*|)$/$2/ or
# bash: declare -ar BASH_VERSINFO=([0]="4" [1]="4")
# zsh: typeset -r var='val'
s/^\S+\s+\S+\s+(\S[^=]*)(=.*|$)/$1/;
$_ } <>;
$vars = join "|",map { quotemeta $_ } @r;
@ -217,7 +218,7 @@ env_parallel() {
(_names_of_ALIASES;
_names_of_FUNCTIONS;
_names_of_VARIABLES) |
cat > $HOME/.parallel/ignored_vars
cat > "$HOME"/.parallel/ignored_vars
return 0
fi
@ -349,13 +350,6 @@ _parset_main() {
# parset "var_a4 var_b4 var_c4" echo ::: {1..3}
# echo $var_c4
_make_TEMP() {
# mktemp does not exist on some OS
perl -e 'use File::Temp qw(tempfile);
$ENV{"TMPDIR"} ||= "/tmp";
print((tempfile(DIR=>$ENV{"TMPDIR"}, TEMPLATE => "parXXXXX"))[1])'
}
_parset_NAME="$1"
if [ "$_parset_NAME" = "" ] ; then
echo parset: Error: No destination variable given. >&2
@ -371,7 +365,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
@ -386,45 +380,18 @@ _parset_main() {
return 255
fi
shift
echo "$_parset_NAME" |
perl -ne 'chomp;for (split /[, ]/) {
# Allow: var_32 var[3]
if(not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/) {
print STDERR "parset: Error: $_ is an invalid variable name.\n";
print STDERR "parset: Error: Variable names must be letter followed by letters or digits.\n";
print STDERR "parset: Error: Usage:\n";
print STDERR "parset: Error: parset varname GNU Parallel options and command\n";
$exitval = 255;
}
}
exit $exitval;
' || return 255
_exit_FILE=`_make_TEMP`
if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then
# $_parset_NAME contains , or space
# Split on , or space to get the names
eval "$(
# Compute results into files
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
# var1=`cat tmpfile1; rm tmpfile1`
# var2=`cat tmpfile2; rm tmpfile2`
parallel -q echo {2}='`cat {1}; rm {1}`' :::: - :::+ $(
echo "$_parset_NAME" | perl -pe 's/,/ /g'
)
);
"
# Bash: declare -A myassoc=( )
# Zsh: typeset -A myassoc=( )
# Ksh: typeset -A myassoc=( )
if (typeset -p "$_parset_NAME" 2>/dev/null; echo) |
perl -ne 'exit not (/^declare[^=]+-A|^typeset[^=]+-A/)' ; then
# This is an associative array
eval "`$_parset_PARALLEL_PRG -k --parset assoc,"$_parset_NAME" "$@"`"
# The eval returns the function!
else
# $_parset_NAME does not contain , or space
# => $_parset_NAME is the name of the array to put data into
# Supported in: bash zsh ksh mksh
# Arrays do not work in: sh ash dash
eval "$_parset_NAME=( $(
# Compute results into files. Save exit value
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "'
) )"
# This is a normal array or a list of variable names
eval "`$_parset_PARALLEL_PRG -k --parset var,"$_parset_NAME" "$@"`"
# The eval returns the function!
fi
unset _parset_NAME _parset_PARALLEL_PRG _parallel_exit_CODE
# Unset _exit_FILE before return
eval "unset _exit_FILE; return \`cat $_exit_FILE; rm $_exit_FILE\`"
}

View file

@ -390,7 +390,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -208,7 +208,7 @@ env_parallel() {
(_names_of_ALIASES;
_names_of_FUNCTIONS;
_names_of_VARIABLES) |
cat > $HOME/.parallel/ignored_vars
cat > "$HOME"/.parallel/ignored_vars
return 0
fi
@ -340,13 +340,6 @@ _parset_main() {
# parset "var_a4 var_b4 var_c4" echo ::: {1..3}
# echo $var_c4
_make_TEMP() {
# mktemp does not exist on some OS
perl -e 'use File::Temp qw(tempfile);
$ENV{"TMPDIR"} ||= "/tmp";
print((tempfile(DIR=>$ENV{"TMPDIR"}, TEMPLATE => "parXXXXX"))[1])'
}
_parset_NAME="$1"
if [ "$_parset_NAME" = "" ] ; then
echo parset: Error: No destination variable given. >&2
@ -362,7 +355,7 @@ _parset_main() {
return 255
fi
if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
echo "parset 20210623 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
@ -377,45 +370,18 @@ _parset_main() {
return 255
fi
shift
echo "$_parset_NAME" |
perl -ne 'chomp;for (split /[, ]/) {
# Allow: var_32 var[3]
if(not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/) {
print STDERR "parset: Error: $_ is an invalid variable name.\n";
print STDERR "parset: Error: Variable names must be letter followed by letters or digits.\n";
print STDERR "parset: Error: Usage:\n";
print STDERR "parset: Error: parset varname GNU Parallel options and command\n";
$exitval = 255;
}
}
exit $exitval;
' || return 255
_exit_FILE=`_make_TEMP`
if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then
# $_parset_NAME contains , or space
# Split on , or space to get the names
# Compute results into files
# var1=`cat tmpfile1; rm tmpfile1`
# var2=`cat tmpfile2; rm tmpfile2`
eval "$(
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
parallel -q echo {2}='`cat {1}; rm {1}`' :::: - :::+ $(
echo "$_parset_NAME" | perl -pe 's/,/ /g'
)
);
"
# Bash: declare -A myassoc=( )
# Zsh: typeset -A myassoc=( )
# Ksh: typeset -A myassoc=( )
if (typeset -p "$_parset_NAME" 2>/dev/null; echo) |
perl -ne 'exit not (/^declare[^=]+-A|^typeset[^=]+-A/)' ; then
# This is an associative array
eval "`$_parset_PARALLEL_PRG -k --parset assoc,"$_parset_NAME" "$@"`"
# The eval returns the function!
else
# $_parset_NAME does not contain , or space
# => $_parset_NAME is the name of the array to put data into
# Supported in: bash zsh ksh mksh
# Arrays do not work in: sh ash dash
# Compute results into files. Save exit value
eval "$_parset_NAME=( $(
($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "'
) )"
# This is a normal array or a list of variable names
eval "`$_parset_PARALLEL_PRG -k --parset var,"$_parset_NAME" "$@"`"
# The eval returns the function!
fi
unset _parset_NAME _parset_PARALLEL_PRG _parallel_exit_CODE
# Unset _exit_FILE before return
eval "unset _exit_FILE; return \`cat $_exit_FILE; rm $_exit_FILE\`"
}

View file

@ -26,7 +26,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20210622;
$Global::version = 20210623;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($opt::version) {

View file

@ -1672,6 +1672,7 @@ sub options_hash() {
"hgrp|hostgrp|hostgroup|hostgroups" => \$opt::hostgroups,
"embed" => \$opt::embed,
"filter=s" => \@opt::filter,
"parset=s" => \$opt::parset,
);
}
@ -1717,6 +1718,40 @@ sub get_options_from_array($@) {
return $retval;
}
sub parse_parset() {
$Global::progname = "parset";
@Global::parset_vars = split /[ ,]/, $opt::parset;
my $var_or_assoc = shift @Global::parset_vars;
# Legal names: var _v2ar arrayentry[2]
my @illegal = (grep { not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/ }
@Global::parset_vars);
if(@illegal) {
::error
("@illegal is an invalid variable name.",
"Variable names must be letter followed by letters or digits.",
"Usage:",
" parset varname GNU Parallel options and command");
wait_and_exit(255);
}
if($var_or_assoc eq "assoc") {
my $var = shift @Global::parset_vars;
print "$var=(";
$Global::parset = "assoc";
$Global::parset_endstring=")\n";
} elsif($var_or_assoc eq "var") {
if($#Global::parset_vars > 0) {
$Global::parset = "var";
} else {
my $var = shift @Global::parset_vars;
print "$var=(";
$Global::parset = "array";
$Global::parset_endstring=")\n";
}
} else {
::die_bug("parset: unknown '$opt::parset'");
}
}
sub parse_options(@) {
# Returns: N/A
init_globals();
@ -1744,6 +1779,7 @@ sub parse_options(@) {
}
::debug("init","Global::shell $Global::shell\n");
$Global::cshell = $Global::shell =~ m:(/[-a-z]*)?csh:;
if(defined $opt::parset) { parse_parset(); }
if(defined $opt::X) { $Global::ContextReplace = 1; }
if(defined $opt::silent) { $Global::verbose = 0; }
if(defined $opt::null) { $/ = "\0"; }
@ -2187,7 +2223,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20210622;
$Global::version = 20210623;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -3859,7 +3895,8 @@ sub progress() {
my $arg = $Global::newest_job ?
$Global::newest_job->{'commandline'}->
replace_placeholders(["\257<\257>"],0,0) : "";
# These chars mess up display in the terminal
# These chars mess up display in the terminal in US-ASCII
# and in some combinations as UTF8 (e.g. ঐ ও ঔ ক 𐅪 𐅫 𐅬)
$arg =~ tr/[\011-\016\033\302-\365]//d;
my $eta_dhms = ::seconds_to_time_units($eta);
my $bar_text =
@ -4949,6 +4986,10 @@ sub wait_and_exit($) {
# Avoid: Warning: unable to close filehandle properly: No space
# left on device during global destruction.
$SIG{__WARN__} = sub {};
if($opt::parset) {
# Make the shell script return $error
print "$Global::parset_endstring\nreturn $error";
}
exit($error);
}
@ -5737,8 +5778,8 @@ sub which(@) {
# ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh
# posh rbash rc rush rzsh sash sh static-sh tcsh yash zsh
my @shells = (qw(ash bash bsd-csh csh dash fdsh fish fizsh
ksh ksh93 lksh mksh pdksh posh rbash rc rush rzsh sash sh
my @shells = (qw(ash bash bsd-csh csh dash fdsh fish fizsh ksh
ksh93 lksh mksh pdksh posh rbash rc rush rzsh sash sh
static-sh tcsh yash zsh -sh -csh -bash),
'-sh (sh)' # sh on FreeBSD
);
@ -6166,12 +6207,12 @@ sub debug(@) {
$Global::debug or return;
@_ = grep { defined $_ ? $_ : "" } @_;
if($Global::debug eq "all" or $Global::debug eq $_[0]) {
if($Global::fd{1}) {
# Original stdout was saved
my $stdout = $Global::fd{1};
print $stdout @_[1..$#_];
if($Global::fd{2}) {
# Original stderr was saved
my $stderr = $Global::fd{2};
print $stderr @_[1..$#_];
} else {
print @_[1..$#_];
print STDERR @_[1..$#_];
}
}
}
@ -10272,8 +10313,6 @@ sub print($) {
$self->print_linebuffer($fdno,$in_fh,$out_fd);
} elsif($opt::files) {
$self->print_files($fdno,$in_fh,$out_fd);
} elsif($opt::tag or defined $opt::tagstring) {
$self->print_tag($fdno,$in_fh,$out_fd);
} else {
$self->print_normal($fdno,$in_fh,$out_fd);
}
@ -10624,10 +10663,6 @@ sub print_linebuffer($) {
}
}
sub print_tag(@) {
return print_normal(@_);
}
sub free_ressources() {
my $self = shift;
if(not $opt::ungroup) {
@ -10641,9 +10676,54 @@ sub free_ressources() {
}
}
sub print_parset($) {
# Wrap output with shell script code to set as variables
my $self = shift;
my ($fdno,$in_fh,$out_fh) = @_;
my $outputlength = 0;
::debug("parset","print $Global::parset");
if($Global::parset eq "assoc") {
# eval "`echo 'declare -A myassoc; myassoc=(
# Each:
# [$'a\tb']=$'a\tb\tc ddd'
# End:
# )'`"
print '[',::Q($self->{'commandline'}->
replace_placeholders(["\257<\257>"],0,0)),']=';
} elsif($Global::parset eq "array") {
# eval "`echo 'myassoc=(
# Each:
# $'a\tb\tc ddd'
# End:
# )'`"
} elsif($Global::parset eq "var") {
# var=$'a\tb\tc ddd'
if(not @Global::parset_vars) {
::error("Too few named destination variables");
::wait_and_exit(255);
}
print shift @Global::parset_vars,"=";
}
local $/ = "\n";
my $tag = $self->tag();
my @out;
while(<$in_fh>) {
$outputlength += length $_;
# Tag lines with \r, too
$_ =~ s/(?<=[\r])(?=.|$)/$tag/gs;
push @out, $tag,$_;
}
# Remove last newline
# This often makes it easier to use the output in shell
@out and ${out[$#out]} =~ s/\n$//s;
print ::Q(join("",@out)),"\n";
return $outputlength;
}
sub print_normal($) {
my $self = shift;
my ($fdno,$in_fh,$out_fd) = @_;
my ($fdno,$in_fh,$out_fh) = @_;
my $buf;
close $self->fh($fdno,"w");
if($? and $opt::compress) {
@ -10656,7 +10736,9 @@ sub print_normal($) {
my $outputlength = 0;
my @output;
if($opt::tag or $opt::tagstring) {
if($Global::parset and $fdno == 1) {
$outputlength += $self->print_parset($fdno,$in_fh,$out_fh);
} elsif($opt::tag or $opt::tagstring) {
# Read line by line
local $/ = "\n";
my $tag = $self->tag();
@ -10664,14 +10746,15 @@ sub print_normal($) {
$outputlength += length $_;
# Tag lines with \r, too
$_ =~ s/(?<=[\r])(?=.|$)/$tag/gs;
print $out_fd $tag,$_;
print $out_fh $tag,$_;
if($Global::membuffer) {
push @{$self->{'output'}{$fdno}}, $tag, $_;
}
}
} else {
# Most efficient way of copying data from $in_fh to $out_fh
while(sysread($in_fh,$buf,131072)) {
print $out_fd $buf;
print $out_fh $buf;
$outputlength += length $buf;
if($Global::membuffer) {
push @{$self->{'output'}{$fdno}}, $buf;

View file

@ -65,6 +65,12 @@ Put output into vars B<$seq, $pwd, $ls>:
parset "${into_vars[*]}" ::: "seq 10" pwd ls
echo "$ls"
Put output into associative array B<myassoc> (not supported for mksh):
typeset -A myassoc
parset myassoc seq ::: 4 5 ::: 6 7
echo "${myassoc[4 7]}"
The commands to run can be an array:
cmd=("echo first" "echo '<<joe \"double space\" cartoon>>'" "pwd")
@ -102,17 +108,6 @@ or Bash/Zsh/Ksh process substitution:
echo "${res[1]}"
echo "${res[99]}"
Put output into an associative array (Bash only):
input=("value A" "value B")
parset res echo This is ::: "${input[@]}"
# Zip the input and res arrays to a single associative array
declare -A myassoc
for ((i=0; $i<${#input[@]}; i++)); do
myassoc[${input[i]}]=${res[i]}
done
echo "${myassoc["value A"]}"
=head3 Installation

View file

@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20210622;
$Global::version = 20210623;
$Global::progname = 'sql';
# This must be done first as this may exec myself

View file

@ -887,6 +887,7 @@ par_empty_command() {
echo B: $b
}
par_empty_input_on_stdin() {
echo 'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=910470'
echo 'This should give no output'

View file

@ -289,7 +289,7 @@ par_test_detected_shell() {
rm -f "$tmp"
cp $(which "$shell") "$tmp"
chmod +x "$tmp"
$tmp -c 'parallel -Dinit echo ::: 1; true' |
stdout $tmp -c 'parallel -Dinit echo ::: 1; true' |
grep Global::shell
rm "$tmp"
}
@ -297,7 +297,7 @@ par_test_detected_shell() {
test_known_shell_c() {
shell="$1"
$shell -c 'parallel -Dinit echo ::: 1; true' |
stdout $shell -c 'parallel -Dinit echo ::: 1; true' |
grep Global::shell
}
export -f test_known_shell_c
@ -305,7 +305,7 @@ par_test_detected_shell() {
test_known_shell_pipe() {
shell="$1"
echo 'parallel -Dinit echo ::: 1; true' |
$shell | grep Global::shell
stdout $shell | grep Global::shell
}
export -f test_known_shell_pipe

View file

@ -8,6 +8,38 @@
# Each should be taking 3-10s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_parset_assoc_arr() {
mytest=$(cat <<'EOF'
mytest() {
shell=`basename $SHELL`
echo 'parset into an assoc array'
. `which env_parallel.$shell`
parset "var1,var2 var3" echo ::: 'val 1' 'val 2' 'val 3'
echo "$var1 $var2 $var3"
parset array echo ::: 'val 1' 'val 2' 'val 3'
echo "${array[0]} ${array[1]} ${array[2]}"
typeset -A assoc
parset assoc echo ::: 'val 1' 'val 2' 'val 3'
echo "${assoc[val 1]} ${assoc[val 2]} ${assoc[val 3]}"
echo Bad var name
parset -badname echo ::: 'val 1' 'val 2' 'val 3'
echo Too few var names
parset v1,v2 echo ::: 'val 1' 'val 2' 'val 3'
echo "$v2"
echo Exit value
parset assoc exit ::: 1 0 0 1; echo $?
parset array exit ::: 1 0 0 1; echo $?
parset v1,v2,v3,v4 exit ::: 1 0 0 1; echo $?
echo Stderr to stderr
parset assoc ls ::: no-such-file
parset array ls ::: no-such-file
parset v1,v2 ls ::: no-such-file1 no-such-file2
}
EOF
)
parallel -k --tag --nonall -Sksh@lo,bash@lo,zsh@lo "$mytest;mytest 2>&1"
}
par_shebang() {
echo '### Test different shebangs'
gp() {

View file

@ -105,11 +105,14 @@ par_dummy() {
par_tmpdir() {
export TMPDIR="/tmp/parsort dir"
rm -rf "$TMPDIR"
echo Should fail
echo Fail: no such dir | parsort
mkdir "$TMPDIR"
echo OK | parsort
chmod -w "$TMPDIR"
echo Should fail
echo Fail: writeable | parsort
rm -rf "$TMPDIR"
}
setup

View file

@ -1676,12 +1676,9 @@ par_test_detected_shell test_unknown_shell ksh93 Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell mksh Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell posh Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell rbash Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell rush Local configuration error occurred.
par_test_detected_shell test_unknown_shell rush Contact the systems administrator for further assistance.
par_test_detected_shell test_unknown_shell rzsh Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell sash Global::shell /usr/bin/sh
par_test_detected_shell test_unknown_shell sh Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell static-sh test_unknown_shell_static-sh: applet not found
par_test_detected_shell test_unknown_shell tcsh Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell yash Global::shell /usr/bin/bash
par_test_detected_shell test_unknown_shell zsh Global::shell /usr/bin/bash
@ -1696,8 +1693,6 @@ par_test_detected_shell test_known_shell_c ksh93 Global::shell /usr/bin/ksh93
par_test_detected_shell test_known_shell_c mksh Global::shell /usr/bin/mksh
par_test_detected_shell test_known_shell_c posh Global::shell /usr/bin/posh
par_test_detected_shell test_known_shell_c rbash Global::shell /usr/bin/rbash
par_test_detected_shell test_known_shell_c rush Local configuration error occurred.
par_test_detected_shell test_known_shell_c rush Contact the systems administrator for further assistance.
par_test_detected_shell test_known_shell_c rzsh Global::shell /usr/bin/rzsh
par_test_detected_shell test_known_shell_c sash Global::shell /usr/bin/sh
par_test_detected_shell test_known_shell_c sh Global::shell /usr/bin/sh
@ -1716,8 +1711,6 @@ par_test_detected_shell test_known_shell_pipe ksh93 Global::shell /usr/bin/ksh93
par_test_detected_shell test_known_shell_pipe mksh Global::shell /usr/bin/mksh
par_test_detected_shell test_known_shell_pipe posh Global::shell /usr/bin/posh
par_test_detected_shell test_known_shell_pipe rbash Global::shell /usr/bin/rbash
par_test_detected_shell test_known_shell_pipe rush Local configuration error occurred.
par_test_detected_shell test_known_shell_pipe rush Contact the systems administrator for further assistance.
par_test_detected_shell test_known_shell_pipe rzsh Global::shell /usr/bin/rzsh
par_test_detected_shell test_known_shell_pipe sash Global::shell /usr/bin/sh
par_test_detected_shell test_known_shell_pipe sh Global::shell /usr/bin/sh

View file

@ -208,6 +208,69 @@ par_multiline_commands echo finish 4
par_multiline_commands parallel: Warning: Command lines contain newline. Forcing --null.
par_multiline_commands 4
par_multiline_commands finish 4
par_parset_assoc_arr bash@lo parset into an assoc array
par_parset_assoc_arr bash@lo val 1 val 2 val 3
par_parset_assoc_arr bash@lo val 1 val 2 val 3
par_parset_assoc_arr bash@lo val 1 val 2 val 3
par_parset_assoc_arr bash@lo Bad var name
par_parset_assoc_arr bash@lo parset: Error: -badname is an invalid variable name.
par_parset_assoc_arr bash@lo parset: Error: Variable names must be letter followed by letters or digits.
par_parset_assoc_arr bash@lo parset: Error: Usage:
par_parset_assoc_arr bash@lo parset: Error: parset varname GNU Parallel options and command
par_parset_assoc_arr bash@lo Too few var names
par_parset_assoc_arr bash@lo parset: Error: Too few named destination variables
par_parset_assoc_arr bash@lo val 2
par_parset_assoc_arr bash@lo Exit value
par_parset_assoc_arr bash@lo 2
par_parset_assoc_arr bash@lo 2
par_parset_assoc_arr bash@lo 2
par_parset_assoc_arr bash@lo Stderr to stderr
par_parset_assoc_arr bash@lo ls: cannot access 'no-such-file': No such file or directory
par_parset_assoc_arr bash@lo ls: cannot access 'no-such-file': No such file or directory
par_parset_assoc_arr bash@lo ls: cannot access 'no-such-file1': No such file or directory
par_parset_assoc_arr bash@lo ls: cannot access 'no-such-file2': No such file or directory
par_parset_assoc_arr ksh@lo parset into an assoc array
par_parset_assoc_arr ksh@lo val 1 val 2 val 3
par_parset_assoc_arr ksh@lo val 1 val 2 val 3
par_parset_assoc_arr ksh@lo val 1 val 2 val 3
par_parset_assoc_arr ksh@lo Bad var name
par_parset_assoc_arr ksh@lo parset: Error: -badname is an invalid variable name.
par_parset_assoc_arr ksh@lo parset: Error: Variable names must be letter followed by letters or digits.
par_parset_assoc_arr ksh@lo parset: Error: Usage:
par_parset_assoc_arr ksh@lo parset: Error: parset varname GNU Parallel options and command
par_parset_assoc_arr ksh@lo Too few var names
par_parset_assoc_arr ksh@lo parset: Error: Too few named destination variables
par_parset_assoc_arr ksh@lo val 2
par_parset_assoc_arr ksh@lo Exit value
par_parset_assoc_arr ksh@lo 2
par_parset_assoc_arr ksh@lo 2
par_parset_assoc_arr ksh@lo 2
par_parset_assoc_arr ksh@lo Stderr to stderr
par_parset_assoc_arr ksh@lo ls: cannot access 'no-such-file': No such file or directory
par_parset_assoc_arr ksh@lo ls: cannot access 'no-such-file': No such file or directory
par_parset_assoc_arr ksh@lo ls: cannot access 'no-such-file1': No such file or directory
par_parset_assoc_arr ksh@lo ls: cannot access 'no-such-file2': No such file or directory
par_parset_assoc_arr zsh@lo parset into an assoc array
par_parset_assoc_arr zsh@lo val 1 val 2 val 3
par_parset_assoc_arr zsh@lo val 1 val 2
par_parset_assoc_arr zsh@lo val 1 val 2 val 3
par_parset_assoc_arr zsh@lo Bad var name
par_parset_assoc_arr zsh@lo parset: Error: -badname is an invalid variable name.
par_parset_assoc_arr zsh@lo parset: Error: Variable names must be letter followed by letters or digits.
par_parset_assoc_arr zsh@lo parset: Error: Usage:
par_parset_assoc_arr zsh@lo parset: Error: parset varname GNU Parallel options and command
par_parset_assoc_arr zsh@lo Too few var names
par_parset_assoc_arr zsh@lo parset: Error: Too few named destination variables
par_parset_assoc_arr zsh@lo val 2
par_parset_assoc_arr zsh@lo Exit value
par_parset_assoc_arr zsh@lo 2
par_parset_assoc_arr zsh@lo 2
par_parset_assoc_arr zsh@lo 2
par_parset_assoc_arr zsh@lo Stderr to stderr
par_parset_assoc_arr zsh@lo ls: cannot access 'no-such-file': No such file or directory
par_parset_assoc_arr zsh@lo ls: cannot access 'no-such-file': No such file or directory
par_parset_assoc_arr zsh@lo ls: cannot access 'no-such-file1': No such file or directory
par_parset_assoc_arr zsh@lo ls: cannot access 'no-such-file2': No such file or directory
par_pipe_regexp ### --pipe --regexp
par_pipe_regexp Record
par_pipe_regexp A2, Start, 5

View file

@ -43,6 +43,13 @@ par_r deb3b52408074435b24fda0e3f624799 -
par_r deb3b52408074435b24fda0e3f624799 -
par_r deb3b52408074435b24fda0e3f624799 -
par_r deb3b52408074435b24fda0e3f624799 -
par_tmpdir Should fail
par_tmpdir parsort: Error: Tmpdir '/tmp/parsort dir' does not exist.
par_tmpdir parsort: Error: Try 'mkdir '/tmp/parsort dir''
par_tmpdir OK
par_tmpdir Should fail
par_tmpdir parsort: Error: Tmpdir '/tmp/parsort dir' is not writable.
par_tmpdir parsort: Error: Try 'chmod +w '/tmp/parsort dir''
par_z ### parsort -z
par_z 583c59e3574e550f6810d0ad20e7ab88 -
par_z 507d17cd3d746cdeeccdb8ad11860181 -

View file

@ -1196,9 +1196,9 @@ par_ksh_man work,
par_ksh_man too
par_ksh_man This may never work
par_ksh_man https://unix.stackexchange.com/questions/457031/extract-full-function-definitions
par_ksh_man /bin/ksh: line XXX: syntax error at line XXX: `(' unexpected
par_ksh_man /bin/ksh: line XXX: syntax error at line XXX: `(' unexpected
par_ksh_man /bin/ksh: line XXX: syntax error at line XXX: `(' unexpected
par_ksh_man /bin/ksh: line XXX: syntax error at line XXX: `{' unmatched
par_ksh_man /bin/ksh: line XXX: syntax error at line XXX: `{' unmatched
par_ksh_man /bin/ksh: line XXX: syntax error at line XXX: `{' unmatched
par_ksh_man exit value 2 should be 2
par_ksh_man Unknown option: no-such-option
par_ksh_man exit value 255 should be 255