From 477c86c7786aa18b39d13598aa845262dfc0a0d2 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 30 Jan 2023 00:40:56 +0100 Subject: [PATCH] parallel: Fixed bug #63703: --shebang-wrap with input with space. --- doc/release_new_version | 15 +- src/parallel | 5 +- testsuite/tests-to-run/parallel-local21.sh | 224 +++++++++++---------- testsuite/wanted-results/parallel-local21 | 42 ++-- testsuite/wanted-results/parallel-tutorial | 2 +- 5 files changed, 152 insertions(+), 136 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 8a4a117a..00162151 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -262,26 +262,23 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20230122 ('Bolsanaristas') released +Subject: GNU Parallel 20230222 ('Leopard/Monterrey Park/Peru<<>>') released -GNU Parallel 20230122 ('Bolsanaristas') has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20230222 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4 Quote of the month: - Colorful output - parallel, with --color flag - tasks more vibrant now - -- ChatGPT + <<>> New in this release: +<<>> + * Bug fixes and man page updates. News about GNU Parallel: -* The Best Ethical Hacking Tools of 2023 (and their basic usage) https://www.purevpn.com/blog/the-best-hacking-tools-of-2023/#11_GNU_Parallel - -* GNU Parallel: criando atividades em paralelo com shell script https://www.vivaolinux.com.br/artigo/GNU-Parallel-criando-atividades-em-paralelo-com-shell-script/ +<<>> GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel b/src/parallel index 69adfdac..7f07ef23 100755 --- a/src/parallel +++ b/src/parallel @@ -3301,10 +3301,11 @@ sub read_options() { } else { @options = shift @ARGV; } - my $script = Q(shift @ARGV); + my $script = Q(Q(shift @ARGV)); + my @args = map{ Q($_) } @ARGV; # exec myself to split $ARGV[0] into separate fields exec "$0 --_pipe-means-argfiles @options @parser $script ". - "::: @ARGV"; + "::: @args"; } } if($ARGV[0] =~ / --shebang(-?wrap)? /) { diff --git a/testsuite/tests-to-run/parallel-local21.sh b/testsuite/tests-to-run/parallel-local21.sh index f1773840..2e327f43 100644 --- a/testsuite/tests-to-run/parallel-local21.sh +++ b/testsuite/tests-to-run/parallel-local21.sh @@ -4,23 +4,31 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +tmp="$(mktemp -d)" +# Test with tmpdir with spaces +TMPDIR="$tmp/ " +export TMPDIR +mkdir -p "$TMPDIR" + par_basic_shebang_wrap() { echo "### Test basic --shebang-wrap" - cat < /tmp/basic--shebang-wrap + script="$TMPDIR"/basic--shebang-wrap + cat < "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl print "Shebang from perl with args @ARGV\n"; EOF - chmod 755 /tmp/basic--shebang-wrap - /tmp/basic--shebang-wrap arg1 arg2 + chmod 755 "$script" + args() { echo arg1; echo arg2; echo "arg3.1 arg3.2"; } + "$script" "$(args)" echo "### Test basic --shebang-wrap Same as" - parallel -k /usr/bin/perl /tmp/basic--shebang-wrap ::: arg1 arg2 + parallel -k /usr/bin/perl "'$script'" ::: "$(args)" echo "### Test basic --shebang-wrap stdin" - (echo arg1; echo arg2) | /tmp/basic--shebang-wrap + args | "$script" echo "### Test basic --shebang-wrap Same as" - (echo arg1; echo arg2) | parallel -k /usr/bin/perl /tmp/basic--shebang-wrap - rm /tmp/basic--shebang-wrap + args | parallel -k /usr/bin/perl "'$script'" + rm "$script" } par_shebang_with_parser_options() { @@ -28,179 +36,184 @@ par_shebang_with_parser_options() { seq 4 5 >/tmp/in45 echo "### Test --shebang-wrap with parser options" - cat < /tmp/with-parser--shebang-wrap + script="$TMPDIR"/with-parser--shebang-wrap + cat < "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl -p print "Shebang from perl with args @ARGV\n"; EOF - chmod 755 /tmp/with-parser--shebang-wrap - /tmp/with-parser--shebang-wrap /tmp/in12 /tmp/in45 + chmod 755 "$script" + "$script" /tmp/in12 /tmp/in45 echo "### Test --shebang-wrap with parser options Same as" - parallel -k /usr/bin/perl -p /tmp/with-parser--shebang-wrap ::: /tmp/in12 /tmp/in45 + parallel -k /usr/bin/perl -p "'$script'" ::: /tmp/in12 /tmp/in45 echo "### Test --shebang-wrap with parser options stdin" - (echo /tmp/in12; echo /tmp/in45) | /tmp/with-parser--shebang-wrap + (echo /tmp/in12; echo /tmp/in45) | "$script" echo "### Test --shebang-wrap with parser options Same as" - (echo /tmp/in12; echo /tmp/in45) | parallel -k /usr/bin/perl /tmp/with-parser--shebang-wrap - rm /tmp/with-parser--shebang-wrap - + (echo /tmp/in12; echo /tmp/in45) | parallel -k /usr/bin/perl "'$script'" + rm "$script" echo "### Test --shebang-wrap --pipe with parser options" - cat < /tmp/pipe--shebang-wrap + script="$TMPDIR"/pipe--shebang-wrap + cat < "$script" #!/usr/local/bin/parallel --shebang-wrap -k --pipe /usr/bin/perl -p print "Shebang from perl with args @ARGV\n"; EOF - chmod 755 /tmp/pipe--shebang-wrap + chmod 755 "$script" echo "### Test --shebang-wrap --pipe with parser options stdin" - cat /tmp/in12 /tmp/in45 | /tmp/pipe--shebang-wrap + cat /tmp/in12 /tmp/in45 | "$script" echo "### Test --shebang-wrap --pipe with parser options Same as" - cat /tmp/in12 /tmp/in45 | parallel -k --pipe /usr/bin/perl\ -p /tmp/pipe--shebang-wrap - rm /tmp/pipe--shebang-wrap + cat /tmp/in12 /tmp/in45 | parallel -k --pipe /usr/bin/perl\ -p "'$script'" + rm "$script" rm /tmp/in12 rm /tmp/in45 } par_shebang_wrap_perl() { - F=/tmp/shebang_wrap_perl - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_perl + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl print "Arguments @ARGV\n"; EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_python() { - F=/tmp/shebang_wrap_python - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_python + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/python3 import sys +sys.argv.pop(0) print('Arguments', str(sys.argv)) EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_bash() { - F=/tmp/shebang_wrap_bash - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_bash + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /bin/bash echo Arguments "$@" EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_sh() { - F=/tmp/shebang_wrap_sh - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_sh + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /bin/sh echo Arguments "$@" EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_ksh() { - F=/tmp/shebang_wrap_ksh - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_ksh + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ksh echo Arguments "$@" EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_zsh() { - F=/tmp/shebang_wrap_zsh - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_zsh + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/zsh echo Arguments "$@" EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_csh() { - F=/tmp/shebang_wrap_csh - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_csh + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /bin/csh echo Arguments "$argv" EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_tcl() { - F=/tmp/shebang_wrap_tcl - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_tcl + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/tclsh puts "Arguments $argv" EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_R() { - F=/tmp/shebang_wrap_R - cat <<'EOF' > $F + # Rscript fucks up if $TMPDIR contains space + TMPDIR=/tmp + script="$TMPDIR"/shebang_wrap_R + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave args <- commandArgs(trailingOnly = TRUE) print(paste("Arguments ",args)) EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } + par_shebang_wrap_gnuplot() { - F=/tmp/shebang_wrap_gnuplot - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_gnuplot + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k ARG={} /usr/bin/gnuplot print "Arguments ", system('echo $ARG') EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_ruby() { - F=/tmp/shebang_wrap_ruby - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_ruby + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ruby print "Arguments " puts ARGV EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_octave() { - F=/tmp/shebang_wrap_octave - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_octave + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/octave printf ("Arguments"); @@ -210,27 +223,27 @@ for i = 1:nargin endfor printf ("\n"); EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_clisp() { - F=/tmp/shebang_wrap_clisp - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_clisp + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/clisp (format t "~&~S~&" 'Arguments) (format t "~&~S~&" *args*) EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_php() { - F=/tmp/shebang_wrap_php - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_php + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/php EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_nodejs() { - F=/tmp/shebang_wrap_nodejs - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_nodejs + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/node var myArgs = process.argv.slice(2); console.log('Arguments ', myArgs); EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_lua() { - F=/tmp/shebang_wrap_lua - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_lua + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/lua io.write "Arguments" @@ -271,22 +284,22 @@ for a = 1, #arg do end print("") EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } par_shebang_wrap_csharp() { - F=/tmp/shebang_wrap_csharp - cat <<'EOF' > $F + script="$TMPDIR"/shebang_wrap_csharp + cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k ARGV={} /usr/bin/csharp var argv = Environment.GetEnvironmentVariable("ARGV"); print("Arguments "+argv); EOF - chmod 755 $F - $F arg1 arg2 arg3 - rm $F + chmod 755 "$script" + "$script" arg1 arg2 "arg3.1 arg3.2" + rm "$script" } export -f $(compgen -A function | grep par_) @@ -294,3 +307,4 @@ export -f $(compgen -A function | grep par_) # -j6 was fastest #compgen -A function | grep par_ | sort | parallel -j$P --tag -k '{} 2>&1' compgen -A function | grep par_ | LC_ALL=C sort | parallel -j6 --tag -k '{} 2>&1' +rmdir "$TMPDIR" "$tmp" diff --git a/testsuite/wanted-results/parallel-local21 b/testsuite/wanted-results/parallel-local21 index 5d8ff9b9..a950e8f5 100644 --- a/testsuite/wanted-results/parallel-local21 +++ b/testsuite/wanted-results/parallel-local21 @@ -1,15 +1,19 @@ par_basic_shebang_wrap ### Test basic --shebang-wrap par_basic_shebang_wrap Shebang from perl with args arg1 par_basic_shebang_wrap Shebang from perl with args arg2 +par_basic_shebang_wrap Shebang from perl with args arg3.1 arg3.2 par_basic_shebang_wrap ### Test basic --shebang-wrap Same as par_basic_shebang_wrap Shebang from perl with args arg1 par_basic_shebang_wrap Shebang from perl with args arg2 +par_basic_shebang_wrap Shebang from perl with args arg3.1 arg3.2 par_basic_shebang_wrap ### Test basic --shebang-wrap stdin par_basic_shebang_wrap Shebang from perl with args arg1 par_basic_shebang_wrap Shebang from perl with args arg2 +par_basic_shebang_wrap Shebang from perl with args arg3.1 arg3.2 par_basic_shebang_wrap ### Test basic --shebang-wrap Same as par_basic_shebang_wrap Shebang from perl with args arg1 par_basic_shebang_wrap Shebang from perl with args arg2 +par_basic_shebang_wrap Shebang from perl with args arg3.1 arg3.2 par_shebang_with_parser_options ### Test --shebang-wrap with parser options par_shebang_with_parser_options Shebang from perl with args par_shebang_with_parser_options 1 @@ -67,55 +71,55 @@ par_shebang_with_parser_options Shebang from perl with args par_shebang_with_parser_options 5 par_shebang_wrap_R [1] "Arguments arg1" par_shebang_wrap_R [1] "Arguments arg2" -par_shebang_wrap_R [1] "Arguments arg3" +par_shebang_wrap_R [1] "Arguments arg3.1 arg3.2" par_shebang_wrap_bash Arguments arg1 par_shebang_wrap_bash Arguments arg2 -par_shebang_wrap_bash Arguments arg3 +par_shebang_wrap_bash Arguments arg3.1 arg3.2 par_shebang_wrap_clisp ARGUMENTS par_shebang_wrap_clisp ("arg1") par_shebang_wrap_clisp ARGUMENTS par_shebang_wrap_clisp ("arg2") par_shebang_wrap_clisp ARGUMENTS -par_shebang_wrap_clisp ("arg3") +par_shebang_wrap_clisp ("arg3.1 arg3.2") par_shebang_wrap_csh Arguments arg1 par_shebang_wrap_csh Arguments arg2 -par_shebang_wrap_csh Arguments arg3 +par_shebang_wrap_csh Arguments arg3.1 arg3.2 par_shebang_wrap_csharp Arguments arg1 par_shebang_wrap_csharp Arguments arg2 -par_shebang_wrap_csharp Arguments arg3 +par_shebang_wrap_csharp Arguments arg3.1 arg3.2 par_shebang_wrap_gnuplot Arguments arg1 par_shebang_wrap_gnuplot Arguments arg2 -par_shebang_wrap_gnuplot Arguments arg3 +par_shebang_wrap_gnuplot Arguments arg3.1 arg3.2 par_shebang_wrap_ksh Arguments arg1 par_shebang_wrap_ksh Arguments arg2 -par_shebang_wrap_ksh Arguments arg3 +par_shebang_wrap_ksh Arguments arg3.1 arg3.2 par_shebang_wrap_lua Arguments arg1 par_shebang_wrap_lua Arguments arg2 -par_shebang_wrap_lua Arguments arg3 +par_shebang_wrap_lua Arguments arg3.1 arg3.2 par_shebang_wrap_nodejs Arguments [ 'arg1' ] par_shebang_wrap_nodejs Arguments [ 'arg2' ] -par_shebang_wrap_nodejs Arguments [ 'arg3' ] +par_shebang_wrap_nodejs Arguments [ 'arg3.1 arg3.2' ] par_shebang_wrap_octave Arguments arg1 par_shebang_wrap_octave Arguments arg2 -par_shebang_wrap_octave Arguments arg3 +par_shebang_wrap_octave Arguments arg3.1 arg3.2 par_shebang_wrap_perl Arguments arg1 par_shebang_wrap_perl Arguments arg2 -par_shebang_wrap_perl Arguments arg3 +par_shebang_wrap_perl Arguments arg3.1 arg3.2 par_shebang_wrap_php Arguments arg1 par_shebang_wrap_php Arguments arg2 -par_shebang_wrap_php Arguments arg3 -par_shebang_wrap_python Arguments ['/tmp/shebang_wrap_python', 'arg1'] -par_shebang_wrap_python Arguments ['/tmp/shebang_wrap_python', 'arg2'] -par_shebang_wrap_python Arguments ['/tmp/shebang_wrap_python', 'arg3'] +par_shebang_wrap_php Arguments arg3.1 arg3.2 +par_shebang_wrap_python Arguments ['arg1'] +par_shebang_wrap_python Arguments ['arg2'] +par_shebang_wrap_python Arguments ['arg3.1 arg3.2'] par_shebang_wrap_ruby Arguments arg1 par_shebang_wrap_ruby Arguments arg2 -par_shebang_wrap_ruby Arguments arg3 +par_shebang_wrap_ruby Arguments arg3.1 arg3.2 par_shebang_wrap_sh Arguments arg1 par_shebang_wrap_sh Arguments arg2 -par_shebang_wrap_sh Arguments arg3 +par_shebang_wrap_sh Arguments arg3.1 arg3.2 par_shebang_wrap_tcl Arguments arg1 par_shebang_wrap_tcl Arguments arg2 -par_shebang_wrap_tcl Arguments arg3 +par_shebang_wrap_tcl Arguments {arg3.1 arg3.2} par_shebang_wrap_zsh Arguments arg1 par_shebang_wrap_zsh Arguments arg2 -par_shebang_wrap_zsh Arguments arg3 +par_shebang_wrap_zsh Arguments arg3.1 arg3.2 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 4aaedd12..9e1a7509 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -1302,7 +1302,7 @@ scientific publication, please cite: author = {Tange, Ole}, - year = 2022, + year = 2023, note = {{GNU Parallel is a general parallelizer to run multiple serial command line programs in parallel without changing them.}},