diff --git a/Makefile.am b/Makefile.am index 5b4296d2..f3947d91 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,7 +19,7 @@ torresults: torsync: - torsocks rsync -Hazv --exclude='*.{pdf,gz,bz2,html,deb}' --exclude authorized_keys --exclude .vagrant/ --exclude actual-results --delete ./ tortest:privat/parallel/ + torsocks rsync -Hazv --exclude='*.{pdf,gz,bz2,html,deb}' --exclude authorized_keys --exclude .vagrant/ --exclude actual-results --exclude '*hwdep*' --delete ./ tortest:privat/parallel/ make tortest web: diff --git a/Makefile.in b/Makefile.in index 4739f6a6..d8e5b66e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -787,7 +787,7 @@ torresults: torsocks rsync -Hazv --progress tortest:privat/parallel/testsuite/actual-results/ testsuite/actual-results/ torsync: - torsocks rsync -Hazv --exclude='*.{pdf,gz,bz2,html,deb}' --exclude authorized_keys --exclude .vagrant/ --exclude actual-results --delete ./ tortest:privat/parallel/ + torsocks rsync -Hazv --exclude='*.{pdf,gz,bz2,html,deb}' --exclude authorized_keys --exclude .vagrant/ --exclude actual-results --exclude '*hwdep*' --delete ./ tortest:privat/parallel/ make tortest web: diff --git a/NEWS b/NEWS index 215adeac..99ffd8c0 100644 --- a/NEWS +++ b/NEWS @@ -6,8 +6,22 @@ New in this release: * {:%fmt} Use printf formatting of replacement strings. +* Bug fixes and man page updates. + +News about GNU Parallel: + +* Powerful GNU parallel, more than a loop + https://www.linkedin.com/pulse/powerful-gnu-parallel-more-than-loop-zhenguo-zhang-18dxc + +* How To Increase File Transfer Speed Using Parallel Rsync? + https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/ + +* Converting WebP Images to PNG Using parallel and dwebp + https://bytefreaks.net/2024/07/27 + +* Turbocharging the Box CLI with GNU Parallel + https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0 -<<>> 20240722 diff --git a/README b/README index b5b0d4f6..6f5c666e 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20240722.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20240722.tar.bz2.sig - gpg parallel-20240722.tar.bz2.sig - bzip2 -dc parallel-20240722.tar.bz2 | tar xvf - - cd parallel-20240722 + wget https://ftpmirror.gnu.org/parallel/parallel-20240822.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20240822.tar.bz2.sig + gpg parallel-20240822.tar.bz2.sig + bzip2 -dc parallel-20240822.tar.bz2 | tar xvf - + cd parallel-20240822 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20240722.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20240722.tar.bz2.sig - gpg parallel-20240722.tar.bz2.sig - bzip2 -dc parallel-20240722.tar.bz2 | tar xvf - - cd parallel-20240722 + wget https://ftpmirror.gnu.org/parallel/parallel-20240822.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20240822.tar.bz2.sig + gpg parallel-20240822.tar.bz2.sig + bzip2 -dc parallel-20240822.tar.bz2 | tar xvf - + cd parallel-20240822 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2024, July 22). GNU Parallel 20240722 ('Assange'). - Zenodo. https://doi.org/10.5281/zenodo.12789352 + Tange, O. (2024, August 22). GNU Parallel 20240822 ('Southport'). + Zenodo. https://doi.org/10.5281/zenodo.13357237 Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. diff --git a/configure b/configure index a5bd25ad..04ed591c 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for parallel 20240722. +# Generated by GNU Autoconf 2.71 for parallel 20240822. # # Report bugs to . # @@ -610,8 +610,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20240722' -PACKAGE_STRING='parallel 20240722' +PACKAGE_VERSION='20240822' +PACKAGE_STRING='parallel 20240822' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1246,7 +1246,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20240722 to adapt to many kinds of systems. +\`configure' configures parallel 20240822 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1313,7 +1313,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20240722:";; + short | recursive ) echo "Configuration of parallel 20240822:";; esac cat <<\_ACEOF @@ -1390,7 +1390,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20240722 +parallel configure 20240822 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -1427,7 +1427,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20240722, which was +It was created by parallel $as_me 20240822, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2379,7 +2379,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20240722' + VERSION='20240822' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -3026,7 +3026,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20240722, which was +This file was extended by parallel $as_me 20240822, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3090,7 +3090,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -parallel config.status 20240722 +parallel config.status 20240822 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index b534459e..32137513 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel],[20240722],[bug-parallel@gnu.org]) +AC_INIT([parallel],[20240822],[bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 2cf696ca..ff5b8a0e 100644 --- a/doc/haikus +++ b/doc/haikus @@ -4,10 +4,6 @@ Quote of the month: - honestly the coolest software i've ever seen gotta be gnu parallel or - ffmpeg, nothing like them - -- @scootykins scoot - Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts. -- @mechanicker@twitter Dhruva @@ -231,6 +227,10 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + honestly the coolest software i've ever seen gotta be gnu parallel or + ffmpeg, nothing like them + -- @scootykins scoot + parallel is frickin great for launching jobs on multiple machines. Ansible and Jenkins and others may be good too but I was able to jump right in with parallel. diff --git a/doc/release_new_version b/doc/release_new_version index 202a318b..3bf78946 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -268,35 +268,34 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20240822 ('Bangladesh/Southport/<<>>') released <<[stable]>> +Subject: GNU Parallel 20240822 ('Southport') released -GNU Parallel 20240822 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20240822 ('Southport') has been released. It is available for download at: lbry://@GnuParallel:4 Quote of the month: - <<>> + honestly the coolest software i've ever seen gotta be gnu parallel or + ffmpeg, nothing like them + -- @scootykins scoot New in this release: -* A format string can be added to replacement strings: {#:%04d} +* --match Match input source with regexp to set replacement fields. -<<>> - -<<* No new features. This is a candidate for a stable release.>> +* {:%fmt} Use printf formatting of replacement strings. * Bug fixes and man page updates. + News about GNU Parallel: -https://www.linkedin.com/pulse/powerful-gnu-parallel-more-than-loop-zhenguo-zhang-18dxc +* Powerful GNU parallel, more than a loop https://www.linkedin.com/pulse/powerful-gnu-parallel-more-than-loop-zhenguo-zhang-18dxc -https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/ +* How To Increase File Transfer Speed Using Parallel Rsync? https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/ -https://bytefreaks.net/2024/07/27 +* Converting WebP Images to PNG Using parallel and dwebp https://bytefreaks.net/2024/07/27 -https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0 - -<<>> +* Turbocharging the Box CLI with GNU Parallel https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0 GNU Parallel - For people who live life in the parallel lane. diff --git a/packager/obs/home-tange/parallel/.osc/_files b/packager/obs/home-tange/parallel/.osc/_files index cfab1682..2a174e39 100644 --- a/packager/obs/home-tange/parallel/.osc/_files +++ b/packager/obs/home-tange/parallel/.osc/_files @@ -1,8 +1,8 @@ - + - - - - - + + + + + diff --git a/packager/obs/home-tange/parallel/.osc/parallel.spec b/packager/obs/home-tange/parallel/.osc/parallel.spec index 37675477..fdcac68e 100644 --- a/packager/obs/home-tange/parallel/.osc/parallel.spec +++ b/packager/obs/home-tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20240722 +Version: 20240822 Release: 2.1 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/src/env_parallel.ash b/src/env_parallel.ash index f060a607..a057da1c 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 40bbbffe..a41e08da 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -605,7 +605,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index c819e7c6..699d923e 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 4c51ba9f..9b335cd1 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index b102c505..8f9300ea 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -570,7 +570,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.pod b/src/env_parallel.pod index b4181873..c0ebc5ae 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -384,7 +384,7 @@ B<--session> is supported. =back -=head2 fish (alpha testing) +=head2 fish (beta testing) =head3 Installation diff --git a/src/env_parallel.sh b/src/env_parallel.sh index d2ea36d9..04a9c148 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 6e0d811b..4cda7a00 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240822 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index 9348a762..24ba9761 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20240723; +$Global::version = 20240822; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 10a3cb28..0873f252 100755 --- a/src/parallel +++ b/src/parallel @@ -2798,7 +2798,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20240808; + $Global::version = 20240822; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -3806,7 +3806,7 @@ sub pQ($) { } sub unquote_printf() { - # Convert \t \n \r \000 \0 + # Convert \t \n \r \xFF \000 \0 # Inputs: # $string = string with \t \n \r \num \0 # Returns: @@ -3815,7 +3815,11 @@ sub unquote_printf() { s/\\t/\t/g; s/\\n/\n/g; s/\\r/\r/g; + # Hex: \xFF + s/\\x([0-9a-fA-F]+)/eval 'sprintf "\\x'.$1.'"'/ge; + # Octal: \007 s/\\(\d\d\d)/eval 'sprintf "\\'.$1.'"'/ge; + # Single digit octal: \7 s/\\(\d)/eval 'sprintf "\\'.$1.'"'/ge; return $_; } @@ -5852,8 +5856,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2024, July 22). GNU Parallel 20240722 ('Assange').", - " Zenodo. https://doi.org/10.5281/zenodo.12789352", + " Tange, O. (2024, August 22). GNU Parallel 20240822 ('Southport').", + " Zenodo. https://doi.org/10.5281/zenodo.13357237", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice @@ -5885,8 +5889,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2024, July 22). GNU Parallel 20240722 ('Assange').", - " Zenodo. https://doi.org/10.5281/zenodo.12789352", + " Tange, O. (2024, August 22). GNU Parallel 20240822 ('Southport').", + " Zenodo. https://doi.org/10.5281/zenodo.13357237", "", # Before changing these line, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and @@ -6012,20 +6016,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2024_12789352,", + "\@software{tange_2024_13357237,", " author = {Tange, Ole},", - " title = {GNU Parallel 20240722 ('Assange')},", - " month = Jul,", + " title = {GNU Parallel 20240822 ('Southport')},", + " month = Aug,", " year = 2023,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.12789352},", - " url = {https://doi.org/10.5281/zenodo.12789352}", + " doi = {10.5281/zenodo.13357237},", + " url = {https://doi.org/10.5281/zenodo.13357237}", "}", "", - "(Feel free to use \\nocite{tange_2024_12789352})", + "(Feel free to use \\nocite{tange_2024_13357237})", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and diff --git a/src/parallel.pod b/src/parallel.pod index b7587861..ee6965a2 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -861,7 +861,7 @@ B<--trim n>. I is a Perl Regular Expression: https://perldoc.perl.org/perlre.html -See also: B<--csv> B<{>IB<}> B<--trim> B<--link> +See also: B<--csv> B<{>IB<}> B<--trim> B<--link> B<--match> =item B<--combineexec> I @@ -942,7 +942,7 @@ See also: B<--compress> Treat input as CSV-format. -B<--colsep> sets the field delimiter. It works very much like +B<--colsep> sets the field delimiter. B<--csv> works very much like B<--colsep> except it deals correctly with quoting. Compare: echo '"1 big, 2 small","2""x4"" plank",12.34' | @@ -957,9 +957,9 @@ Even quoted newlines are parsed correctly: echo 'Line 2 in field 1";value 2') | parallel --csv --colsep ';' echo Field 1: {1} Field 2: {2} -When used with B<--pipe> only pass full CSV-records. +When used with B<--pipe> it will only pass full CSV-records. -See also: B<--pipe> B<--link> B<{>IB<}> B<--colsep> B<--header> +See also: B<--pipe> B<--link> B<{>IB<}> B<--colsep> B<--header> B<--match> =item B<--ctag> (obsolete: use B<--color> B<--tag>) @@ -1000,10 +1000,10 @@ See also: TIME POSTFIXES B<--retries> B<--ssh-delay> =item B<-d> I -Input items are terminated by I. +Input records are terminated by I. The specified delimiter may be characters, C-style character escapes -such as \n, or octal or hexadecimal escape codes. Octal and +such as \n, or octal (\012) or hexadecimal (\x0A) escape codes. Octal and hexadecimal escape codes are understood as for the printf command. See also: B<--colsep> @@ -1800,9 +1800,13 @@ See also: B<-X> B<--xargs> Match input source with regexp to set replacement fields. -Each capture group i.e. '(...)' can be accessed with a replacement -field. The replacement fields are named B<{m.n}> where m is the input -source and n is the capture group number: +With B<--match> you can often avoid pre-processing your input with +B or similar to extract the relevant fields. + +You can access each capture group i.e. the parenthesis in regexp +'(...)' with a replacement field. The replacement fields are named +B<{m.n}> where m is the input source and n is the capture group +number: parallel --match '(.*),(.*)_(.*)' echo {1.2} {1.3} {1.1} \ ::: Gold,Heart_of @@ -1821,7 +1825,7 @@ input source 1: parallel --match '' --match '([a-z]+)' echo {1} {2.1} \ ::: Telephone ::: 10sanitizer -See also: B<{>IB<}> +See also: B<{>IB<}> B<--colsep> =item B<--memfree> I diff --git a/src/parallel_examples.pod b/src/parallel_examples.pod index dd33f7b3..99d6e8fc 100644 --- a/src/parallel_examples.pod +++ b/src/parallel_examples.pod @@ -178,6 +178,10 @@ You have a bunch of file. You want them sorted into dirs. The dir of each file should be named the first letter of the file name. parallel 'mkdir -p {=s/(.).*/$1/=}; mv {} {=s/(.).*/$1/=}' ::: * + +In practice you would probably not use a perl replacement string but +instead B<--match>: + parallel --match '(.)' 'mkdir -p {1.1} && mv {} {1.1}' ::: * @@ -199,15 +203,21 @@ Match first initial and last name: Re-arrange (stupid) US date format into (nice) ISO-8601: - parallel --match '(.*)/(.*)/(.*)' echo {1.3}-{1.1}-{1.2} \ - ::: 12/31/1969 01/19/2038 06/01/2002 + parallel --match '(.*)/(.*)/(.*)' echo {1.3}-{1.1:%02d}-{1.2:%02d} \ + ::: 12/31/1969 1/19/2038 6/1/2002 Match url into domain and path: parallel --match 'https://(.*?)/(.*)' echo Domain: {1.1} Path: {1.2} \ ::: https://example.com/dir/page https://gnu.org/s/parallel -Match host:port from a log file: +Get URLs into dirs named by 2nd level domain name, +e.g. https://www.gnu.org/s/parallel will be put into the dir gnu.org. + + cat urls | parallel --match '//[^/]*?([^/.]+\.[^/.]+)/' \ + 'mkdir -p {1.1} && cd {1.1} && wget {}' + +Match host.domain:port from a log file: cat log | parallel --match '\b([a-z0-9.]+):(\d+)\b' echo host:{1.1} port:{1.2} @@ -241,6 +251,40 @@ Make an international dialing prefix table: Note how input source 2 reuses the B<--match> from input source 1. + +=head2 EXAMPLE: Replacement fields from CSV file with headers + +This is an advanced example. You have: + + Date;Name;Location + 3/8/1978;"Beeblebrox; Zaphod";"Betelgeuse V" + 10/12/1979;"Dent; Arthur";Earth + 1/5/1981;Slartibartfast;Magrathea + +You want: + + Z. Beeblebrox: 1978-03-08/BET + A. Dent: 1979-10-12/EAR + Slartibartfast: 1981-01-05/MAG + +Run: + + parallel --csv --colsep ';' --header : --match "(\d+)/(\d+)/(\d+)" \ + --match "^([^;]+)(; (.))?" --match "(...)" \ + echo '{=Name.3 s/(.)/$1. /;=}'{Name.1}: \ + {Date.3}-{Date.1:%02d}-{Date.2:%02d}/'{=Location.1 $_=uc =}' \ + :::: people.csv + +B<--csv> parses the input as CSV with B<--colsep ;> as the separator - +dealing correctly with quoted strings. The input is split into 3 +columns. B<--header :> makes the columns available as +B<{>IB<}>. Each column has their corresponding B<--match> +so each field can be accessed as +B<{>I.I<#>B<}>. B is a perl expression that +appends ". " if the name has an initial. B<:%02d> formats single +digits as two digits. B upper cases the argument. + + =head2 EXAMPLE: Calling Bash functions If the composed command is longer than a line, it becomes hard to @@ -371,6 +415,10 @@ If the barcode does not contain '_R1', you can do: parallel --plus myprocess {} {/_R1/_R2} ::: *_R1.fq.gz +Or you can use B<--match>: + + parallel --match '(.*)_R1(.*)' myprocess {} {1.1}_R2{1.2} ::: *_R1.fq.gz + =head2 EXAMPLE: Removing strings from the argument @@ -393,6 +441,10 @@ To remove a string anywhere you can use regular expressions with parallel --plus echo {/demo_/} ::: demo_mycode remove_demo_here +You can often also use B<--match>: + + parallel --match '(.*)demo_(.*)' echo {1.1}{1.2} ::: demo_mycode remove_demo_here + =head2 EXAMPLE: Download 24 images for each of the past 30 days @@ -482,6 +534,7 @@ downloads 10 videos in parallel. B<{=> and B<=}> mark a perl expression. B perl-quotes the string. B is the date in ISO8601 with time. + =head2 EXAMPLE: Save output in ISO8601 dirs Save output from B every second into dirs named diff --git a/src/parsort b/src/parsort index a4204142..9205d574 100755 --- a/src/parsort +++ b/src/parsort @@ -137,7 +137,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20240723; +$Global::version = 20240822; if($opt::version) { version(); exit 0; } # Remove -D and --parallel=N my @s = (grep { ! /^-D$|^--parallel=\S+$/ } diff --git a/src/sql b/src/sql index 0b8570bd..ce32b4e8 100755 --- a/src/sql +++ b/src/sql @@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20240723; + $Global::version = 20240822; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index c3676a31..a68e14e5 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -10,14 +10,21 @@ install_packages() { INSTALL=${INSTALL:-"sudo apt -y install"} # The testsuite depends on this: - test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl" - test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs" - test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl" + test_pkgs="imagemagick expect pv zenity" + test_pkgs="$test_pkgs autossh sshpass lsh-client lsh-utils jq" + test_pkgs="$test_pkgs libpod-simple-perl libtext-csv-perl" + test_pkgs="$test_pkgs libtest-nowarnings-perl pod2pdf" + test_pkgs="$test_pkgs xemacs21" + test_pkgs="$test_pkgs gawk lua5.3 clisp php-cli nodejs guile-3.0 fsharp" + test_pkgs="$test_pkgs mono-csharp-shell octave openjdk-11-jdk-headless tcl" + test_pkgs="$test_pkgs racket libevent-dev python-is-python3" test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386 virtualbox" - test_pkgs="$test_pkgs libtest-nowarnings-perl xemacs21 pv zenity" - test_pkgs="$test_pkgs lsh-client lsh-utils torsocks net-tools python-is-python3" - test_pkgs="$test_pkgs reiserfsprogs btrfs-progs exfatprogs f2fs-tools xfsprogs" + test_pkgs="$test_pkgs torsocks net-tools" + test_pkgs="$test_pkgs reiserfsprogs btrfs-progs exfatprogs f2fs-tools" + test_pkgs="$test_pkgs xfsprogs" + test_snap_pkgs="groovy julia" + # Debian package packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake" packaging_pkgs="$packaging_pkgs python3-m2crypto alien python3-pip" @@ -30,14 +37,14 @@ install_packages() { sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap" # Compression - compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2" + compression_pkgs="zstd clzip lz4 lzop pigz pixz gzip plzip pbzip2" compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip" compression_pkgs_missing="pxz" # Shells # (csh = bsd-csh that is broken) - # (ksh93 gone) - shell_pkgs="ash dash fdclone fish fizsh ksh ksh93u+m mksh posh rc rush sash" + # (ash ksh93 gone) + shell_pkgs="dash fdclone fish fizsh ksh ksh93u+m mksh posh rc rush sash" shell_pkgs="$shell_pkgs tcsh yash zsh busybox-static" # Databases @@ -59,6 +66,13 @@ install_packages() { echo Packages missing return 1 fi + if sudo parallel -j1 snap install --classic ::: $test_snap_pkgs; then + # OK + true + else + echo Packages missing: $test_snap_pkgs + return 1 + fi if parallel -j1 --tag --halt now,success=1 $INSTALL ::: $alt_database ; then # OK true @@ -85,7 +99,11 @@ install_sqlite2() { cd /tmp/sqlite2 git clone https://github.com/sqlite/sqlite cd sqlite - git checkout a2cfd57491ca357eda2a206cbcd94c6e6c7d7d93 + git checkout cc83b6e071ba69943f175a038d2625ae3d6abf47 + # New GCC needs , so add that + perl -i -pe \ + 's/echo (.#include )(.)/printf $1\\n#include \\n$2/' \ + Makefile.in ./configure --prefix=/usr/local && make -j && sudo make install } @@ -276,7 +294,7 @@ add_server_to_hosts() { } vagrant_up() { - parallel -j0 --ll --tag 'cd {};vagrant up' ::: vagrant/*/* + parallel -j0 --ll --tag 'cd {//};vagrant up' ::: vagrant/*/*/Vagrantfile # After this you should be able to 'vagrant ssh' } @@ -347,9 +365,9 @@ add_key_to_auth_key() { cat ~/.ssh/*pub >> ~/.ssh/authorized_keys parallel --tag --timeout 20 sshaddvagrant ::: $vagrant_names parallel-server{1..4} parallel --tag --timeout 20 sshaddlocal ::: $ips lo localhost - parallel --tag --timeout 5 ssh vagrant@{} echo OK ::: $vagrant_names parallel-server{1..4} + parallel --tag --timeout 5 ssh vagrant@{} echo ssh vagrant@{} OK ::: $vagrant_names parallel-server{1..4} parallel ssh vagrant@{} 'echo "PATH=\$PATH:\$HOME/bin" >> .bashrc' ::: $vagrant_names - parallel --tag --timeout 5 ssh {} echo OK ::: $ips lo localhost + parallel --tag --timeout 5 ssh {} echo ssh {} OK ::: $ips lo localhost } install_rsync_on_vagrant() { @@ -657,6 +675,8 @@ rsync_versions() { autoreconf -fiv # strlcat strlcpy perl -i -pe 's/void (strlc..)/void _$1/' util.c + # Force FNM_CASEFOLD to be set + export CFLAGS='-Wno-incompatible-pointer-types -D_GNU_SOURCE=1' configure_make_proto_install $1 } build-v3.0.0() { @@ -687,7 +707,7 @@ rsync_versions() { install_tangetools() { ( - git clone https://codeberg.org/tange/tangetools + git clone https://git.data.coop/tange/tangetools cd tangetools && make && sudo make install diff --git a/testsuite/tests-to-run/parallel-hwdep1.sh b/testsuite/tests-to-run/parallel-hwdep1.sh index 92ef69c9..abbb16e6 100755 --- a/testsuite/tests-to-run/parallel-hwdep1.sh +++ b/testsuite/tests-to-run/parallel-hwdep1.sh @@ -5,7 +5,7 @@ # Jobs that depend on the hardware # (e.g number of CPU threads, terminal type) -par__environment_too_big_ash() { +par__environment_too_big_dash() { myscript=$(cat <<'_EOF' echo 'bug #50815: env_parallel should warn if the environment is too big' len_var=60 @@ -17,7 +17,7 @@ par__environment_too_big_ash() { len_fun_quote=1 len_fun_quote_remote=1 - . `which env_parallel.ash`; + . `which env_parallel.dash`; repeat() { # Repeat input string n*1000 times @@ -72,7 +72,7 @@ par__environment_too_big_ash() { bigfunc() { true; } _EOF ) - stdout ssh ash@lo "$myscript" | perl -pe 's/(\d)\d\d\d\d/${1}XXXX/g' + stdout ssh dash@lo "$myscript" | perl -pe 's/(\d)\d\d\d\d/${1}XXXX/g' } par__environment_too_big_dash() { diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 0dfde398..f46e5a9f 100755 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -16,6 +16,20 @@ export -f stdsort # Test amount of parallelization # parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5} +par_parcat_args_stdin() { + echo 'bug #51690: parcat: read args from stdin' + # parcat reads files line by line + # so this does not work if TMPDIR contains \n + TMPDIR='/tmp/Y/ "$tmp1" + echo OK2 > "$tmp2" + (echo "$tmp1"; echo "$tmp2") | parcat | sort + rm "$tmp1" "$tmp2" +} + par_--rpl_group_bug() { echo 'Bug in --rpl group: $$1_' parallel --rpl '{a(.)b} s/$$1_c/o/g' echo {aDb} ::: GD_cD_cd @@ -806,20 +820,6 @@ par_X_eta_div_zero() { perl -ne '/../ and print' } -par_parcat_args_stdin() { - echo 'bug #51690: parcat: read args from stdin' - # parcat reads files line by line - # so this does not work if TMPDIR contains \n - TMPDIR='/tmp/Y/ "$tmp1" - echo OK2 > "$tmp2" - (echo "$tmp1"; echo "$tmp2") | parcat | sort - rm "$tmp1" "$tmp2" -} - par_parcat_rm() { echo 'bug #51691: parcat --rm remove fifo when opened' tmp1=$(mktemp) diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index c8fac023..62d373e3 100755 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -665,7 +665,7 @@ par_shellquote() { "$shell" -c perl\ -e\ \'print\ pack\(\"c\*\",1..255\)\'\ \|\ parallel\ -0\ --shellquote } export -f doit - parallel --tag -q -k doit {} ::: ash bash csh dash fish fizsh ksh2020 ksh93 lksh mksh posh rzsh sash sh static-sh tcsh yash zsh csh tcsh + parallel --tag -q -k doit {} ::: bash csh dash fish fizsh ksh2020 ksh93 lksh mksh posh rzsh sash sh static-sh tcsh yash zsh csh tcsh } par_tmp_full() { diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index fbf8951e..b93f4a82 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -433,7 +433,7 @@ par_testquote() { export -f testquote # "sash script" does not work # "sash -f script" does, but is currently not supported by GNU Parallel - parallel --tag -k testquote ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh posh rbash rc rzsh "sash -f" sh static-sh tcsh yash zsh + parallel --tag -k testquote ::: bash csh dash fdsh fish fizsh ksh ksh93 mksh posh rbash rc rzsh "sash -f" sh static-sh tcsh yash zsh # "fdsh" is currently not supported by GNU Parallel: # It gives ioctl(): Interrupted system call parallel --tag -k testquote ::: fdsh diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 0ff490bd..860a017c 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -290,7 +290,7 @@ par_exit_code() { echo '# Ideally the command should return the same' echo '# with or without parallel' # These give the same exit code prepended with 'true;' or not - OK="ash csh dash fish fizsh ksh2020 posh rc sash sh tcsh" + OK="csh dash fish fizsh ksh2020 posh rc sash sh tcsh" # These do not give the same exit code prepended with 'true;' or not BAD="bash ksh93 mksh static-sh yash zsh" doit $OK $BAD @@ -530,7 +530,7 @@ par_memfree() { par_test_detected_shell() { echo '### bug #42913: Dont use $SHELL but the shell currently running' - shells="ash bash csh dash fish fizsh ksh ksh93 mksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh" + shells="bash csh dash fish fizsh ksh ksh93 mksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh" test_unknown_shell() { shell="$1" tmp="/tmp/test_unknown_shell_$shell" diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index ce7e7488..b6585fe1 100755 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -84,7 +84,7 @@ par_warn_when_exporting_func() { PARALLEL_SHELL=$1 parallel --env myfunc -S lo myfunc ::: OK } export -f myrun - parallel -k --tag myrun ::: /bin/{sh,bash} /usr/bin/{ash,csh,dash,ksh,tcsh,zsh} + parallel -k --tag myrun ::: /bin/{sh,bash} /usr/bin/{csh,dash,ksh,tcsh,zsh} } par_exporting_in_zsh() { @@ -221,5 +221,5 @@ par_z_multiple_hosts_repeat_arg() { export -f $(compgen -A function | grep par_) compgen -A function | G par_ "$@" | LC_ALL=C sort | - parallel --timeout 250 -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | + parallel --timeout 250 -j75% --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | perl -pe 's:/usr/bin:/bin:g;' diff --git a/testsuite/tests-to-run/parallel-local-ssh5.sh b/testsuite/tests-to-run/parallel-local-ssh5.sh index b5290fa2..e4d5ffb5 100644 --- a/testsuite/tests-to-run/parallel-local-ssh5.sh +++ b/testsuite/tests-to-run/parallel-local-ssh5.sh @@ -46,7 +46,8 @@ par_--ssh_autossh() { stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh ls foo_autossh* rm foo_autossh* - ) | grep -Ev 'Warning: remote port forwarding failed for listen' + ) | grep -Ev 'Warning: remote port forwarding failed for listen' | + perl -pe 's/Receiver=[0-9.]+/Receiver=9.9.9/g' } par_fish_exit() { diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 2f21f1aa..a281e437 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -29,63 +29,6 @@ export -f retry ## par_*_man = tests from the man page # -par__man_ash() { - echo '### ash' - - myscript=$(cat <<'_EOF' - echo "### From man env_parallel" - - . `which env_parallel.ash`; - - alias myecho='echo aliases with \= \& \" \!'" \'" - myecho work - env_parallel myecho ::: work - env_parallel -S server myecho ::: work - env_parallel --env myecho myecho ::: work - env_parallel --env myecho -S server myecho ::: work - - alias multiline='echo multiline - echo aliases with \= \& \" \!'" \'" - multiline work - env_parallel multiline ::: work - env_parallel -S server multiline ::: work - env_parallel --env multiline multiline ::: work - env_parallel --env multiline -S server multiline ::: work - alias multiline="dummy" - - # Functions are not supported in ash - - myvar='variables with = & " !'" '" - echo "$myvar" work - env_parallel echo '"$myvar"' ::: work - env_parallel -S server echo '"$myvar"' ::: work - env_parallel --env myvar echo '"$myvar"' ::: work - env_parallel --env myvar -S server echo '"$myvar"' ::: work - - multivar='multiline - variables with = & " !'" '" - echo "$multivar" work - env_parallel echo '"$multivar"' ::: work - env_parallel -S server echo '"$multivar"' ::: work - env_parallel --env multivar echo '"$multivar"' ::: work - env_parallel --env multivar -S server echo '"$multivar"' ::: work - - # Arrays are not supported in ash - - # Exporting of functions is not supported - # env_parallel --argsep --- env_parallel -k echo ::: multi level --- env_parallel - - env_parallel ::: true false true false - echo exit value $? should be 2 - - env_parallel --no-such-option 2>&1 >/dev/null - # Sleep 1 to delay output to stderr to avoid race - echo exit value $? should be 255 `sleep 1` -_EOF - ) - ssh ash@lo "$myscript" -} - par__man_bash() { echo '### bash' @@ -624,55 +567,6 @@ _EOF ssh zsh@lo "$myscript" } - -par_--env_underscore_ash() { - echo '### ash' - myscript=$(cat <<'_EOF' - echo "### Testing of --env _" - - alias not_copied_alias="echo BAD" -# not_copied_func() { echo BAD; }; - not_copied_var=BAD -# not_copied_array=(BAD BAD BAD); - . `which env_parallel.ash`; - env_parallel --record-env; - alias myecho="echo \$myvar aliases"; -# myfunc() { myecho functions $*; }; - myvar="variables in"; -# myarray=(and arrays in); -# env_parallel myfunc ::: work; -# env_parallel -S server myfunc ::: work; - env_parallel --env myvar,myecho myecho ::: work; - env_parallel --env myvar,myecho -S server myecho ::: work; - env_parallel --env _ myecho ::: work; - env_parallel --env _ -S server myecho ::: work; - - env_parallel --env _ -S server not_copied_alias ::: error=OK; -# env_parallel --env _ -S server not_copied_func ::: error=OK; - env_parallel --env _ -S server echo \$not_copied_var ::: error=OK; -# env_parallel --env _ -S server echo \${not_copied_array[@]} ::: error=OK; - - echo myvar >> ~/.parallel/ignored_vars; - env_parallel --env _ myecho ::: work; - env_parallel --env _ -S server myecho ::: work; -# echo myarray >> ~/.parallel/ignored_vars; -# env_parallel --env _ myfunc ::: work; -# env_parallel --env _ -S server myfunc ::: work; - echo myecho >> ~/.parallel/ignored_vars; - env_parallel --env _ myecho ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; - env_parallel --env _ -S server myecho ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; -# echo myfunc >> ~/.parallel/ignored_vars; -# env_parallel --env _ myfunc ::: work; -# echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; -# env_parallel --env _ -S server myfunc ::: work; -# echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; -_EOF - ) - ssh ash@lo "$myscript" -} - par_--env_underscore_bash() { echo '### bash' myscript=$(cat <<'_EOF' @@ -1109,39 +1003,6 @@ _EOF # + remote, locally # + variables, variables with funky content, arrays, assoc array, functions, aliases -par_funky_ash() { - myscript=$(cat <<'_EOF' - . `which env_parallel.ash`; - - myvar="myvar works" - funky=`perl -e "print pack \"c*\", 2..255"` -# Arrays not supported -# myarray=("" array_val2 3 "" 5 " space 6 ") -# typeset -A assocarr -# assocarr[a]=assoc_val_a -# assocarr[b]=assoc_val_b - alias alias_echo="echo 3 arg"; - - func_echo() { - echo $*; - echo "$myvar" -# echo "${myarray[5]}" -# echo ${assocarr[a]} - echo Funky-"$funky"-funky - } - - env_parallel alias_echo ::: alias_works -# env_parallel func_echo ::: function_works - env_parallel -S ash@lo alias_echo ::: alias_works_over_ssh -# env_parallel -S ash@lo func_echo ::: function_works_over_ssh - echo - echo "$funky" | parallel --shellquote -_EOF - ) - # Order is often different. Dunno why. So sort - ssh ash@lo "$myscript" 2>&1 | LC_ALL=C sort -} - par_funky_bash() { myscript=$(cat <<'_EOF' . `which env_parallel.bash`; @@ -1452,31 +1313,6 @@ _EOF ssh zsh@lo "$myscript" 2>&1 | LC_ALL=C sort } -par_env_parallel_ash() { - myscript=$(cat <<'_EOF' - . `which env_parallel.ash`; - echo 'bug #50435: Remote fifo broke in 20150522' - # Due to $PARALLEL_TMP being transferred - OK=OK - echo data from stdin | env_parallel --pipe -S lo --fifo 'cat {} && echo $OK' - echo data from stdin | env_parallel --pipe -S lo --cat 'cat {} && echo $OK' - - echo 'bug #52534: Tail of multiline alias is ignored' - alias myalias='echo alias line 1 - echo alias line 2 - echo alias line 3 - ' - alias myalias2='echo alias2 line 1 - echo alias2 line 2 - ' - env_parallel myalias ::: myalias2 - env_parallel -S lo myalias ::: myalias2 -_EOF - ) - # Order is often different. Dunno why. So sort - ssh ash@lo "$myscript" 2>&1 | LC_ALL=C sort -} - par_env_parallel_bash() { myscript=$(cat <<'_EOF' echo 'bug #50435: Remote fifo broke in 20150522' @@ -1670,10 +1506,6 @@ _EOF ssh zsh@lo "$myscript" 2>&1 | LC_ALL=C sort } -par_environment_too_big_ash() { - echo moved to hwdep1.sh -} - par_environment_too_big_bash() { myscript=$(cat <<'_EOF' echo 'bug #50815: env_parallel should warn if the environment is too big' @@ -1979,69 +1811,6 @@ par_environment_too_big_zsh() { echo moved to hwdep1.sh } -par_parset_ash() { - myscript=$(cat <<'_EOF' - echo 'parset' - . `which env_parallel.ash` - -# Arrays not supported -# echo '### parset into array' -# parset arr1 echo ::: foo bar baz -# echo ${arr1[0]} ${arr1[1]} ${arr1[2]} - - echo '### parset into vars with comma' - parset comma3,comma2,comma1 echo ::: baz bar foo - echo $comma1 $comma2 $comma3 - - echo '### parset into vars with space' - parset 'space3 space2 space1' echo ::: baz bar foo - echo $space1 $space2 $space3 - - echo '### parset with newlines' - parset 'newline3 newline2 newline1' seq ::: 3 2 1 - echo "$newline1" - echo "$newline2" - echo "$newline3" - -# Arrays not supported -# echo '### parset into indexed array vars' -# parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo -# echo ${myarray[*]} -# echo ${myarray[4]} ${myarray[5]} ${myarray[6]} - - echo '### env_parset' - myfun() { - myecho myfun "$@"; - } - alias myecho='echo myecho "$myvar"' - myvar="myvar" -# Arrays not supported -# myarr=("myarr 0" "myarr 1" "myarr 2") - mynewline="`echo newline1;echo newline2;`" -# Arrays not supported -# env_parset arr1 myfun ::: foo bar baz -# echo "${arr1[0]} ${arr1[1]} ${arr1[2]}" - env_parset comma3,comma2,comma1 myecho ::: baz bar foo - echo "$comma1 $comma2 $comma3" - env_parset 'space3 space2 space1' myecho ::: baz bar foo - echo "$space1 $space2 $space3" - env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1 - echo "$newline1" - echo "$newline2" - echo "$newline3" -# Arrays not supported -# env_parset 'myarray[6],myarray[5],myarray[4]' myfun ::: baz bar foo -# echo "${myarray[*]}" -# echo "${myarray[4]} ${myarray[5]} ${myarray[6]}" - parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0 - echo Exit value 2 = $? - env_parset a,b,c,d 'echo {};exit {}' ::: 0 1 1 0 - echo Exit value 2 = $? -_EOF - ) - ssh ash@lo "$myscript" -} - par_parset_bash() { myscript=$(cat <<'_EOF' echo 'parset' @@ -2448,107 +2217,6 @@ _EOF ### env_parallel_session -par_env_parallel_--session_ash() { - myscript=$(cat <<'_EOF' - . `which env_parallel.ash` - echo '### Test env_parallel --session' - - level0var=l0var -# Arrays and functions not supported -# level0arr=(level0 array) -# level0func() { echo l0func; } - alias level0alias='echo l0alias' - - env_parallel --session - - level1var=l1var -# Arrays and functions not supported -# level1arr=(level1 array) -# level1func() { echo l1func; } - alias level1alias='echo l1alias' - - echo '### level0 should be hidden, level1 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK - - env_parallel --session - - level2var=l2var -# Arrays and functions not supported -# level2arr=(level2 array) -# level2func() { echo l2func; } - alias level2alias='echo l2alias' - - echo '### level0+1 should be hidden, level2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail - env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK - env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK - - env_parallel --endsession - - echo '### level0 should be hidden, level1+2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK - env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK - env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK - - env_parallel --endsession - - echo '### level0+1+2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: OK - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: OK - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK - env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK - env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK - - alias aliasbefore='echo before' -# Functions not supported -# varbefore='before' -# funcbefore() { echo 'before' "$@"; } -# Arrays not supported -# arraybefore=(array before) - env_parallel --session - # stuff defined - env_parallel aliasbefore ::: must_fail - env_parallel -S lo aliasbefore ::: must_fail -# Functions not supported -# env_parallel funcbefore ::: must_fail -# env_parallel -S lo funcbefore ::: must_fail - env_parallel echo '$varbefore' ::: no_before - env_parallel -S lo echo '$varbefore' ::: no_before -# Arrays not supported -# env_parallel echo '${arraybefore[*]}' ::: no_before -# env_parallel -S lo echo '${arraybefore[*]}' ::: no_before - alias aliasafter='echo after' - varafter='after' -# Functions not supported -# funcafter() { echo 'after' "$@"; } -# Arrays not supported -# arrayafter=(array after) - env_parallel aliasafter ::: aliasafter_OK - env_parallel -S lo aliasafter ::: aliasafter_OK -# Functions not supported -# env_parallel funcafter ::: funcafter_OK -# env_parallel -S lo funcafter ::: funcafter_OK - env_parallel echo '$varafter' ::: varafter_OK - env_parallel -S lo echo '$varafter' ::: varafter_OK -# Arrays not supported -# env_parallel echo '${arrayafter[*]}' ::: arrayafter_OK -# env_parallel -S lo echo '${arrayafter[*]}' ::: arrayafter_OK - unset PARALLEL_IGNORED_NAMES -_EOF - ) - ssh ash@lo "$myscript" -} - par_env_parallel_--session_bash() { myscript=$(cat <<'_EOF' echo '### Test env_parallel --session / --end-session' diff --git a/testsuite/tests-to-run/parallel-local-ssh9.sh b/testsuite/tests-to-run/parallel-local-ssh9.sh index 245f651a..1325c06c 100755 --- a/testsuite/tests-to-run/parallel-local-ssh9.sh +++ b/testsuite/tests-to-run/parallel-local-ssh9.sh @@ -4,30 +4,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -par_ash_embed() { - myscript=$(cat <<'_EOF' - echo '--embed' - parallel --embed | tac | perl -pe ' - /^parallel/ and not $seen++ and s{^}{ -echo \$b -parset a,b,c echo ::: ParsetOK ParsetOK ParsetOK -env_parallel echo ::: env_parallel_OK -env_parallel --env myvar echo {} --env \\\$myvar ::: env_parallel -myvar=OK -parallel echo ::: parallel_OK -PATH=/usr/sbin:/usr/bin:/sbin:/bin -# Do not look for parallel in /usr/local/bin -#. \`which env_parallel.ash\` -} - ' | tac > parallel-embed - chmod +x parallel-embed - ./parallel-embed - rm parallel-embed -_EOF - ) - ssh ash@lo "$myscript" -} - par_bash_embed() { myscript=$(cat <<'_EOF' echo '--embed' @@ -157,7 +133,7 @@ par_env_parallel_big_env() { echo '### bug #54128: command too long when exporting big env' . env_parallel.bash env_parallel --session - a=`rand | perl -pe 's/\0//g'| head -c 40000` + a=`rand | perl -pe 's/\0//g'| head -c 10000` env_parallel -Slo echo ::: OK 2>&1 a=`rand | perl -pe 's/\0//g'| head -c 50000` env_parallel -Slo echo THIS SHOULD ::: FAIL 2>/dev/null || echo OK diff --git a/testsuite/wanted-results/parallel-hwdep1 b/testsuite/wanted-results/parallel-hwdep1 index 79204554..dd596462 100644 --- a/testsuite/wanted-results/parallel-hwdep1 +++ b/testsuite/wanted-results/parallel-hwdep1 @@ -1,21 +1,3 @@ -par__environment_too_big_ash bug #5XXXX: env_parallel should warn if the environment is too big -par__environment_too_big_ash OK_bigvar -par__environment_too_big_ash OK_bigvar_remote -par__environment_too_big_ash OK_bigvar_quote -par__environment_too_big_ash OK_bigvar_quote_remote -par__environment_too_big_ash OK_bigfunc -par__environment_too_big_ash OK_bigfunc_remote -par__environment_too_big_ash OK_bigfunc_quote -par__environment_too_big_ash OK_bigfunc_quote_remote -par__environment_too_big_ash Rest should fail - functions not supported in ash -par__environment_too_big_ash parallel: Error: Command line too long (7XXXX >= 6XXXX) at input 0: fail_bigvar -par__environment_too_big_ash parallel: Error: Command line too long (7XXXX >= 6XXXX) at input 0: fail_bigvar_remote -par__environment_too_big_ash parallel: Error: Command line too long (8XXXX >= 6XXXX) at input 0: fail_bigvar_quote -par__environment_too_big_ash parallel: Error: Command line too long (8XXXX >= 6XXXX) at input 0: fail_bigvar_quote_remote -par__environment_too_big_ash fail_bigfunc-not-supported -par__environment_too_big_ash fail_bigfunc_remote-not-supported -par__environment_too_big_ash fail_bigfunc_quote-not-supported -par__environment_too_big_ash fail_bigfunc_quote_remote-not-supported par__environment_too_big_dash bug #5XXXX: env_parallel should warn if the environment is too big par__environment_too_big_dash OK_bigvar par__environment_too_big_dash OK_bigvar_remote diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 7f60ba36..3101efa7 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -1372,8 +1372,6 @@ par_seqreplace_long_line 9 1 1 101 par_seqreplace_long_line 90 1 1 201 par_seqreplace_long_line 1 parallel: Error: Command line too long (309 >= 210) at input 0: 100 par_shellquote ### Test --shellquote in all shells -par_shellquote ash ' -par_shellquote ash par_shellquote ash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' par_shellquote bash ' par_shellquote bash par_shellquote bash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' par_shellquote csh \\\\\\\\ "\ diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 32291dc8..707f65bc 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -960,8 +960,6 @@ par_test_m_X a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 par_test_m_X ### Test -q {.} par_test_m_X 'a' par_test_m_X 'a' -par_testquote ash "#&/ -par_testquote ash ()*=?' par_testquote bash "#&/ par_testquote bash ()*=?' par_testquote csh "#&/ diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index fdbfdec9..c9c03495 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -1643,16 +1643,6 @@ par_bin 9 2 par_exit_code bug #52207: Exit status 0 when child job is killed, even with "now,fail=1" par_exit_code # Ideally the command should return the same par_exit_code # with or without parallel -par_exit_code ash /tmp/mysleep 100 137 -par_exit_code ash parallel --halt-on-error now,fail=1 /tmp/mysleep ::: 100 137 -par_exit_code ash parallel --halt-on-error now,done=1 /tmp/mysleep ::: 100 137 -par_exit_code ash parallel --halt-on-error now,done=1 /bin/true ::: 100 0 -par_exit_code ash parallel --halt-on-error now,done=1 exit ::: 100 100 -par_exit_code ash true;/tmp/mysleep 100 137 -par_exit_code ash parallel --halt-on-error now,fail=1 'true;/tmp/mysleep' ::: 100 137 -par_exit_code ash parallel --halt-on-error now,done=1 'true;/tmp/mysleep' ::: 100 137 -par_exit_code ash parallel --halt-on-error now,done=1 'true;/bin/true' ::: 100 0 -par_exit_code ash parallel --halt-on-error now,done=1 'true;exit' ::: 100 100 par_exit_code csh /tmp/mysleep 100 137 par_exit_code csh parallel --halt-on-error now,fail=1 /tmp/mysleep ::: 100 137 par_exit_code csh parallel --halt-on-error now,done=1 /tmp/mysleep ::: 100 137 @@ -3069,7 +3059,6 @@ par_slow_total_jobs 1 par_slow_total_jobs 2 par_slow_total_jobs parallel: Warning: Reading X arguments took longer than XX seconds. par_test_detected_shell ### bug #42913: Dont use $SHELL but the shell currently running -par_test_detected_shell test_unknown_shell ash Global::shell /usr/bin/bash par_test_detected_shell test_unknown_shell bash Global::shell /usr/bin/bash par_test_detected_shell test_unknown_shell csh Global::shell /usr/bin/bash par_test_detected_shell test_unknown_shell dash Global::shell /usr/bin/bash @@ -3086,7 +3075,6 @@ par_test_detected_shell test_unknown_shell sh Global::shell /usr/bin/bash par_test_detected_shell test_unknown_shell tcsh Global::shell /usr/bin/bash par_test_detected_shell test_unknown_shell yash Global::shell /usr/bin/bash par_test_detected_shell test_unknown_shell zsh Global::shell /usr/bin/bash -par_test_detected_shell test_known_shell_c ash Global::shell /usr/bin/ash par_test_detected_shell test_known_shell_c bash Global::shell /usr/bin/bash par_test_detected_shell test_known_shell_c csh Global::shell /usr/bin/csh par_test_detected_shell test_known_shell_c dash Global::shell /usr/bin/dash @@ -3104,7 +3092,6 @@ par_test_detected_shell test_known_shell_c static-sh Global::shell /usr/bin/stat par_test_detected_shell test_known_shell_c tcsh Global::shell /usr/bin/tcsh par_test_detected_shell test_known_shell_c yash Global::shell /usr/bin/yash par_test_detected_shell test_known_shell_c zsh Global::shell /usr/bin/zsh -par_test_detected_shell test_known_shell_pipe ash Global::shell /usr/bin/ash par_test_detected_shell test_known_shell_pipe bash Global::shell /usr/bin/bash par_test_detected_shell test_known_shell_pipe csh Global::shell /usr/bin/csh par_test_detected_shell test_known_shell_pipe dash Global::shell /usr/bin/dash diff --git a/testsuite/wanted-results/parallel-local-race02 b/testsuite/wanted-results/parallel-local-race02 index 5a8afeb2..c5a92ae6 100644 --- a/testsuite/wanted-results/parallel-local-race02 +++ b/testsuite/wanted-results/parallel-local-race02 @@ -1,7 +1,10 @@ WHY DOES THIS FAIL? ### --env _ +OK from fubar test In csh this may fail with ignored_var: Undefined variable. ### --env _ with explicit mentioning of normally ignored var $ignored_var +OK from fubar should be copied test +OK from fubar should be copied test bug #46120: Suspend should suspend (at least local) children it should burn 1.9 CPU seconds, but no more than that The 5 second sleep will make it be killed by timeout when it fgs diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 3bfb0f0a..8c16eeb5 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -321,7 +321,6 @@ par_warn_when_exporting_func /bin/sh Run function in /bin/sh par_warn_when_exporting_func /bin/sh parallel: Warning: Shell functions may not be supported in /bin/sh. par_warn_when_exporting_func /bin/bash Run function in /bin/bash par_warn_when_exporting_func /bin/bash Function run: OK -par_warn_when_exporting_func /bin/ash Run function in /bin/ash par_warn_when_exporting_func /bin/csh Run function in /bin/csh par_warn_when_exporting_func /bin/csh parallel: Warning: Shell functions may not be supported in /bin/csh. par_warn_when_exporting_func /bin/csh CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset myfunc diff --git a/testsuite/wanted-results/parallel-local-ssh5 b/testsuite/wanted-results/parallel-local-ssh5 index 82145cd7..337601f2 100644 --- a/testsuite/wanted-results/parallel-local-ssh5 +++ b/testsuite/wanted-results/parallel-local-ssh5 @@ -37,7 +37,7 @@ par_--ssh_autossh AUTOSSH_DEBUG - turn logging to maximum verbosity an par_--ssh_autossh stderr par_--ssh_autossh par_--ssh_autossh rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] -par_--ssh_autossh rsync error: error in rsync protocol data stream (code 12) at io.c(231) [Receiver=3.2.7] +par_--ssh_autossh rsync error: error in rsync protocol data stream (code 12) at io.c(231) [Receiver=9.9.9] par_--ssh_autossh foo_autossh par_--ssh_lsh ### --ssh lsh par_--ssh_lsh OK diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 23b7439c..65eceb4b 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -1,18 +1,4 @@ ### test --env _, env_parallel for different shells -par_--env_underscore_ash ### ash -par_--env_underscore_ash ### Testing of --env _ -par_--env_underscore_ash variables in aliases work -par_--env_underscore_ash variables in aliases work -par_--env_underscore_ash variables in aliases work -par_--env_underscore_ash variables in aliases work -par_--env_underscore_ash script: 4: not_copied_alias: not found -par_--env_underscore_ash error=OK -par_--env_underscore_ash aliases work -par_--env_underscore_ash aliases work -par_--env_underscore_ash /bin/ash: 1: myecho: not found -par_--env_underscore_ash OK if no myecho ^^^^^^^^^^^^^^^^^ -par_--env_underscore_ash script: 2: myecho: not found -par_--env_underscore_ash OK if no myecho ^^^^^^^^^^^^^^^^^ par_--env_underscore_bash ### bash par_--env_underscore_bash ### Testing of --env _ par_--env_underscore_bash variables in aliases in and arrays in functions work @@ -87,7 +73,7 @@ par_--env_underscore_ksh aliases in functions work par_--env_underscore_ksh aliases in functions work par_--env_underscore_ksh /bin/ksh: myecho: not found par_--env_underscore_ksh OK if no myecho ^^^^^^^^^^^^^^^^^ -par_--env_underscore_ksh script[9]: myecho: not found +par_--env_underscore_ksh script: myecho: not found par_--env_underscore_ksh OK if no myecho ^^^^^^^^^^^^^^^^^ par_--env_underscore_ksh /bin/ksh: myfunc: not found par_--env_underscore_ksh OK if no myfunc ^^^^^^^^^^^^^^^^^ @@ -101,8 +87,8 @@ par_--env_underscore_mksh variables in aliases in and arrays in functions work par_--env_underscore_mksh variables in aliases in and arrays in functions work par_--env_underscore_mksh variables in aliases in and arrays in functions work par_--env_underscore_mksh variables in aliases in and arrays in functions work -par_--env_underscore_mksh W: script[9]: not_copied_alias: inaccessible or not found -par_--env_underscore_mksh W: script[9]: not_copied_func: inaccessible or not found +par_--env_underscore_mksh EW: script[9]: not_copied_alias: inaccessible or not found +par_--env_underscore_mksh EW: script[9]: not_copied_func: inaccessible or not found par_--env_underscore_mksh error=OK par_--env_underscore_mksh error=OK par_--env_underscore_mksh aliases in and arrays in functions work @@ -113,9 +99,9 @@ par_--env_underscore_mksh aliases in functions work par_--env_underscore_mksh The myecho alias is replaced in the function causing this not to fail par_--env_underscore_mksh aliases in functions work par_--env_underscore_mksh The myecho alias is replaced in the function causing this not to fail -par_--env_underscore_mksh W: /bin/mksh: myfunc: inaccessible or not found +par_--env_underscore_mksh EW: /bin/mksh: myfunc: inaccessible or not found par_--env_underscore_mksh OK if no myfunc ^^^^^^^^^^^^^^^^^ -par_--env_underscore_mksh W: script[9]: myfunc: inaccessible or not found +par_--env_underscore_mksh EW: script[9]: myfunc: inaccessible or not found par_--env_underscore_mksh OK if no myfunc ^^^^^^^^^^^^^^^^^ par_--env_underscore_sh ### sh par_--env_underscore_sh ### Testing of --env _ @@ -173,41 +159,6 @@ par_--env_underscore_zsh zsh:1: command not found: myfunc par_--env_underscore_zsh OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc par_--env_underscore_zsh script:2: command not found: myfunc par_--env_underscore_zsh OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc -par__man_ash ### ash -par__man_ash ### From man env_parallel -par__man_ash aliases with = & " ! ' work -par__man_ash aliases with = & " ! ' work -par__man_ash aliases with = & " ! ' work -par__man_ash aliases with = & " ! ' work -par__man_ash aliases with = & " ! ' work -par__man_ash multiline -par__man_ash aliases with = & " ! ' work -par__man_ash multiline -par__man_ash aliases with = & " ! ' work -par__man_ash multiline -par__man_ash aliases with = & " ! ' work -par__man_ash multiline -par__man_ash aliases with = & " ! ' work -par__man_ash multiline -par__man_ash aliases with = & " ! ' work -par__man_ash variables with = & " ! ' work -par__man_ash variables with = & " ! ' work -par__man_ash variables with = & " ! ' work -par__man_ash variables with = & " ! ' work -par__man_ash variables with = & " ! ' work -par__man_ash multiline -par__man_ash variables with = & " ! ' work -par__man_ash multiline -par__man_ash variables with = & " ! ' work -par__man_ash multiline -par__man_ash variables with = & " ! ' work -par__man_ash multiline -par__man_ash variables with = & " ! ' work -par__man_ash multiline -par__man_ash variables with = & " ! ' work -par__man_ash exit value 2 should be 2 -par__man_ash Unknown option: no-such-option -par__man_ash exit value 255 should be 255 par__man_bash ### bash par__man_bash ### From man env_parallel par__man_bash aliases with = & " ! ' work @@ -705,107 +656,6 @@ par__man_zsh env_parallel par__man_zsh exit value 2 should be 2 par__man_zsh Unknown option: no-such-option par__man_zsh exit value 255 should be 255 -par_env_parallel_--session_ash ### Test env_parallel --session -par_env_parallel_--session_ash ### level0 should be hidden, level1 should be transferred -par_env_parallel_--session_ash -par_env_parallel_--session_ash script: 99: level0func: not found -par_env_parallel_--session_ash script: 99: level0alias: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash -par_env_parallel_--session_ash /bin/ash: 999: level0func: not found -par_env_parallel_--session_ash /bin/ash: 999: level0alias: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash l1var -par_env_parallel_--session_ash l1alias -par_env_parallel_--session_ash script: 99: level1func: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash l1var -par_env_parallel_--session_ash l1alias -par_env_parallel_--session_ash /bin/ash: 999: level1func: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash ### level0+1 should be hidden, level2 should be transferred -par_env_parallel_--session_ash -par_env_parallel_--session_ash script: 4: level0func: not found -par_env_parallel_--session_ash script: 4: level0alias: not found -par_env_parallel_--session_ash script: 4: Bad substitution -par_env_parallel_--session_ash -par_env_parallel_--session_ash /bin/ash: 3: level0func: not found -par_env_parallel_--session_ash /bin/ash: 3: level0alias: not found -par_env_parallel_--session_ash /bin/ash: 3: Bad substitution -par_env_parallel_--session_ash -par_env_parallel_--session_ash script: 4: level1func: not found -par_env_parallel_--session_ash script: 4: level1alias: not found -par_env_parallel_--session_ash script: 4: Bad substitution -par_env_parallel_--session_ash -par_env_parallel_--session_ash /bin/ash: 3: level1func: not found -par_env_parallel_--session_ash /bin/ash: 3: level1alias: not found -par_env_parallel_--session_ash /bin/ash: 3: Bad substitution -par_env_parallel_--session_ash l2var -par_env_parallel_--session_ash l2alias -par_env_parallel_--session_ash script: 4: level2func: not found -par_env_parallel_--session_ash script: 4: Bad substitution -par_env_parallel_--session_ash l2var -par_env_parallel_--session_ash l2alias -par_env_parallel_--session_ash /bin/ash: 3: level2func: not found -par_env_parallel_--session_ash /bin/ash: 3: Bad substitution -par_env_parallel_--session_ash ### level0 should be hidden, level1+2 should be transferred -par_env_parallel_--session_ash -par_env_parallel_--session_ash script: 99: level0func: not found -par_env_parallel_--session_ash script: 99: level0alias: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash -par_env_parallel_--session_ash /bin/ash: 999: level0func: not found -par_env_parallel_--session_ash /bin/ash: 999: level0alias: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash l1var -par_env_parallel_--session_ash l1alias -par_env_parallel_--session_ash script: 99: level1func: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash l1var -par_env_parallel_--session_ash l1alias -par_env_parallel_--session_ash /bin/ash: 999: level1func: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash l2var -par_env_parallel_--session_ash l2alias -par_env_parallel_--session_ash script: 99: level2func: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash l2var -par_env_parallel_--session_ash l2alias -par_env_parallel_--session_ash /bin/ash: 999: level2func: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash ### level0+1+2 should be transferred -par_env_parallel_--session_ash l0var -par_env_parallel_--session_ash l0alias -par_env_parallel_--session_ash script: 99: level0func: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash l0var -par_env_parallel_--session_ash l0alias -par_env_parallel_--session_ash /bin/ash: 999: level0func: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash l1var -par_env_parallel_--session_ash l1alias -par_env_parallel_--session_ash script: 99: level1func: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash l1var -par_env_parallel_--session_ash l1alias -par_env_parallel_--session_ash /bin/ash: 999: level1func: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash l2var -par_env_parallel_--session_ash l2alias -par_env_parallel_--session_ash script: 99: level2func: not found -par_env_parallel_--session_ash script: 99: Bad substitution -par_env_parallel_--session_ash l2var -par_env_parallel_--session_ash l2alias -par_env_parallel_--session_ash /bin/ash: 999: level2func: not found -par_env_parallel_--session_ash /bin/ash: 999: Bad substitution -par_env_parallel_--session_ash /bin/ash: 1: aliasbefore: not found -par_env_parallel_--session_ash script: 2: aliasbefore: not found -par_env_parallel_--session_ash no_before -par_env_parallel_--session_ash no_before -par_env_parallel_--session_ash after aliasafter_OK -par_env_parallel_--session_ash after aliasafter_OK -par_env_parallel_--session_ash after varafter_OK -par_env_parallel_--session_ash after varafter_OK par_env_parallel_--session_bash ### Test env_parallel --session / --end-session par_env_parallel_--session_bash ### level0 should be hidden, level1 should be transferred par_env_parallel_--session_bash @@ -1130,12 +980,12 @@ par_env_parallel_--session_mksh ### Test env_parallel --session par_env_parallel_--session_mksh ### level0 should be hidden, level1 should be transferred par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: script[9]: level0func: inaccessible or not found -par_env_parallel_--session_mksh W: script[9]: level0alias: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level0func: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level0alias: inaccessible or not found par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: /bin/mksh: level0func: inaccessible or not found -par_env_parallel_--session_mksh W: /bin/mksh: level0alias: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level0func: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level0alias: inaccessible or not found par_env_parallel_--session_mksh l1var par_env_parallel_--session_mksh l1func par_env_parallel_--session_mksh l1alias @@ -1147,20 +997,20 @@ par_env_parallel_--session_mksh level1 array OK par_env_parallel_--session_mksh ### level0+1 should be hidden, level2 should be transferred par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: script[9]: level0func: inaccessible or not found -par_env_parallel_--session_mksh W: script[9]: level0alias: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level0func: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level0alias: inaccessible or not found par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: /bin/mksh: level0func: inaccessible or not found -par_env_parallel_--session_mksh W: /bin/mksh: level0alias: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level0func: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level0alias: inaccessible or not found par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: script[9]: level1func: inaccessible or not found -par_env_parallel_--session_mksh W: script[9]: level1alias: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level1func: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level1alias: inaccessible or not found par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: /bin/mksh: level1func: inaccessible or not found -par_env_parallel_--session_mksh W: /bin/mksh: level1alias: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level1func: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level1alias: inaccessible or not found par_env_parallel_--session_mksh l2var par_env_parallel_--session_mksh l2func par_env_parallel_--session_mksh l2alias @@ -1172,12 +1022,12 @@ par_env_parallel_--session_mksh level2 array OK par_env_parallel_--session_mksh ### level0 should be hidden, level1+2 should be transferred par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: script[9]: level0func: inaccessible or not found -par_env_parallel_--session_mksh W: script[9]: level0alias: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level0func: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: level0alias: inaccessible or not found par_env_parallel_--session_mksh par_env_parallel_--session_mksh fail -par_env_parallel_--session_mksh W: /bin/mksh: level0func: inaccessible or not found -par_env_parallel_--session_mksh W: /bin/mksh: level0alias: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level0func: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: level0alias: inaccessible or not found par_env_parallel_--session_mksh l1var par_env_parallel_--session_mksh l1func par_env_parallel_--session_mksh l1alias @@ -1219,10 +1069,10 @@ par_env_parallel_--session_mksh l2var par_env_parallel_--session_mksh l2func par_env_parallel_--session_mksh l2alias par_env_parallel_--session_mksh level2 array OK -par_env_parallel_--session_mksh W: /bin/mksh: aliasbefore: inaccessible or not found -par_env_parallel_--session_mksh W: script[9]: aliasbefore: inaccessible or not found -par_env_parallel_--session_mksh W: /bin/mksh: funcbefore: inaccessible or not found -par_env_parallel_--session_mksh W: script[9]: funcbefore: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: aliasbefore: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: aliasbefore: inaccessible or not found +par_env_parallel_--session_mksh EW: /bin/mksh: funcbefore: inaccessible or not found +par_env_parallel_--session_mksh EW: script[9]: funcbefore: inaccessible or not found par_env_parallel_--session_mksh no_before par_env_parallel_--session_mksh no_before par_env_parallel_--session_mksh no_before @@ -1450,22 +1300,6 @@ par_env_parallel_--session_zsh after varafter_OK par_env_parallel_--session_zsh after varafter_OK par_env_parallel_--session_zsh array after arrayafter_OK par_env_parallel_--session_zsh array after arrayafter_OK -par_env_parallel_ash OK -par_env_parallel_ash OK -par_env_parallel_ash alias line 1 -par_env_parallel_ash alias line 1 -par_env_parallel_ash alias line 2 -par_env_parallel_ash alias line 2 -par_env_parallel_ash alias line 3 -par_env_parallel_ash alias line 3 -par_env_parallel_ash alias2 line 1 -par_env_parallel_ash alias2 line 1 -par_env_parallel_ash alias2 line 2 -par_env_parallel_ash alias2 line 2 -par_env_parallel_ash bug #50435: Remote fifo broke in 20150522 -par_env_parallel_ash bug #52534: Tail of multiline alias is ignored -par_env_parallel_ash data from stdin -par_env_parallel_ash data from stdin par_env_parallel_bash OK par_env_parallel_bash OK par_env_parallel_bash alias line 1 @@ -1606,7 +1440,6 @@ par_env_parallel_zsh bug #50435: Remote fifo broke in 20150522 par_env_parallel_zsh bug #52534: Tail of multiline alias is ignored par_env_parallel_zsh data from stdin par_env_parallel_zsh data from stdin -par_environment_too_big_ash moved to hwdep1.sh par_environment_too_big_bash bug #50815: env_parallel should warn if the environment is too big par_environment_too_big_bash OK_bigvar par_environment_too_big_bash OK_bigvar_remote @@ -1683,14 +1516,7 @@ par_environment_too_big_sh fail_bigfunc_quote-not-supported par_environment_too_big_sh fail_bigfunc_quote_remote-not-supported par_environment_too_big_tcsh Not implemented par_environment_too_big_zsh moved to hwdep1.sh -par_funky_ash -par_funky_ash ' ' -par_funky_ash ' par_funky_ash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_funky_ash 3 arg alias_works -par_funky_ash 3 arg alias_works_over_ssh par_funky_bash -par_funky_bash  -par_funky_bash  par_funky_bash -funkymultiline par_funky_bash -funkymultiline par_funky_bash space 6 @@ -1702,8 +1528,8 @@ par_funky_bash 3 arg alias_works par_funky_bash 3 arg alias_works_over_ssh par_funky_bash Funkyline- par_funky_bash  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyline par_funky_bash Funkyline- par_funky_bash  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyline -par_funky_bash Funkymultiline- -par_funky_bash Funkymultiline- +par_funky_bash Funkymultiline- +par_funky_bash Funkymultiline- par_funky_bash assoc_val_a par_funky_bash assoc_val_a par_funky_bash function_works @@ -1797,35 +1623,6 @@ par_funky_zsh function_works par_funky_zsh function_works_over_ssh par_funky_zsh myvar works par_funky_zsh myvar works -par_parset_ash parset -par_parset_ash ### parset into vars with comma -par_parset_ash foo bar baz -par_parset_ash ### parset into vars with space -par_parset_ash foo bar baz -par_parset_ash ### parset with newlines -par_parset_ash 1 -par_parset_ash 1 -par_parset_ash 2 -par_parset_ash 1 -par_parset_ash 2 -par_parset_ash 3 -par_parset_ash ### env_parset -par_parset_ash myecho myvar foo myecho myvar bar myecho myvar baz -par_parset_ash myecho myvar foo myecho myvar bar myecho myvar baz -par_parset_ash newline1 -par_parset_ash newline2 -par_parset_ash 1 -par_parset_ash newline1 -par_parset_ash newline2 -par_parset_ash 1 -par_parset_ash 2 -par_parset_ash newline1 -par_parset_ash newline2 -par_parset_ash 1 -par_parset_ash 2 -par_parset_ash 3 -par_parset_ash Exit value 2 = 2 -par_parset_ash Exit value 2 = 2 par_parset_bash parset par_parset_bash ### parset into array par_parset_bash foo bar baz diff --git a/testsuite/wanted-results/parallel-local-ssh9 b/testsuite/wanted-results/parallel-local-ssh9 index a8404362..dbb9bd97 100644 --- a/testsuite/wanted-results/parallel-local-ssh9 +++ b/testsuite/wanted-results/parallel-local-ssh9 @@ -134,24 +134,6 @@ par__sshlogin_range parallel: Warning: Could not figure out number of cpus on ip par__sshlogin_range parallel: Warning: Could not figure out number of cpus on ip251.0.9.109 (). Using 1. par__sshlogin_range parallel: Warning: Could not figure out number of cpus on ip251.0.9.110 (). Using 1. par__sshlogin_range parallel: Warning: Could not figure out number of cpus on ip251.0.9.111 (). Using 1. -par_ash_embed --embed -par_ash_embed Redirect the output to a file and add your changes at the end: -par_ash_embed /usr/local/bin/parallel --embed > new_script -par_ash_embed Put -par_ash_embed your -par_ash_embed code -par_ash_embed here -par_ash_embed parallel_OK -par_ash_embed env_parallel --env OK -par_ash_embed env_parallel_OK -par_ash_embed ParsetOK -par_ash_embed Put -par_ash_embed your -par_ash_embed code -par_ash_embed here -par_ash_embed Put your code here -par_ash_embed You can also activate GNU Parallel for interactive use by: -par_ash_embed . ./parallel-embed par_bash_embed --embed par_bash_embed Redirect the output to a file and add your changes at the end: par_bash_embed /usr/local/bin/parallel --embed > new_script diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index bd886e5d..bc378cdd 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -720,7 +720,6 @@ BASH_FUNC_replace_tmpdir%% BASH_FUNC_run_once%% BASH_FUNC_run_test%% _ -mysqlrootpass qTMPDIR qqTMPDIR testsuitedir @@ -1062,19 +1061,16 @@ cat: num_%header: No such file or directory /usr/bin/bash: line 2: foo: command not found #!/usr/bin/perl print "@ARGV\n" -Warning: unknown mime-type for "@ARGV\n" -- using "application/octet-stream" Error: no such file "@ARGV\n" parallel perl_echo ::: foo bar /usr/bin/bash: line 1: perl_echo: command not found #!/usr/bin/parallel --shebang-wrap /usr/bin/perl print "@ARGV\n" -Warning: unknown mime-type for "@ARGV\n" -- using "application/octet-stream" Error: no such file "@ARGV\n" perl_echo foo bar /usr/bin/bash: line 1: perl_echo: command not found #!/usr/bin/parallel --shebang-wrap /usr/bin/perl print "Arguments @ARGV\n"; -Warning: unknown mime-type for "Arguments @ARGV\n" -- using "application/octet-stream" Error: no such file "Arguments @ARGV\n" #!/usr/bin/parallel --shebang-wrap /usr/bin/python print 'Arguments', str(sys.argv) @@ -1095,7 +1091,6 @@ Arguments #!/usr/bin/parallel --shebang-wrap /usr/bin/ruby print "Arguments " puts ARGV -Warning: unknown mime-type for "Arguments " -- using "application/octet-stream" Error: no such file "Arguments " /usr/bin/bash: line 4: puts: command not found #!/usr/bin/parallel --shebang-wrap /usr/bin/octave diff --git a/testsuite/wanted-results/sem01 b/testsuite/wanted-results/sem01 index 2c59d396..41d9af72 100644 --- a/testsuite/wanted-results/sem01 +++ b/testsuite/wanted-results/sem01 @@ -100,10 +100,12 @@ par_exit 30 environment: line 1: PID Power failure bash -c 'kill -'$1' par_exit 31 Bash exit value 159 par_exit 31 Sem exit value 159 par_exit 31 environment: line 1: PID Bad system call (core dumped) bash -c 'kill -'$1' $$' -par_exit 32 Bash exit value 0 -par_exit 32 Sem exit value 0 -par_exit 33 Bash exit value 0 -par_exit 33 Sem exit value 0 +par_exit 32 Bash exit value 160 +par_exit 32 Sem exit value 160 +par_exit 32 environment: line 1: PID Unknown signal 32 bash -c 'kill -'$1' $$' +par_exit 33 Bash exit value 161 +par_exit 33 Sem exit value 161 +par_exit 33 environment: line 1: PID Unknown signal 33 bash -c 'kill -'$1' $$' par_exit 34 Bash exit value 162 par_exit 34 Sem exit value 162 par_exit 34 environment: line 1: PID Real-time signal 0 bash -c 'kill -'$1' $$'