diff --git a/NEWS b/NEWS index 79c20dfa..6bff90cc 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,33 @@ +20210122 + +* --memsuspend suspends jobs when there is little memory free. This + way you can run jobs in parallel that added up potentially will take + up more RAM than is available. + +* $PARALLEL_ARGHOSTGROUPS and the replacement string {agrp} will give + the hostgroup given on the argument when using --hostgroup. + +* Handy time functions for {= =}: yyyy_mm_dd_hh_mm_ss() + yyyy_mm_dd_hh_mm() yyyy_mm_dd() yyyymmddhhmmss() yyyymmddhhmm() + yyyymmdd() hash($str) + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* Software Engineering For Scientists + https://canvas.stanford.edu/courses/133091 + +* Optional Individual Submission 4 Job Handling - GNU Parallel + https://www.youtube.com/watch?v=-tX0bFAYAxM + +* 关于gnu parallel：GnuParallel：在集群上并行化脚本，脚本将文件写入主 + 节点https://www.codenong.com/25172209/ + +* すぐ忘れてしまう、仕事で使う技 + https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c + + 20201222 * --pipe engine changed making --pipe alpha quality. diff --git a/README b/README index faca628e..45186b0c 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20201222.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20201222.tar.bz2.sig - gpg parallel-20201222.tar.bz2.sig - bzip2 -dc parallel-20201222.tar.bz2 | tar xvf - - cd parallel-20201222 + wget https://ftpmirror.gnu.org/parallel/parallel-20210122.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20210122.tar.bz2.sig + gpg parallel-20210122.tar.bz2.sig + bzip2 -dc parallel-20210122.tar.bz2 | tar xvf - + cd parallel-20210122 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20201222.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20201222.tar.bz2.sig - gpg parallel-20201222.tar.bz2.sig - bzip2 -dc parallel-20201222.tar.bz2 | tar xvf - - cd parallel-20201222 + wget https://ftpmirror.gnu.org/parallel/parallel-20210122.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20210122.tar.bz2.sig + gpg parallel-20210122.tar.bz2.sig + bzip2 -dc parallel-20210122.tar.bz2 | tar xvf - + cd parallel-20210122 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2020, December 22). GNU Parallel 20201222 ('Vaccine'). - Zenodo. https://doi.org/10.5281/zenodo.4381888 + Tange, O. (2021, January 22). GNU Parallel 20210122 ('Capitol Riots'). + Zenodo. https://doi.org/10.5281/zenodo.4454976 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ole Tange, http://ole.tange.dk and Free diff --git a/configure b/configure index f1a4e9fc..9fa7713f 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20201222. +# Generated by GNU Autoconf 2.69 for parallel 20210122. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20201222' -PACKAGE_STRING='parallel 20201222' +PACKAGE_VERSION='20210122' +PACKAGE_STRING='parallel 20210122' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20201222 to adapt to many kinds of systems. +\`configure' configures parallel 20210122 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20201222:";; + short | recursive ) echo "Configuration of parallel 20210122:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20201222 +parallel configure 20210122 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20201222, which was +It was created by parallel $as_me 20210122, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20201222' + VERSION='20210122' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20201222, which was +This file was extended by parallel $as_me 20210122, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20201222 +parallel config.status 20210122 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 81a5e876..15bd3406 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20201222], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20210122], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 95902767..77df6c77 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,5 +1,11 @@ Quote of the month: + GNU Parallel and ripgrep would be your friend here. Ripgrep is fast, really fast. + -- CENAPT @cenaptech@twitter + + GNU Parallel is a priceless timesaver + -- June "Wakalix" @TWakalix@twitter + GNU Parallel is great -- Newton's Flaming Laser Sword @swordgoesfwoosh@twitter @@ -88,6 +94,10 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + I think many people would be surprised to learn that GNU parallel is + "just" a single Perl script. + -- Peter Menzel @ptr_menzel@twitter + GNU Parallel is my single favourite tool for batch processing data from the command line. -- Jeff Wintersinger @jwintersinger diff --git a/doc/release_new_version b/doc/release_new_version index ba0e3b39..638a8559 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -16,6 +16,8 @@ With the same things that goes in the announce mail == Testsuite == +Start termux on Android +Start: wssh mac ping fi.dk cd testsuite; make mem; make == Update version == @@ -203,28 +205,31 @@ Please help spreading GNU Parallel by making a testimonial video like Juan Sierr It does not have to be as detailed as Juan's. It is perfectly fine if you just say your name, and what field you are using GNU Parallel for. Quote of the month: -<<>> + + I think many people would be surprised to learn that GNU parallel is "just" a single Perl script. + -- Peter Menzel @ptr_menzel@twitter + New in this release: +* --memsuspend suspends jobs when there is little memory free. This way you can run jobs in parallel that added up potentially will take up more RAM than is available. + * $PARALLEL_ARGHOSTGROUPS and the replacement string {agrp} will give the hostgroup given on the argument when using --hostgroup. * Handy time functions for {= =}: yyyy_mm_dd_hh_mm_ss() yyyy_mm_dd_hh_mm() yyyy_mm_dd() yyyymmddhhmmss() yyyymmddhhmm() yyyymmdd() hash($str) -<<>> - - * Bug fixes and man page updates. News about GNU Parallel: -https://www.codenong.com/25172209/ +* Software Engineering For Scientists https://canvas.stanford.edu/courses/133091 -https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c +* Optional Individual Submission 4 Job Handling - GNU Parallel https://www.youtube.com/watch?v=-tX0bFAYAxM -https://canvas.stanford.edu/courses/133091 +* 关于gnu parallel：GnuParallel：在集群上并行化脚本，脚本将文件写入主节点https://www.codenong.com/25172209/ + +* すぐ忘れてしまう、仕事で使う技 https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c -https://www.youtube.com/watch?v=-tX0bFAYAxM <<>> diff --git a/packager/obs/home:tange/parallel/.osc/_files b/packager/obs/home:tange/parallel/.osc/_files index 72c1aacc..ac8a0f62 100644 --- a/packager/obs/home:tange/parallel/.osc/_files +++ b/packager/obs/home:tange/parallel/.osc/_files @@ -1,7 +1,7 @@ - - - - - - + + + + + + diff --git a/packager/obs/home:tange/parallel/.osc/parallel.spec b/packager/obs/home:tange/parallel/.osc/parallel.spec index 09974147..d562fa4b 100644 --- a/packager/obs/home:tange/parallel/.osc/parallel.spec +++ b/packager/obs/home:tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20201222 +Version: 20210122 Release: 1.3 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 54c1fdfb..9a9ef390 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -382,7 +382,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 4f9d374c..fa60cc93 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -384,7 +384,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 97099924..745bb735 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -382,7 +382,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 34a7999f..de4afdc2 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -365,7 +365,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 25a1d338..711bfb71 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -368,7 +368,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 81ac8c0d..03267fca 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -382,7 +382,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 9f4b6bdc..ef9483a1 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -359,7 +359,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20201223 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210122 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index c3c37e4a..9b211e11 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20201223; +$Global::version = 20210122; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 1bcf509f..28fc07cc 100755 --- a/src/parallel +++ b/src/parallel @@ -2151,7 +2151,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20210113; + $Global::version = 20210122; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -4933,8 +4933,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2020, December 22). GNU Parallel 20201222 ('Vaccine').", - " Zenodo. https://doi.org/10.5281/zenodo.4381888", + " Tange, O. (2021, January 22). GNU Parallel 20210122 ('Capitol Riots').", + " Zenodo. https://doi.org/10.5281/zenodo.4454976", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice @@ -4964,8 +4964,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2020, December 22). GNU Parallel 20201222 ('Vaccine').", - " Zenodo. https://doi.org/10.5281/zenodo.4381888", + " Tange, O. (2021, January 22). GNU Parallel 20210122 ('Capitol Riots').", + " Zenodo. https://doi.org/10.5281/zenodo.4454976", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -5078,20 +5078,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2020_4381888,", + "\@software{tange_2021_4454976,", " author = {Tange, Ole},", - " title = {GNU Parallel 20201222 ('Vaccine')},", - " month = Dec,", + " title = {GNU Parallel 20210122 ('Capitol Riots')},", + " month = Jan,", " year = 2020,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.4381888},", - " url = {https://doi.org/10.5281/zenodo.4381888}", + " doi = {10.5281/zenodo.4454976},", + " url = {https://doi.org/10.5281/zenodo.4454976}", "}", "", - "(Feel free to use \\nocite{tange_2020_4381888})", + "(Feel free to use \\nocite{tange_2021_4454976})", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -9280,12 +9280,12 @@ sub sshlogin_wrap($) { my @bashfunc = map { my $v=$_; s/BASH_FUNC_(.*)(\(\)|%%)/$1/; - "$_$ENV{$v};export -f $_ >/dev/null;" } @bash_functions; + "$_$ENV{$v};\nexport -f $_ 2> /dev/null;\n" } @bash_functions; # eval $bashfuncset will set $bashfunc my $bashfuncset; if(@bashfunc) { # Functions are not supported for all shells - if($Global::shell !~ m:(bash|rbash|zsh|rzsh|dash|ksh):) { + if($Global::shell !~ m:(^|/)(ash|bash|rbash|zsh|rzsh|dash|ksh):) { ::warning("Shell functions may not be supported in $Global::shell."); } $bashfuncset = diff --git a/src/parallel.pod b/src/parallel.pod index eaed3ded..89e3e6e6 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -467,7 +467,7 @@ contains B. B<-a foo> B<-a bar> will result in the combinations (1,a) (1,b) (1,c) (2,a) (2,b) (2,c). This is useful for replacing nested for-loops. -See also B<--link> and B<{>IB<}>. +See also: B<--link> and B<{>IB<}>. =item B<--arg-file-sep> I @@ -549,7 +549,7 @@ I is small (<10), slower if it is big (>100). B<--bin> requires B<--pipe> and a fixed numeric value for B<--jobs>. -See also B<--shard>, B<--group-by>, B<--roundrobin>. +See also: B<--shard>, B<--group-by>, B<--roundrobin>. =item B<--bg> @@ -645,7 +645,7 @@ you can do: B. Implies B<--pipe> unless B<--pipepart> is used. -See also B<--fifo>. +See also: B<--fifo>. =item B<--cleanup> @@ -846,7 +846,7 @@ foreground (opposite B<--bg>), and wait for completion of the command before exiting. -See also B<--bg>, B. +See also: B<--bg>, B. =item B<--fifo> @@ -860,7 +860,7 @@ Beware: If data is not read from the fifo, the job will block forever. Implies B<--pipe> unless B<--pipepart> is used. -See also B<--cat>. +See also: B<--cat>. =item B<--filter-hosts> @@ -971,7 +971,7 @@ UserID when grouping: cat table.csv | parallel --pipe --colsep , --header : \ --group-by 'UserID s/\D//g' -kN1 wc -See also B<--shard>, B<--roundrobin>. +See also: B<--shard>, B<--roundrobin>. =item B<--help> @@ -1151,7 +1151,7 @@ If the host is long, you can use B to pretty print it: cat joblog | column -t -See also B<--resume> B<--resume-failed>. +See also: B<--resume> B<--resume-failed>. =item B<--jobs> I @@ -1427,6 +1427,34 @@ to be run later. B<--retries> must be set to determine how many times GNU B should retry a given job. +See also: B<--memsuspend> + + +=item B<--memsuspend> I (alpha testing) + +Suspend jobs when there is less than 2 * I memory free. The +I can be postfixed with K, M, G, T, P, k, m, g, t, or p which +would multiply the size with 1024, 1048576, 1073741824, 1099511627776, +1125899906842624, 1000, 1000000, 1000000000, 1000000000000, or +1000000000000000, respectively. + +If the available memory falls below 2 * I, GNU B +will suspend some of the running jobs. If the available memory falls +below I, only one job will be running. + +If a single job takes up at most I RAM, all jobs will complete +without running out of memory. If you have swap available, you can +usually lower I to around half the size of a single job - with +the slight risk of swapping a little. + +Jobs will be resumed when more RAM is available - typically when the +oldest job completes. + +B<--memsuspend> only works on local jobs because there is no obvious +way to suspend remote jobs. + +See also: B<--memfree> + =item B<--minversion> I @@ -1629,7 +1657,7 @@ everything at startup. By sending GNU B SIGUSR2 you can toggle turning on/off B<--progress> on a running GNU B process. -See also B<--eta> and B<--bar>. +See also: B<--eta> and B<--bar>. =item B<--max-line-length-allowed> @@ -1767,7 +1795,7 @@ B<--memfree> may give better results, so try using that first. Record current environment variables in ~/.parallel/ignored_vars. This is useful before using B<--env _>. -See also B<--env>, B<--session>. +See also: B<--env>, B<--session>. =item B<--recstart> I @@ -1933,7 +1961,7 @@ will generate the files: my_foo/stderr my_foo/stdout -See also B<--files>, B<--tag>, B<--header>, B<--joblog>. +See also: B<--files>, B<--tag>, B<--header>, B<--joblog>. =item B<--resume> @@ -1945,7 +1973,7 @@ sequence numbers in B<--joblog> then the input, the command, and B<--joblog> all have to remain unchanged; otherwise GNU B may run wrong commands. -See also B<--joblog>, B<--results>, B<--resume-failed>, B<--retries>. +See also: B<--joblog>, B<--results>, B<--resume-failed>, B<--retries>. =item B<--resume-failed> @@ -1958,7 +1986,7 @@ numbers in B<--joblog> then the input, the command, and B<--joblog> all have to remain unchanged; otherwise GNU B may run wrong commands. -See also B<--joblog>, B<--resume>, B<--retry-failed>, B<--retries>. +See also: B<--joblog>, B<--resume>, B<--retry-failed>, B<--retries>. =item B<--retry-failed> @@ -2028,7 +2056,7 @@ line: 6 [...] 2 0 echo 5;sleep .5; exit 2 4 [...] 1 0 echo 7;sleep .7; exit 1 -See also B<--joblog>, B<--resume>, B<--resume-failed>, B<--retries>. +See also: B<--joblog>, B<--resume>, B<--resume-failed>, B<--retries>. =item B<--retries> I @@ -2095,7 +2123,7 @@ impossible to track which input block corresponds to which output. B<--roundrobin> implies B<--pipe>, except if B<--pipepart> is given. -See also B<--group-by>, B<--shard>. +See also: B<--group-by>, B<--shard>. =item B<--rpl> 'I I' @@ -2220,7 +2248,7 @@ Used with B<--fg>, B<--wait>, and B<--semaphorename>. The command B is an alias for B. -See also B. +See also: B. =item B<--semaphorename> I @@ -2238,7 +2266,7 @@ The semaphore is stored in ~/.parallel/semaphores/ Implies B<--semaphore>. -See also B. +See also: B. =item B<--semaphoretimeout> I @@ -2251,7 +2279,7 @@ If I < 0: If the semaphore is not released within I seconds, exit. Implies B<--semaphore>. -See also B. +See also: B. =item B<--seqreplace> I @@ -2268,7 +2296,7 @@ variables with names in B<$PARALLEL_IGNORED_NAMES> will not be copied. Only supported in B. -See also B<--env>, B<--record-env>. +See also: B<--env>, B<--record-env>. =item B<--shard> I @@ -2290,7 +2318,7 @@ I is small (<10), slower if it is big (>100). B<--shard> requires B<--pipe> and a fixed numeric value for B<--jobs>. -See also B<--bin>, B<--group-by>, B<--roundrobin>. +See also: B<--bin>, B<--group-by>, B<--roundrobin>. =item B<--shebang> @@ -2695,7 +2723,7 @@ B<--timeout 1d3.5h16.6m4s>. Print the job to be run on stderr (standard error). -See also B<-v>, B<-p>. +See also: B<-v>, B<-p>. =item B<--transfer> @@ -2900,7 +2928,7 @@ Wait for all commands to complete. Used with B<--semaphore> or B<--sqlmaster>. -See also B. +See also: B. =item B<-X> @@ -2920,7 +2948,7 @@ unexpected results if B<{}> is used as part of a word. Support for B<-X> with B<--sshlogin> is limited and may fail. -See also B<-m>. +See also: B<-m>. =item B<--exit> diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 6de1bd9d..d09e9424 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -63,9 +63,9 @@ around 100 machines - depending on the duration of each job. GNU B busy waits. This is because the reason why a job is not started may be due to load average (when using B<--load>), and -thus it will not make sense to wait for a job to finish. Instead the -load average must be checked again. Load average is not the only -reason: B<--timeout> has a similar problem. +thus it will not make sense to just wait for a job to finish. Instead +the load average must be rechecked regularly. Load average is not the +only reason: B<--timeout> has a similar problem. To not burn up too much CPU GNU B sleeps exponentially longer and longer if nothing happens, maxing out at 1 second. diff --git a/src/parsort b/src/parsort index c22db80d..f806a37d 100755 --- a/src/parsort +++ b/src/parsort @@ -118,7 +118,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20210102; +$Global::version = 20210122; if($opt::version) { version(); exit 0; } @Global::sortoptions = shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); diff --git a/src/sql b/src/sql index 7336e232..476a35d9 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20201223; + $Global::version = 20210122; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 5075067e..f350aef7 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -952,7 +952,7 @@ par_hash_and_time_functions() { yyyymmdd()) =}' ::: 1 | perl -pe 's/\d/9/g' parallel echo '{= $_=hash($_) =}' ::: 1 | - perl -pe 's/[a-f0-9]/X/g' + perl -pe 's/[a-f0-9]+/X/g' } export -f $(compgen -A function | grep par_) diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 0b233885..8e7e1d76 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -921,16 +921,6 @@ par_sql_colsep() { rm /tmp/parallel-sql-colsep-$$ } -par_sql_CSV() { - echo '### CSV write to the right place' - rm -rf /tmp/parallel-CSV - mkdir /tmp/parallel-CSV - parallel --sqlandworker csv:///%2Ftmp%2Fparallel-CSV/OK echo ::: 'ran OK' - ls /tmp/parallel-CSV - stdout parallel --sqlandworker csv:///%2Fmust%2Ffail/fail echo ::: 1 | - perl -pe 's/\d/0/g' -} - export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | diff --git a/testsuite/tests-to-run/parallel-local-race02.sh b/testsuite/tests-to-run/parallel-local-race02.sh index 10fa265c..2099811a 100644 --- a/testsuite/tests-to-run/parallel-local-race02.sh +++ b/testsuite/tests-to-run/parallel-local-race02.sh @@ -29,6 +29,16 @@ echo 1=OK $?' | grep -v '\[1\]' | grep -v 'SHA256' #} +par_sql_CSV() { + echo '### CSV write to the right place' + rm -rf /tmp/parallel-CSV + mkdir /tmp/parallel-CSV + parallel --sqlandworker csv:///%2Ftmp%2Fparallel-CSV/OK echo ::: 'ran OK' + ls /tmp/parallel-CSV + stdout parallel --sqlandworker csv:///%2Fmust%2Ffail/fail echo ::: 1 | + perl -pe 's/\d/0/g' +} + par_hostgroup() { echo '### --hostgroup force ncpu' parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8} | sort diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 7fce198b..3c5ae520 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -4,112 +4,146 @@ unset run_test unset run_once # SSH only allowed to localhost/lo -# --retries if ssh dies -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 -r -echo '### --env _' - fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test - fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test -echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' - fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test - fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test +par_env_underscore() { + echo '### --env _' + echo ignored_var >> ~/.parallel/ignored_vars + unset $(compgen -A function | grep par_) + ignored_var="should not be copied" + export ignored_var + fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $ignored_var' ::: test + echo 'In csh this may fail with ignored_var: Undefined variable.' + fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $ignored_var' ::: test 2>&1 -echo 'bug #40137: SHELL not bash: Warning when exporting funcs' - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning + echo '### --env _ with explicit mentioning of normally ignored var $ignored_var' + ignored_var="should be copied" + fUbAr="OK FUBAR" parallel -S parallel@lo --env ignored_var,_ echo '$fUbAr $ignored_var' ::: test + fUbAr="OK FUBAR" parallel -S csh@lo --env ignored_var,_ echo '$fUbAr $ignored_var' ::: test 2>&1 +} -echo '### zsh' +par_warn_when_exporting_func() { + echo 'bug #40137: SHELL not bash: Warning when exporting funcs' + myrun() { + . <(printf 'myfunc() {\necho Function run: $1\n}') + export -f myfunc + echo "Run function in $1" + PARALLEL_SHELL=$1 parallel --env myfunc -S lo myfunc ::: OK + } + export -f myrun + parallel -k --tag myrun ::: /bin/{sh,bash} /usr/bin/{ash,csh,dash,ksh,tcsh,zsh} +} -echo 'env in zsh' - echo 'Normal variable export' - export B=\'; - PARALLEL_SHELL=/usr/bin/zsh parallel --env B echo '$B' ::: a +par_exporting_in_zsh() { + echo '### zsh' + + echo 'env in zsh' + echo 'Normal variable export' + export B=\'" Var with quote" + PARALLEL_SHELL=/usr/bin/zsh parallel --env B echo '$B' ::: OK - echo 'Function export as variable' - export myfuncvar="() { echo myfuncvar \$*; }"; - PARALLEL_SHELL=/usr/bin/zsh parallel --env myfuncvar myfuncvar ::: a + echo 'Function export as variable' + export myfuncvar="() { echo myfuncvar as var \$*; }" + PARALLEL_SHELL=/usr/bin/zsh parallel --env myfuncvar myfuncvar ::: OK - echo 'Function export as function' - myfunc() { echo myfunc $*; }; - export -f myfunc; - PARALLEL_SHELL=/usr/bin/zsh parallel --env myfunc myfunc ::: a + echo 'Function export as function' + myfunc() { echo myfunc ran $*; } + export -f myfunc + PARALLEL_SHELL=/usr/bin/zsh parallel --env myfunc myfunc ::: OK - - ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; + ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' - ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; + ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK' +} -echo '### csh' - echo "3 big vars run remotely - length(base64) > 1000" - stdout ssh csh@lo 'setenv A `seq 200|xargs`; +par_bigvar_csh() { + echo '### csh' + echo "3 big vars run remotely - length(base64) > 1000" + stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' -echo '### csh2' - echo "3 big vars run locally" - stdout ssh csh@lo 'setenv A `seq 200|xargs`; + echo '### csh2' + echo "3 big vars run locally" + stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' +} - -echo '### rc' - echo "3 big vars run remotely - length(base64) > 1000" - stdout ssh rc@lo 'A=`{seq 200}; +par_bigvar_rc() { + echo '### rc' + echo "3 big vars run remotely - length(base64) > 1000" + stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel -Src@lo --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' -echo '### rc2' - echo "3 big vars run locally" - stdout ssh rc@lo 'A=`{seq 200}; + echo '### rc2' + echo "3 big vars run locally" + stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' +} -echo '### Test tmux works on different shells' - (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output'; - (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | grep -v 'See output'; +par_tmux_different_shells() { + echo '### Test tmux works on different shells' + (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | + grep -v 'See output'; + (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | + grep -v 'See output'; - export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; - stdout ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; - stdout ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; - stdout ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?' | grep -v 'See output'; - stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?' | grep -v 'See output'; - stdout ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; - stdout ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; - echo "# command is currently too long for csh. Maybe it can be fixed?"; - stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; - stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output' + export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; + stdout ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; + stdout ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; + stdout ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?' | grep -v 'See output'; + stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?' | grep -v 'See output'; + stdout ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; + stdout ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; + echo "# command is currently too long for csh. Maybe it can be fixed?"; + stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; + stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output' +} -echo '### works' - stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output' +par_tmux_length() { + echo '### works' + stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' + stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output' -echo '### These blocked due to length' - stdout parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Scsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Stcsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Szsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Scsh@lo --tmux echo ::: 111111111111111111111111111111111111111111111111111111111 | grep -v 'See output' + echo '### These blocked due to length' + stdout parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' + stdout parallel -Scsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' + stdout parallel -Stcsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' + stdout parallel -Szsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' + stdout parallel -Scsh@lo --tmux echo ::: 111111111111111111111111111111111111111111111111111111111 | grep -v 'See output' +} -echo '### bug #43746: --transfer and --return of multiple inputs {1} and {2}' -echo '### and:' -echo '### bug #44371: --trc with csh complains' - cd /tmp; echo 1 > file1; echo 2 > file2; - parallel -Scsh@lo --transferfile {1} --transferfile {2} --trc {1}.a --trc {2}.b - '(cat {1}; echo A {1}) > {1}.a; (cat {2};echo B {2}) > {2}.b' ::: file1 ::: file2; - cat file1.a file2.b; - rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b +par_transfer_return_multiple_inputs() { + echo '### bug #43746: --transfer and --return of multiple inputs {1} and {2}' + echo '### and:' + echo '### bug #44371: --trc with csh complains' + cd /tmp; echo file1 output line 1 > file1; echo file2 output line 3 > file2 + parallel -Scsh@lo --transferfile {1} --transferfile {2} --trc {1}.a --trc {2}.b \ + '(cat {1}; echo A {1} output line 2) > {1}.a; (cat {2};echo B {2} output line 4) > {2}.b' ::: file1 ::: file2 + cat file1.a file2.b + rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b +} -echo '### bug #44143: csh and nice' - parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK +par_csh_nice() { + echo '### bug #44143: csh and nice' + parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK +} -echo '### bug #45575: -m and multiple hosts repeats first args' - seq 1 3 | parallel -X -S 2/lo,2/: -k echo +par_multiple_hosts_repeat_arg() { + echo '### bug #45575: -m and multiple hosts repeats first args' + seq 1 3 | parallel -X -S 2/lo,2/: -k echo +} -EOF +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | LC_ALL=C sort | + parallel --timeout 3000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | + perl -pe 's:/usr/bin:/bin:g;' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 5e37ae4f..02ccc099 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -199,7 +199,7 @@ par_halt_one_job parallel: This job failed: par_halt_one_job echo 1;exit 1 par_hash_and_time_functions ### Functions for replacement string par_hash_and_time_functions 9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99999999999999 999999999999 99999999 -par_hash_and_time_functions XxXXXXXXXX +par_hash_and_time_functions XxX par_help ### Test --help and -h: Help output (just check we get the same amount of lines) par_help Output from -h and --help par_help 37 diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index bb5e6f47..18ababd8 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -713,10 +713,6 @@ par_seqreplace_long_line ### Test --seqreplace and line too long par_seqreplace_long_line 9 1 1 101 par_seqreplace_long_line 90 1 1 201 par_seqreplace_long_line 1 parallel: Error: Command line too long (309 >= 210) at input 0: 100 -par_sql_CSV ### CSV write to the right place -par_sql_CSV ran OK -par_sql_CSV ok -par_sql_CSV parallel: Error: /must/fail is not a directory. par_sql_colsep ### SQL should add Vn columns for --colsep par_sql_colsep /a/A/1/11/ par_sql_colsep /a/A/2/22/ diff --git a/testsuite/wanted-results/parallel-local-race02 b/testsuite/wanted-results/parallel-local-race02 index 3fd6f542..d21ebff9 100644 --- a/testsuite/wanted-results/parallel-local-race02 +++ b/testsuite/wanted-results/parallel-local-race02 @@ -147,6 +147,10 @@ par_retries_bug_from_2010 4 par_retries_bug_from_2010 4 par_retries_bug_from_2010 4 par_retries_bug_from_2010 2 +par_sql_CSV ### CSV write to the right place +par_sql_CSV ran OK +par_sql_CSV ok +par_sql_CSV parallel: Error: /must/fail is not a directory. par_testhalt now fail 0 true ### testhalt --halt now,fail=0 par_testhalt now fail 0 true job 1 par_testhalt now fail 0 true parallel: This job failed: diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 9d9d0c70..bce6919c 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -1,118 +1,81 @@ -echo '### --env _' -### --env _ - fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR test - fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR test -echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' -### --env _ with explicit mentioning of normally ignored var $DEBEMAIL - fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR ole@tange.dk test - fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR ole@tange.dk test -echo 'bug #40137: SHELL not bash: Warning when exporting funcs' -bug #40137: SHELL not bash: Warning when exporting funcs - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning -no_warning - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning -warning -echo '### zsh' -### zsh -echo 'env in zsh' -env in zsh - echo 'Normal variable export' -Normal variable export - export B=\'; PARALLEL_SHELL=/usr/bin/zsh parallel --env B echo '$B' ::: a -' a - echo 'Function export as variable' -Function export as variable - export myfuncvar="() { echo myfuncvar \$*; }"; PARALLEL_SHELL=/usr/bin/zsh parallel --env myfuncvar myfuncvar ::: a -myfuncvar a - echo 'Function export as function' -Function export as function - myfunc() { echo myfunc $*; }; export -f myfunc; PARALLEL_SHELL=/usr/bin/zsh parallel --env myfunc myfunc ::: a -myfunc a - ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' -function from zsh to zsh OK - ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; export fun; parallel -S parallel@lo --env fun fun ::: OK' -function from zsh to bash OK -echo '### csh' -### csh - echo "3 big vars run remotely - length(base64) > 1000" -3 big vars run remotely - length(base64) > 1000 - stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' - 1 200 692 - 1 200 692 - 1 150 547 -echo '### csh2' -### csh2 - echo "3 big vars run locally" -3 big vars run locally - stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' - 1 200 692 - 1 200 692 - 1 150 547 -echo '### rc' -### rc - echo "3 big vars run remotely - length(base64) > 1000" -3 big vars run remotely - length(base64) > 1000 - stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel -Src@lo --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' - 1 200 692 - 1 200 692 - 1 150 547 -echo '### rc2' -### rc2 - echo "3 big vars run locally" -3 big vars run locally - stdout ssh rc@lo 'A=`{seq 200}; B=`{seq 200 -1 1}; C=`{seq 300 -2 1}; parallel --env A,B,C -k echo '"'"'${}|wc'"'"' ::: A B C' - 1 200 692 - 1 200 692 - 1 150 547 -echo '### Test tmux works on different shells' -### Test tmux works on different shells - (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output'; -0 - (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | grep -v 'See output'; -4 - export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; stdout ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; echo "# command is currently too long for csh. Maybe it can be fixed?"; stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output' -0 -4 -0 -4 -0 -4 -# command is currently too long for csh. Maybe it can be fixed? -0 -4 -echo '### works' -### works - stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output' -echo '### These blocked due to length' -### These blocked due to length - stdout parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Scsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Stcsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Szsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' - stdout parallel -Scsh@lo --tmux echo ::: 111111111111111111111111111111111111111111111111111111111 | grep -v 'See output' -echo '### bug #43746: --transfer and --return of multiple inputs {1} and {2}' -### bug #43746: --transfer and --return of multiple inputs {1} and {2} -echo '### and:' -### and: -echo '### bug #44371: --trc with csh complains' -### bug #44371: --trc with csh complains - cd /tmp; echo 1 > file1; echo 2 > file2; parallel -Scsh@lo --transferfile {1} --transferfile {2} --trc {1}.a --trc {2}.b '(cat {1}; echo A {1}) > {1}.a; (cat {2};echo B {2}) > {2}.b' ::: file1 ::: file2; cat file1.a file2.b; rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b -1 -A file1 -2 -B file2 -echo '### bug #44143: csh and nice' -### bug #44143: csh and nice - parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK -OK -echo '### bug #45575: -m and multiple hosts repeats first args' -### bug #45575: -m and multiple hosts repeats first args - seq 1 3 | parallel -X -S 2/lo,2/: -k echo -1 -2 -3 +par_bigvar_csh ### csh +par_bigvar_csh 3 big vars run remotely - length(base64) > 1000 +par_bigvar_csh 1 200 692 +par_bigvar_csh 1 200 692 +par_bigvar_csh 1 150 547 +par_bigvar_csh ### csh2 +par_bigvar_csh 3 big vars run locally +par_bigvar_csh 1 200 692 +par_bigvar_csh 1 200 692 +par_bigvar_csh 1 150 547 +par_bigvar_rc ### rc +par_bigvar_rc 3 big vars run remotely - length(base64) > 1000 +par_bigvar_rc 1 200 692 +par_bigvar_rc 1 200 692 +par_bigvar_rc 1 150 547 +par_bigvar_rc ### rc2 +par_bigvar_rc 3 big vars run locally +par_bigvar_rc 1 200 692 +par_bigvar_rc 1 200 692 +par_bigvar_rc 1 150 547 +par_csh_nice ### bug #44143: csh and nice +par_csh_nice OK +par_env_underscore ### --env _ +par_env_underscore OK FUBAR test +par_env_underscore In csh this may fail with ignored_var: Undefined variable. +par_env_underscore ignored_var: Undefined variable. +par_env_underscore ### --env _ with explicit mentioning of normally ignored var $ignored_var +par_env_underscore OK FUBAR should be copied test +par_env_underscore OK FUBAR should be copied test +par_exporting_in_zsh ### zsh +par_exporting_in_zsh env in zsh +par_exporting_in_zsh Normal variable export +par_exporting_in_zsh ' Var with quote OK +par_exporting_in_zsh Function export as variable +par_exporting_in_zsh myfuncvar as var OK +par_exporting_in_zsh Function export as function +par_exporting_in_zsh myfunc ran OK +par_exporting_in_zsh function from zsh to zsh OK +par_exporting_in_zsh function from zsh to bash OK +par_multiple_hosts_repeat_arg ### bug #45575: -m and multiple hosts repeats first args +par_multiple_hosts_repeat_arg parallel: Warning: Using -X or -m with --sshlogin may fail. +par_multiple_hosts_repeat_arg 1 +par_multiple_hosts_repeat_arg 2 +par_multiple_hosts_repeat_arg 3 +par_tmux_different_shells ### Test tmux works on different shells +par_tmux_different_shells 0 +par_tmux_different_shells 4 +par_tmux_different_shells 0 +par_tmux_different_shells 4 +par_tmux_different_shells 0 +par_tmux_different_shells 4 +par_tmux_different_shells 0 +par_tmux_different_shells 4 +par_tmux_different_shells # command is currently too long for csh. Maybe it can be fixed? +par_tmux_different_shells 0 +par_tmux_different_shells 4 +par_tmux_length ### works +par_tmux_length ### These blocked due to length +par_transfer_return_multiple_inputs ### bug #43746: --transfer and --return of multiple inputs {1} and {2} +par_transfer_return_multiple_inputs ### and: +par_transfer_return_multiple_inputs ### bug #44371: --trc with csh complains +par_transfer_return_multiple_inputs file1 output line 1 +par_transfer_return_multiple_inputs A file1 output line 2 +par_transfer_return_multiple_inputs file2 output line 3 +par_transfer_return_multiple_inputs B file2 output line 4 +par_warn_when_exporting_func bug #40137: SHELL not bash: Warning when exporting funcs +par_warn_when_exporting_func /bin/sh Run function in /bin/sh +par_warn_when_exporting_func /bin/sh parallel: Warning: Shell functions may not be supported in /bin/sh. +par_warn_when_exporting_func /bin/bash Run function in /bin/bash +par_warn_when_exporting_func /bin/bash Function run: OK +par_warn_when_exporting_func /bin/ash Run function in /bin/ash +par_warn_when_exporting_func /bin/csh Run function in /bin/csh +par_warn_when_exporting_func /bin/csh parallel: Warning: Shell functions may not be supported in /bin/csh. +par_warn_when_exporting_func /bin/csh CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset myfunc +par_warn_when_exporting_func /bin/dash Run function in /bin/dash +par_warn_when_exporting_func /bin/ksh Run function in /bin/ksh +par_warn_when_exporting_func /bin/tcsh Run function in /bin/tcsh +par_warn_when_exporting_func /bin/tcsh parallel: Warning: Shell functions may not be supported in /bin/tcsh. +par_warn_when_exporting_func /bin/tcsh CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset myfunc +par_warn_when_exporting_func /bin/zsh Run function in /bin/zsh +par_warn_when_exporting_func /bin/zsh Function run: OK