diff --git a/NEWS b/NEWS index f741e127..dbe02946 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,26 @@ +20211222 + +New in this release: + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* The first version of Parallel will be 20 years on 2022-01-06 + +* HerrComp Intro HPC gnu parallel + https://www.youtube.com/watch?v=tcgjKWfgWQY + +* 多数のファイルを処理する場合の工夫 + https://qiita.com/toramili/items/44a770cfb9ef1edc9e26 + +* How To Install parallel on Ubuntu 20.04 + https://installati.one/ubuntu/20.04/parallel/ + +* Distributing embarrassingly parallel tasks GNU Parallel + https://ulhpc-tutorials.readthedocs.io/en/latest/sequential/gnu-parallel/ + + 20211122 New in this release: diff --git a/README b/README index 765de884..efeafe79 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20211122.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20211122.tar.bz2.sig - gpg parallel-20211122.tar.bz2.sig - bzip2 -dc parallel-20211122.tar.bz2 | tar xvf - - cd parallel-20211122 + wget https://ftpmirror.gnu.org/parallel/parallel-20211222.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20211222.tar.bz2.sig + gpg parallel-20211222.tar.bz2.sig + bzip2 -dc parallel-20211222.tar.bz2 | tar xvf - + cd parallel-20211222 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20211122.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20211122.tar.bz2.sig - gpg parallel-20211122.tar.bz2.sig - bzip2 -dc parallel-20211122.tar.bz2 | tar xvf - - cd parallel-20211122 + wget https://ftpmirror.gnu.org/parallel/parallel-20211222.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20211222.tar.bz2.sig + gpg parallel-20211222.tar.bz2.sig + bzip2 -dc parallel-20211222.tar.bz2 | tar xvf - + cd parallel-20211222 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2021, November 22). GNU Parallel 20211122 ('Peng Shuai'). - Zenodo. https://doi.org/10.5281/zenodo.5719513 + Tange, O. (2021, December 22). GNU Parallel 20211222 ('Støjberg'). + Zenodo. https://doi.org/10.5281/zenodo.5797028 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ole Tange, http://ole.tange.dk and diff --git a/configure b/configure index 3602bad0..9f88ff19 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20211122. +# Generated by GNU Autoconf 2.69 for parallel 20211222. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20211122' -PACKAGE_STRING='parallel 20211122' +PACKAGE_VERSION='20211222' +PACKAGE_STRING='parallel 20211222' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20211122 to adapt to many kinds of systems. +\`configure' configures parallel 20211222 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20211122:";; + short | recursive ) echo "Configuration of parallel 20211222:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20211122 +parallel configure 20211222 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20211122, which was +It was created by parallel $as_me 20211222, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20211122' + VERSION='20211222' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20211122, which was +This file was extended by parallel $as_me 20211222, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20211122 +parallel config.status 20211222 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 5aa52953..f58701a9 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20211122], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20211222], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 37ee1e58..92b85a97 100644 --- a/doc/haikus +++ b/doc/haikus @@ -53,7 +53,7 @@ Quote of the month: This is a fantastic tool, and I wish I had upgraded from xargs years ago! -- Stuart Anderson - GNU Parallel and ripgrep would be your friend here. Ripgrep is fast, really fast. + GNU Parallel and ripgrep would be your friend here. -- CENAPT @cenaptech@twitter GNU Parallel is great @@ -82,9 +82,6 @@ Quote of the month: involved -- rrees @rrees@twitter - My favorite man page is that of GNU parallel. - -- Jeroen Janssens @jeroenhjanssens@twitter - I have gotten a *ton* of mileage out of jq, awk, and GNU parallel, even at multi-GB sizes. -- Eric Wolak @ericthewolak@twitter @@ -144,6 +141,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + My favorite man page is that of GNU parallel. + -- Jeroen Janssens @jeroenhjanssens@twitter + GNU parallel 便利すぎ -- @butagannen@twitter 豚顔面 diff --git a/doc/release_new_version b/doc/release_new_version index 030b9939..e92f3c67 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -254,16 +254,16 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20211122 ('Peng Shuai') released [stable] +Subject: GNU Parallel 20211222 ('Støjberg') released [stable] -GNU Parallel 20211122 ('Peng Shuai') [stable] has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20211222 ('Støjberg') [stable] has been released. It is available for download at: lbry://@GnuParallel:4 No new functionality was introduced so this is a good candidate for a stable release. Quote of the month: - GNU parallel 便利すぎ - -- @butagannen@twitter 豚顔面 + My favorite man page is that of GNU parallel. + -- Jeroen Janssens @jeroenhjanssens@twitter New in this release: @@ -271,9 +271,16 @@ New in this release: News about GNU Parallel: -* Otimizando o seu Hacking com o GNU Parallel - Mateus Buogo https://www.youtube.com/watch?v=xIpPPZXDKGU +* The first version of Parallel will be 20 years on 2022-01-06 + +* HerrComp Intro HPC gnu parallel https://www.youtube.com/watch?v=tcgjKWfgWQY + +* 多数のファイルを処理する場合の工夫 https://qiita.com/toramili/items/44a770cfb9ef1edc9e26 + +* How To Install parallel on Ubuntu 20.04 https://installati.one/ubuntu/20.04/parallel/ + +* Distributing embarrassingly parallel tasks GNU Parallel https://ulhpc-tutorials.readthedocs.io/en/latest/sequential/gnu-parallel/ -* SMACK 12 - Do it faster! Simple ways to use all those cores (and GPUs) efficiently. https://www.youtube.com/watch?v=lXDGY7NvOYk Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html diff --git a/packager/obs/home:tange/parallel/.osc/_files b/packager/obs/home:tange/parallel/.osc/_files index a540877b..b2bfc0d9 100644 --- a/packager/obs/home:tange/parallel/.osc/_files +++ b/packager/obs/home:tange/parallel/.osc/_files @@ -1,6 +1,7 @@ - - - - - + + + + + + diff --git a/packager/obs/home:tange/parallel/.osc/parallel.spec b/packager/obs/home:tange/parallel/.osc/parallel.spec index 6a658e13..b284bd13 100644 --- a/packager/obs/home:tange/parallel/.osc/parallel.spec +++ b/packager/obs/home:tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20211122 +Version: 20211222 Release: 1.3 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/src/env_parallel b/src/env_parallel index d8e9f280..1059e166 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -30,10 +30,10 @@ installer() { source="$1" script="$2" into="$3" - if grepq $script $into; then + if grepq $script "$into"; then true already installed else - echo $source \`which $script\` >> $into + echo $source \`which $script\` >> "$into" fi } @@ -42,32 +42,32 @@ while test $# -gt 0; do case $key in -i|--install) - installer . env_parallel.bash $HOME/.bashrc - installer . env_parallel.sh $HOME/.shrc - installer . env_parallel.zsh $HOME/.zshenv - installer source env_parallel.ksh $HOME/.kshrc - installer source env_parallel.mksh $HOME/.mkshrc + installer . env_parallel.bash "$HOME"/.bashrc + installer . env_parallel.sh "$HOME"/.shrc + installer . env_parallel.zsh "$HOME"/.zshenv + installer source env_parallel.ksh "$HOME"/.kshrc + installer source env_parallel.mksh "$HOME"/.mkshrc echo $SHELL | grepq /pdksh && - installer . env_parallel.pdksh $HOME/.profile + installer . env_parallel.pdksh "$HOME"/.profile echo $SHELL | grepq /ash && - installer . env_parallel.ash $HOME/.profile + installer . env_parallel.ash "$HOME"/.profile echo $SHELL | grepq /dash && - installer . env_parallel.dash $HOME/.profile - installer source env_parallel.csh $HOME/.cshrc - installer source env_parallel.tcsh $HOME/.tcshrc - mkdir -p $HOME/.config/fish - grepq env_parallel.fish $HOME/.config/fish/config.fish || - echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish + installer . env_parallel.dash "$HOME"/.profile + installer source env_parallel.csh "$HOME"/.cshrc + installer source env_parallel.tcsh "$HOME"/.tcshrc + mkdir -p "$HOME"/.config/fish + grepq env_parallel.fish "$HOME"/.config/fish/config.fish || + echo '. (which env_parallel.fish)' >> "$HOME"/.config/fish/config.fish echo 'Installed env_parallel in:' - echo " " $HOME/.bashrc - echo " " $HOME/.shrc - echo " " $HOME/.zshenv - echo " " $HOME/.config/fish/config.fish - echo " " $HOME/.kshrc - echo " " $HOME/.mkshrc - echo " " $HOME/.profile - echo " " $HOME/.cshrc - echo " " $HOME/.tcshrc + echo " " "$HOME"/.bashrc + echo " " "$HOME"/.shrc + echo " " "$HOME"/.zshenv + echo " " "$HOME"/.config/fish/config.fish + echo " " "$HOME"/.kshrc + echo " " "$HOME"/.mkshrc + echo " " "$HOME"/.profile + echo " " "$HOME"/.cshrc + echo " " "$HOME"/.tcshrc exit ;; *) diff --git a/src/env_parallel.ash b/src/env_parallel.ash index d6c716c3..357bdb65 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index b70edbdf..2e3ecde2 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -384,7 +384,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index a1323994..55430e81 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 138ccff9..4d3e5bc6 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -363,7 +363,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 4eeda8ee..c69e50b2 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -365,7 +365,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 0a659c83..c1595211 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -390,7 +390,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 2300d873..1a5bfa27 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -355,7 +355,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20211122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20211222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index a3f8c50f..a27b81c2 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20211122; +$Global::version = 20211222; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 61f5731c..4c9c437b 100755 --- a/src/parallel +++ b/src/parallel @@ -924,7 +924,7 @@ sub group_by_loop($$) { return $loop; } -sub group_by_stdin_filter() { +sub pipe_group_by_setup() { # Record separator with 119 bit random value $opt::recend = ''; $opt::recstart = @@ -2264,7 +2264,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20211122; + $Global::version = 20211222; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -5077,8 +5077,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2021, November 22). GNU Parallel 20211122 ('Peng Shuai').", - " Zenodo. https://doi.org/10.5281/zenodo.5719513", + " Tange, O. (2021, December 22). GNU Parallel 20211222 ('Støjberg').", + " Zenodo. https://doi.org/10.5281/zenodo.5797028", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice @@ -5110,8 +5110,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2021, November 22). GNU Parallel 20211122 ('Peng Shuai').", - " Zenodo. https://doi.org/10.5281/zenodo.5719513", + " Tange, O. (2021, December 22). GNU Parallel 20211222 ('Støjberg').", + " Zenodo. https://doi.org/10.5281/zenodo.5797028", "", # Before changing these line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -5236,20 +5236,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2021_5719513,", + "\@software{tange_2021_5797028,", " author = {Tange, Ole},", - " title = {GNU Parallel 20211122 ('Peng Shuai')},", - " month = Nov,", + " title = {GNU Parallel 20211222 ('Støjberg')},", + " month = Dec,", " year = 2021,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.5719513},", - " url = {https://doi.org/10.5281/zenodo.5719513}", + " doi = {10.5281/zenodo.5797028},", + " url = {https://doi.org/10.5281/zenodo.5797028}", "}", "", - "(Feel free to use \\nocite{tange_2021_5719513})", + "(Feel free to use \\nocite{tange_2021_5797028})", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -13991,15 +13991,16 @@ sub main() { if($opt::pipepart) { pipepart_setup(); - } elsif($opt::pipe and $opt::tee) { - pipe_tee_setup(); - } elsif($opt::pipe and $opt::shard or $opt::bin) { - pipe_shard_setup(); + } elsif($opt::pipe) { + if($opt::tee) { + pipe_tee_setup(); + } elsif($opt::shard or $opt::bin) { + pipe_shard_setup(); + } elsif($opt::groupby) { + pipe_group_by_setup(); + } } - if(not $opt::pipepart and $opt::groupby) { - group_by_stdin_filter(); - } if($opt::eta or $opt::bar or $opt::shuf or $Global::halt_pct) { # Count the number of jobs or shuffle all jobs # before starting any. diff --git a/src/parsort b/src/parsort index 7482ea62..4f53a092 100755 --- a/src/parsort +++ b/src/parsort @@ -122,7 +122,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20211122; +$Global::version = 20211222; if($opt::version) { version(); exit 0; } @Global::sortoptions = grep { ! /^-D$/ } shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); diff --git a/src/sql b/src/sql index 97e4c93c..4837cd2b 100755 --- a/src/sql +++ b/src/sql @@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20211122; + $Global::version = 20211222; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 50577bda..12bdb6a0 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -8,61 +8,6 @@ # Each should be taking 10-30s and be possible to run in parallel # I.e.: No race conditions, no logins -par_shard() { - echo '### --shard' - # Each of the 5 lines should match: - # ##### ##### ###### - seq 100000 | parallel --pipe --shard 1 -j5 wc | - perl -pe 's/(.*\d{5,}){3}/OK/' - # Data should be sharded to all processes - shard_on_col() { - col=$1 - seq 10 99 | shuf | perl -pe 's/(.)/$1\t/g' | - parallel --pipe --shard $col -j2 --colsep "\t" sort -k$col | - field $col | sort | uniq -c - } - shard_on_col 1 - shard_on_col 2 - - shard_on_col_name() { - colname=$1 - col=$2 - (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | - parallel --header : --pipe --shard $colname -j2 --colsep "\t" sort -k$col | - field $col | sort | uniq -c - } - shard_on_col_name A 1 - shard_on_col_name B 2 - - shard_on_col_expr() { - colexpr="$1" - col=$2 - (seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | - parallel --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | - field $col | sort | uniq -c - } - shard_on_col_expr '1 $_%=3' 1 - shard_on_col_expr '2 $_%=3' 2 - - shard_on_col_name_expr() { - colexpr="$1" - col=$2 - (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | - parallel --header : --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | - field $col | sort | uniq -c - } - shard_on_col_name_expr 'A $_%=3' 1 - shard_on_col_name_expr 'B $_%=3' 2 - - echo '*** broken' - # Shorthand for --pipe -j+0 - seq 100000 | parallel --shard 1 wc | - perl -pe 's/(.*\d{5,}){3}/OK/' - # Combine with arguments - seq 100000 | parallel --shard 1 echo {}\;wc ::: {1..5} ::: a b | - perl -pe 's/(.*\d{5,}){3}/OK/' -} - par_bin() { echo '### Test --bin' seq 10 | parallel --pipe --bin 1 -j4 wc | sort diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 3c47fc95..cfe94082 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -8,6 +8,61 @@ # Each should be taking 30-100s and be possible to run in parallel # I.e.: No race conditions, no logins +par_shard() { + echo '### --shard' + # Each of the 5 lines should match: + # ##### ##### ###### + seq 100000 | parallel --pipe --shard 1 -j5 wc | + perl -pe 's/(.*\d{5,}){3}/OK/' + # Data should be sharded to all processes + shard_on_col() { + col=$1 + seq 10 99 | shuf | perl -pe 's/(.)/$1\t/g' | + parallel --pipe --shard $col -j2 --colsep "\t" sort -k$col | + field $col | sort | uniq -c + } + shard_on_col 1 + shard_on_col 2 + + shard_on_col_name() { + colname=$1 + col=$2 + (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | + parallel --header : --pipe --shard $colname -j2 --colsep "\t" sort -k$col | + field $col | sort | uniq -c + } + shard_on_col_name A 1 + shard_on_col_name B 2 + + shard_on_col_expr() { + colexpr="$1" + col=$2 + (seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | + parallel --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | + field $col | sort | uniq -c + } + shard_on_col_expr '1 $_%=3' 1 + shard_on_col_expr '2 $_%=3' 2 + + shard_on_col_name_expr() { + colexpr="$1" + col=$2 + (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | + parallel --header : --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | + field $col | sort | uniq -c + } + shard_on_col_name_expr 'A $_%=3' 1 + shard_on_col_name_expr 'B $_%=3' 2 + + echo '*** broken' + # Shorthand for --pipe -j+0 + seq 200000 | parallel --pipe --shard 1 wc | + perl -pe 's/(.*\d{5,}){3}/OK/' + # Combine with arguments + seq 200000 | parallel --pipe --shard 1 echo {}\;wc ::: {1..5} ::: a b | + perl -pe 's/(.*\d{5,}){3}/OK/' +} + par_test_diff_roundrobin_k() { echo '### test there is difference on -k' . $(which env_parallel.bash) diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 45921cf4..d7273f78 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -936,97 +936,6 @@ par_semaphore job3a 4 par_semaphore job2b 5 par_semaphore job3b 6 par_semaphore done -par_shard ### --shard -par_shard OK -par_shard OK -par_shard OK -par_shard OK -par_shard OK -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 2 c1 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 2 c2 -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 2 c1 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 2 c2 -par_shard *** broken -par_shard parallel: Error: --shard requires --jobs to be higher than the number of -par_shard parallel: Error: arguments. Increase --jobs. -par_shard parallel: Error: --shard requires --jobs to be higher than the number of -par_shard parallel: Error: arguments. Increase --jobs. par_shellquote ### Test --shellquote in all shells par_shellquote ash ' par_shellquote ash par_shellquote ash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index 488a4e0f..17ce2dbc 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -1640,6 +1640,93 @@ par_race_condition1 9 par_race_condition1 10 par_round_robin_blocks bug #49664: --round-robin does not complete par_round_robin_blocks 8 +par_shard ### --shard +par_shard OK +par_shard OK +par_shard OK +par_shard OK +par_shard OK +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 2 c1 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 2 c2 +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 2 c1 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 2 c2 +par_shard *** broken par_sighup ### Test SIGHUP par_sighup 1 par_sighup 10 diff --git a/testsuite/wanted-results/parallel-local15 b/testsuite/wanted-results/parallel-local15 index 908fc2ad..7098a476 100644 --- a/testsuite/wanted-results/parallel-local15 +++ b/testsuite/wanted-results/parallel-local15 @@ -171,8 +171,8 @@ ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_= 4 ab 4 b 6 a -parallel: Warning: Only enough file handles to run 251 jobs in parallel. -parallel: Warning: Try running 'parallel -j0 -N 251 --pipe parallel -j0' +parallel: Warning: Only enough file handles to run 252 jobs in parallel. +parallel: Warning: Try running 'parallel -j0 -N 252 --pipe parallel -j0' parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`) parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf parallel: Warning: or increasing /proc/sys/fs/file-max @@ -285,8 +285,8 @@ ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\ 4 ab 4 b 6 a -parallel: Warning: Only enough file handles to run 251 jobs in parallel. -parallel: Warning: Try running 'parallel -j0 -N 251 --pipe parallel -j0' +parallel: Warning: Only enough file handles to run 252 jobs in parallel. +parallel: Warning: Try running 'parallel -j0 -N 252 --pipe parallel -j0' parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`) parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf parallel: Warning: or increasing /proc/sys/fs/file-max diff --git a/testsuite/wanted-results/parallel-macos b/testsuite/wanted-results/parallel-macos index 5c8a0d1b..f176e0c2 100644 --- a/testsuite/wanted-results/parallel-macos +++ b/testsuite/wanted-results/parallel-macos @@ -1,12 +1,12 @@ -par_big_func 1 7968 191232 -par_big_func 1 5368 128772 +par_big_func 1 7964 191136 +par_big_func 1 5372 128868 par_big_func_name 1 4960 119040 par_big_func_name 1 40 960 par_big_var_func_name 1 4956 118944 par_big_var_func_name 1 4956 118944 par_big_var_func_name 1 3424 82116 -par_many_args 1 16400 32800 -par_many_args 1 3600 7200 +par_many_args 1 16396 32792 +par_many_args 1 3604 7208 par_many_func 1 4372 104928 par_many_func 1 2296 55076 par_many_var 1 5124 122976 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 574ebbf4..1ddb45b1 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -499,8 +499,8 @@ echo C sleep 1 echo 0 > my_jobs wait -parallel: Warning: Only enough file handles to run 247 jobs in parallel. -parallel: Warning: Try running 'parallel -j0 -N 247 --pipe parallel -j0' +parallel: Warning: Only enough file handles to run 248 jobs in parallel. +parallel: Warning: Try running 'parallel -j0 -N 248 --pipe parallel -j0' parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`) parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf parallel: Warning: or increasing /proc/sys/fs/file-max