diff --git a/Makefile.am b/Makefile.am index b10b94ef..4234028e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -167,8 +167,11 @@ pack_unpack_and_test_build: cd parallel-$(YYYYMMDD) && \ ./configure && make -j && sudo make -j install -zshcompletion=${datarootdir}/zsh/site-functions/_parallel +zshcompletiondir=${datarootdir}/zsh/site-functions +zshcompletion=${zshcompletiondir}/_parallel install-data-hook: + mkdir -p ${datarootdir}/bash-completion/completions + mkdir -p ${zshcompletiondir} parallel --shell-completion bash > ${datarootdir}/bash-completion/completions/parallel echo '#compdef parallel' > ${zshcompletion} echo '(( $$+functions[_comp_parallel] )) ||' >> ${zshcompletion} diff --git a/Makefile.in b/Makefile.in index fe268348..63541f54 100644 --- a/Makefile.in +++ b/Makefile.in @@ -281,7 +281,8 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ SUBDIRS = src -zshcompletion = ${datarootdir}/zsh/site-functions/_parallel +zshcompletiondir = ${datarootdir}/zsh/site-functions +zshcompletion = ${zshcompletiondir}/_parallel EXTRA_DIST = CITATION CITATION.cff CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt all: config.h $(MAKE) $(AM_MAKEFLAGS) all-recursive @@ -928,6 +929,8 @@ pack_unpack_and_test_build: cd parallel-$(YYYYMMDD) && \ ./configure && make -j && sudo make -j install install-data-hook: + mkdir -p ${datarootdir}/bash-completion/completions + mkdir -p ${zshcompletiondir} parallel --shell-completion bash > ${datarootdir}/bash-completion/completions/parallel echo '#compdef parallel' > ${zshcompletion} echo '(( $$+functions[_comp_parallel] )) ||' >> ${zshcompletion} diff --git a/NEWS b/NEWS index aea63ecb..d04e8888 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,20 @@ +20220622 + +New in this release: + +* , can be used in --sshlogin if quoted as \, or ,, + +* --plus {/#regexp/str} replace ^regexp with str. + +* --plus {/%regexp/str} replace regexp$ with str. + +* --plus {//regexp/str} replace every regexp with str. + +* 'make install' installs bash+zsh completion files. + +* Bug fixes and man page updates. + + 20220522 New in this release: diff --git a/README b/README index 755eba21..4f123315 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20220522.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20220522.tar.bz2.sig - gpg parallel-20220522.tar.bz2.sig - bzip2 -dc parallel-20220522.tar.bz2 | tar xvf - - cd parallel-20220522 + wget https://ftpmirror.gnu.org/parallel/parallel-20220622.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20220622.tar.bz2.sig + gpg parallel-20220622.tar.bz2.sig + bzip2 -dc parallel-20220622.tar.bz2 | tar xvf - + cd parallel-20220622 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20220522.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20220522.tar.bz2.sig - gpg parallel-20220522.tar.bz2.sig - bzip2 -dc parallel-20220522.tar.bz2 | tar xvf - - cd parallel-20220522 + wget https://ftpmirror.gnu.org/parallel/parallel-20220622.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20220622.tar.bz2.sig + gpg parallel-20220622.tar.bz2.sig + bzip2 -dc parallel-20220622.tar.bz2 | tar xvf - + cd parallel-20220622 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2022, May 22). GNU Parallel 20220522 ('NATO'). - Zenodo. https://doi.org/10.5281/zenodo.6570228 + Tange, O. (2022, June 22). GNU Parallel 20220622 ('Bongbong'). + Zenodo. https://doi.org/10.5281/zenodo.6682930 Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. diff --git a/configure b/configure index c10135b4..a671e075 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for parallel 20220522. +# Generated by GNU Autoconf 2.71 for parallel 20220622. # # Report bugs to . # @@ -610,8 +610,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20220522' -PACKAGE_STRING='parallel 20220522' +PACKAGE_VERSION='20220622' +PACKAGE_STRING='parallel 20220622' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1246,7 +1246,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20220522 to adapt to many kinds of systems. +\`configure' configures parallel 20220622 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1313,7 +1313,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20220522:";; + short | recursive ) echo "Configuration of parallel 20220622:";; esac cat <<\_ACEOF @@ -1390,7 +1390,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20220522 +parallel configure 20220622 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -1427,7 +1427,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20220522, which was +It was created by parallel $as_me 20220622, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2379,7 +2379,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20220522' + VERSION='20220622' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -3026,7 +3026,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20220522, which was +This file was extended by parallel $as_me 20220622, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3090,7 +3090,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -parallel config.status 20220522 +parallel config.status 20220622 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 3aba47f2..6142c381 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel],[20220522],[bug-parallel@gnu.org]) +AC_INIT([parallel],[20220622],[bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 7d9df0ad..b1448c60 100644 --- a/doc/haikus +++ b/doc/haikus @@ -13,9 +13,6 @@ Quote of the month: I used GNU Parallel to run a script locally that did what a big distributed system did, quicker and more reliably. It got to the point where people would just ask me to "run the thing" on my laptop instead of waiting for the cron. -- winrid@ycombinator - Parallel has been (and still is) super useful and simple tool for speeding up all kinds of shell tasks during my career. - -- ValtteriL@ycombinator - parallel is so useful and i use it multiple times daily. -- arrakeen@ycombinator @@ -189,6 +186,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + Parallel has been (and still is) super useful and simple tool for speeding up all kinds of shell tasks during my career. + -- ValtteriL@ycombinator + It's amazing how fast you can get with bash pipelines and GNU Parallel. -- Eric Pauley @EricPauley_ diff --git a/doc/release_new_version b/doc/release_new_version index f375d0e8..a556ba24 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -260,30 +260,23 @@ GNU Parallel 20220522 ('NATO') has been released. It is available for download a Quote of the month: - It's amazing how fast you can get with bash pipelines and GNU Parallel. - -- Eric Pauley @EricPauley_ - + Parallel has been (and still is) super useful and simple tool for speeding up all kinds of shell tasks during my career. + -- ValtteriL@ycombinator + New in this release: -* --latest-line shows only the latest line of running jobs. +* , can be used in --sshlogin if quoted as \, or ,, -* --color colors output in different colors per job (this obsoletes --ctag). +* --plus {/#regexp/str} replace ^regexp with str. -* xargs compatibility: --process-slot-var foo sets $foo to jobslot-1. +* --plus {/%regexp/str} replace regexp$ with str. -* xargs compatibility: --open-tty opens the terminal on stdin (standard input). +* --plus {//regexp/str} replace every regexp with str. + +* 'make install' installs bash+zsh completion files. * Bug fixes and man page updates. -News about GNU Parallel: - -* Building a fault-tolerant work queue for command-line executions with GNU Parallel https://www.jvt.me/posts/2022/04/28/shell-queue/ - -* Make your code do more, with less https://www.blopig.com/blog/2022/05/make-your-code-do-more-with-less/ - -* W09 B IntroHPC II 20220513 gnu parallel https://www.youtube.com/watch?v=2udvC90rIa4 - -Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html GNU Parallel - For people who live life in the parallel lane. diff --git a/install-sh b/install-sh index 8175c640..ec298b53 100755 --- a/install-sh +++ b/install-sh @@ -1,7 +1,7 @@ #!/bin/sh # install - install a program, script, or datafile -scriptversion=2018-03-11.20; # UTC +scriptversion=2020-11-14.01; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the @@ -69,6 +69,11 @@ posix_mkdir= # Desired mode of installed file. mode=0755 +# Create dirs (including intermediate dirs) using mode 755. +# This is like GNU 'install' as of coreutils 8.32 (2020). +mkdir_umask=22 + +backupsuffix= chgrpcmd= chmodcmd=$chmodprog chowncmd= @@ -99,18 +104,28 @@ Options: --version display version info and exit. -c (ignored) - -C install only if different (preserve the last data modification time) + -C install only if different (preserve data modification time) -d create directories instead of installing files. -g GROUP $chgrpprog installed files to GROUP. -m MODE $chmodprog installed files to MODE. -o USER $chownprog installed files to USER. + -p pass -p to $cpprog. -s $stripprog installed files. + -S SUFFIX attempt to back up existing files, with suffix SUFFIX. -t DIRECTORY install into DIRECTORY. -T report an error if DSTFILE is a directory. Environment variables override the default commands: CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG + +By default, rm is invoked with -f; when overridden with RMPROG, +it's up to you to specify -f if you want it. + +If -S is not specified, no backups are attempted. + +Email bug reports to bug-automake@gnu.org. +Automake home page: https://www.gnu.org/software/automake/ " while test $# -ne 0; do @@ -137,8 +152,13 @@ while test $# -ne 0; do -o) chowncmd="$chownprog $2" shift;; + -p) cpprog="$cpprog -p";; + -s) stripcmd=$stripprog;; + -S) backupsuffix="$2" + shift;; + -t) is_target_a_directory=always dst_arg=$2 @@ -255,6 +275,10 @@ do dstdir=$dst test -d "$dstdir" dstdir_status=$? + # Don't chown directories that already exist. + if test $dstdir_status = 0; then + chowncmd="" + fi else # Waiting for this to be detected by the "$cpprog $src $dsttmp" command @@ -301,22 +325,6 @@ do if test $dstdir_status != 0; then case $posix_mkdir in '') - # Create intermediate dirs using mode 755 as modified by the umask. - # This is like FreeBSD 'install' as of 1997-10-28. - umask=`umask` - case $stripcmd.$umask in - # Optimize common cases. - *[2367][2367]) mkdir_umask=$umask;; - .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; - - *[0-7]) - mkdir_umask=`expr $umask + 22 \ - - $umask % 100 % 40 + $umask % 20 \ - - $umask % 10 % 4 + $umask % 2 - `;; - *) mkdir_umask=$umask,go-w;; - esac - # With -d, create the new directory with the user-specified mode. # Otherwise, rely on $mkdir_umask. if test -n "$dir_arg"; then @@ -326,52 +334,49 @@ do fi posix_mkdir=false - case $umask in - *[123567][0-7][0-7]) - # POSIX mkdir -p sets u+wx bits regardless of umask, which - # is incompatible with FreeBSD 'install' when (umask & 300) != 0. - ;; - *) - # Note that $RANDOM variable is not portable (e.g. dash); Use it - # here however when possible just to lower collision chance. - tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + # The $RANDOM variable is not portable (e.g., dash). Use it + # here however when possible just to lower collision chance. + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ - trap 'ret=$?; rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null; exit $ret' 0 + trap ' + ret=$? + rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null + exit $ret + ' 0 - # Because "mkdir -p" follows existing symlinks and we likely work - # directly in world-writeable /tmp, make sure that the '$tmpdir' - # directory is successfully created first before we actually test - # 'mkdir -p' feature. - if (umask $mkdir_umask && - $mkdirprog $mkdir_mode "$tmpdir" && - exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1 - then - if test -z "$dir_arg" || { - # Check for POSIX incompatibilities with -m. - # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or - # other-writable bit of parent directory when it shouldn't. - # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. - test_tmpdir="$tmpdir/a" - ls_ld_tmpdir=`ls -ld "$test_tmpdir"` - case $ls_ld_tmpdir in - d????-?r-*) different_mode=700;; - d????-?--*) different_mode=755;; - *) false;; - esac && - $mkdirprog -m$different_mode -p -- "$test_tmpdir" && { - ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"` - test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" - } - } - then posix_mkdir=: - fi - rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" - else - # Remove any dirs left behind by ancient mkdir implementations. - rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null - fi - trap '' 0;; - esac;; + # Because "mkdir -p" follows existing symlinks and we likely work + # directly in world-writeable /tmp, make sure that the '$tmpdir' + # directory is successfully created first before we actually test + # 'mkdir -p'. + if (umask $mkdir_umask && + $mkdirprog $mkdir_mode "$tmpdir" && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + test_tmpdir="$tmpdir/a" + ls_ld_tmpdir=`ls -ld "$test_tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$test_tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" + else + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null + fi + trap '' 0;; esac if @@ -382,7 +387,7 @@ do then : else - # The umask is ridiculous, or mkdir does not conform to POSIX, + # mkdir does not conform to POSIX, # or it failed possibly due to a race condition. Create the # directory the slow way, step by step, checking for races as we go. @@ -411,7 +416,7 @@ do prefixes= else if $posix_mkdir; then - (umask=$mkdir_umask && + (umask $mkdir_umask && $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break # Don't fail if two instances are running concurrently. test -d "$prefix" || exit 1 @@ -451,7 +456,18 @@ do trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 # Copy the file name to the temp name. - (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + (umask $cp_umask && + { test -z "$stripcmd" || { + # Create $dsttmp read-write so that cp doesn't create it read-only, + # which would cause strip to fail. + if test -z "$doit"; then + : >"$dsttmp" # No need to fork-exec 'touch'. + else + $doit touch "$dsttmp" + fi + } + } && + $doit_exec $cpprog "$src" "$dsttmp") && # and set any options; do chmod last to preserve setuid bits. # @@ -477,6 +493,13 @@ do then rm -f "$dsttmp" else + # If $backupsuffix is set, and the file being installed + # already exists, attempt a backup. Don't worry if it fails, + # e.g., if mv doesn't support -f. + if test -n "$backupsuffix" && test -f "$dst"; then + $doit $mvcmd -f "$dst" "$dst$backupsuffix" 2>/dev/null + fi + # Rename the file to the real destination. $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || @@ -491,9 +514,9 @@ do # file should still install successfully. { test ! -f "$dst" || - $doit $rmcmd -f "$dst" 2>/dev/null || + $doit $rmcmd "$dst" 2>/dev/null || { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && - { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } + { $doit $rmcmd "$rmtmp" 2>/dev/null; :; } } || { echo "$0: cannot unlink or rename $dst" >&2 (exit 1); exit 1 diff --git a/missing b/missing index 625aeb11..1fe1611f 100755 --- a/missing +++ b/missing @@ -3,7 +3,7 @@ scriptversion=2018-03-07.03; # UTC -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2021 Free Software Foundation, Inc. # Originally written by Fran,cois Pinard , 1996. # This program is free software; you can redistribute it and/or modify diff --git a/packager/obs/home-tange/parallel/.osc/_files b/packager/obs/home-tange/parallel/.osc/_files index aa01fefa..2bf7dff4 100644 --- a/packager/obs/home-tange/parallel/.osc/_files +++ b/packager/obs/home-tange/parallel/.osc/_files @@ -1,8 +1,7 @@ - - - - - - - + + + + + + diff --git a/packager/obs/home-tange/parallel/.osc/parallel.spec b/packager/obs/home-tange/parallel/.osc/parallel.spec index 8e37d633..95e98fde 100644 --- a/packager/obs/home-tange/parallel/.osc/parallel.spec +++ b/packager/obs/home-tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20220522 +Version: 20220622 Release: 1.3 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/packager/releasescripts/updateversion b/packager/releasescripts/updateversion index 140b8b06..2bfc627d 100755 --- a/packager/releasescripts/updateversion +++ b/packager/releasescripts/updateversion @@ -15,7 +15,7 @@ updater() { export MON=`date +%b` export MONTH=`date +%B` TAG=$(echo $SPCTAG | perl -pe 's/ //g') - echo Tag=$TAG = "$SPCTAG" Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI + echo Tag=$TAG=\"$SPCTAG\" Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI export TITLE="GNU Parallel $YYYYMMDD ('$SPCTAG')" if [ -z "$DOINO" ] ; then diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 0edde44d..629cda57 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -395,7 +395,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index fd0cf7ab..e850aa0f 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -397,7 +397,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 19333228..b4ea2f11 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -395,7 +395,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index c11196de..00aa039c 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -378,7 +378,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index e1a5789f..c77186d3 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -380,7 +380,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index d0bfa7c6..21595a7b 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -395,7 +395,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 8b4ed1cd..9114da86 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -370,7 +370,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20220522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index d9ece34a..26173699 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20220522; +$Global::version = 20220622; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index daea1336..3ff20c84 100755 --- a/src/parallel +++ b/src/parallel @@ -2654,7 +2654,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20220607; + $Global::version = 20220622; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -5525,8 +5525,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2022, May 22). GNU Parallel 20220522 ('NATO').", - " Zenodo. https://doi.org/10.5281/zenodo.6570228", + " Tange, O. (2022, June 22). GNU Parallel 20220622 ('Bongbong').", + " Zenodo. https://doi.org/10.5281/zenodo.6682930", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice @@ -5558,8 +5558,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2022, May 22). GNU Parallel 20220522 ('NATO').", - " Zenodo. https://doi.org/10.5281/zenodo.6570228", + " Tange, O. (2022, June 22). GNU Parallel 20220622 ('Bongbong').", + " Zenodo. https://doi.org/10.5281/zenodo.6682930", "", # Before changing these line, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and @@ -5686,20 +5686,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2022_6570228,", + "\@software{tange_2022_6682930,", " author = {Tange, Ole},", - " title = {GNU Parallel 20220522 ('NATO')},", - " month = May,", + " title = {GNU Parallel 20220622 ('Bongbong')},", + " month = Jun,", " year = 2022,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.6570228},", - " url = {https://doi.org/10.5281/zenodo.6570228}", + " doi = {10.5281/zenodo.6682930},", + " url = {https://doi.org/10.5281/zenodo.6682930}", "}", "", - "(Feel free to use \\nocite{tange_2022_6570228})", + "(Feel free to use \\nocite{tange_2022_6682930})", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and diff --git a/src/parallel.pod b/src/parallel.pod index dc77a065..4b4e4936 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -748,7 +748,7 @@ B<--transfer>, B<--transferfile> or B<--return>. See also: B<--basefile> B<--transfer> B<--transferfile> B<--sshlogin> B<--return> -=item B<--color> (alpha testing) +=item B<--color> (beta testing) Colour output. @@ -1569,9 +1569,9 @@ Example: See also: B<--line-buffer> -=item B<--line-buffer> (alpha testing) +=item B<--line-buffer> (beta testing) -=item B<--lb> (alpha testing) +=item B<--lb> (beta testing) Buffer output on line basis. @@ -1811,9 +1811,9 @@ there will be several job sequence 1. See also: B<--nonall> B<--sshlogin> -=item B<--open-tty> (alpha testing) +=item B<--open-tty> (beta testing) -=item B<-o> (alpha testing) +=item B<-o> (beta testing) Open terminal tty. @@ -1912,7 +1912,7 @@ called with B<--sshlogin>). See also: B<--profile> -=item B<--plus> +=item B<--plus> (alpha testing) Add more replacement strings. @@ -1963,7 +1963,7 @@ inspired by bash's parameter expansion: See also: B<--rpl> B<{}> -=item B<--process-slot-var> I (alpha testing) +=item B<--process-slot-var> I (beta testing) Set the environment variable I to the jobslot number-1. @@ -2846,7 +2846,7 @@ B<--shebang-wrap> must be set as the first option. See also: B<--shebang> -=item B<--shell-completion> I (alpha testing) +=item B<--shell-completion> I (beta testing) Generate shell completion code for interactive shells. @@ -2995,13 +2995,13 @@ I is in seconds, but can be postfixed with s, m, h, or d. See also: TIME POSTFIXES B<--sshlogin> B<--delay> -=item B<--sshlogin> I<[@hostgroups/][ncpus/]sshlogin[,[@hostgroups/][ncpus/]sshlogin[,...]]> +=item B<--sshlogin> I<[@hostgroups/][ncpus/]sshlogin[,[@hostgroups/][ncpus/]sshlogin[,...]]> (alpha testing) -=item B<--sshlogin> I<@hostgroup> +=item B<--sshlogin> I<@hostgroup> (alpha testing) -=item B<-S> I<[@hostgroups/][ncpus/]sshlogin[,[@hostgroups/][ncpus/]sshlogin[,...]]> +=item B<-S> I<[@hostgroups/][ncpus/]sshlogin[,[@hostgroups/][ncpus/]sshlogin[,...]]> (alpha testing) -=item B<-S> I<@hostgroup> +=item B<-S> I<@hostgroup> (alpha testing) Distribute jobs to remote computers. diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 0fe78494..6e7e780f 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -3436,6 +3436,81 @@ https://github.com/rudymatela/evenmoreutils/blob/master/man/p.1.adoc (Last checked: 2022-04) +=head2 DIFFERENCES BETWEEN senechal AND GNU Parallel + +Summary (see legend above): + +=over + +=item I1 - - - - - - + +=item M1 - M3 - - M6 + +=item O1 - O3 O4 - - - x x - + +=item E1 - - - - - - + +=item - - - - - - - - - + +=item - - + +=back + +B only starts the first job after reading the last job, and +output from the first job is only printed after the last job finishes. + +1 byte of output requites 3.5 bytes of RAM. + +This makes it impossible to have a total output bigger than the +virtual memory. + +Even though output is kept in RAM outputing is quite slow: 30 MB/s. + +Output larger than 4 GB causes random problems - it looks like a race +condition. + +This: + + echo 1 | seneschal --prefix='yes `seq 1000`|head -c 1G' >/dev/null + +takes 4100(!) CPU seconds to run on a 64C64T server, but only 140 CPU +seconds on a 4C8T laptop. So it looks like B wastes a lot +of CPU time coordinating the CPUs. + +Compare this to: + + echo 1 | time -v parallel -N0 'yes `seq 1000`|head -c 1G' >/dev/null + +which takes 3-8 CPU seconds. + +=head3 EXAMPLES FROM seneschal README.md + + 1$ echo $REPOS | seneschal --prefix="cd {} && git pull" + + # If $REPOS is newline separated + 1$ echo "$REPOS" | parallel -k "cd {} && git pull" + # If $REPOS is space separated + 1$ echo -n "$REPOS" | parallel -d' ' -k "cd {} && git pull" + + COMMANDS="pwd + sleep 5 && echo boom + echo Howdy + whoami" + + 2$ echo "$COMMANDS" | seneschal --debug + + 2$ echo "$COMMANDS" | parallel -k -v + + 3$ ls -1 | seneschal --prefix="pushd {}; git pull; popd;" + + 3$ ls -1 | parallel -k "pushd {}; git pull; popd;" + # Or if current dir also contains files: + 3$ parallel -k "pushd {}; git pull; popd;" ::: */ + +https://github.com/TheWizardTower/seneschal +(Last checked: 2022-06) + + =head2 Todo http://code.google.com/p/push/ (cannot compile) @@ -3467,6 +3542,7 @@ https://vicerveza.homeunix.net/~viric/soft/ts/ https://github.com/chapmanjacobd/que + =head1 TESTING OTHER TOOLS There are certain issues that are very common on parallelizing diff --git a/src/parsort b/src/parsort index d37aad40..16f1ca99 100755 --- a/src/parsort +++ b/src/parsort @@ -122,7 +122,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20220522; +$Global::version = 20220622; if($opt::version) { version(); exit 0; } @Global::sortoptions = grep { ! /^-D$/ } shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); diff --git a/src/sql b/src/sql index 2781e4fb..4dfb669b 100755 --- a/src/sql +++ b/src/sql @@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20220522; + $Global::version = 20220622; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index a309775f..d0bae1d3 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -35,7 +35,7 @@ install_packages() { # Shells # (csh = bsd-csh that is broken) shell_pkgs="ash dash fdclone fish fizsh ksh ksh93u+m ksh93 mksh posh rc rush sash" - shell_pkgs="$shell_pkgs tcsh yash zsh" + shell_pkgs="$shell_pkgs tcsh yash zsh busybox-static" # Databases database_pkgs="postgresql mysql-server sqlite" @@ -187,7 +187,7 @@ create_shell_logins() { echo '# (Re-)create user' # Racecondition: if multiple adds a group it will the same group ID - shellsplus | + (shellsplus; echo withpassword) | parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user || (echo Creation failed: $?; false) } diff --git a/testsuite/wanted-results/parallel-local-300s b/testsuite/wanted-results/parallel-local-300s index 0f2c381a..c4f4f39d 100644 --- a/testsuite/wanted-results/parallel-local-300s +++ b/testsuite/wanted-results/parallel-local-300s @@ -234,7 +234,16 @@ par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000/sr par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Nothing to be done for 'install-exec-am'. -par_test_build_and_install make[0]: Nothing to be done for 'install-data-am'. +par_test_build_and_install make install-data-hook +par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' +par_test_build_and_install mkdir -p /tmp/parallel-install/share/bash-completion/completions +par_test_build_and_install mkdir -p /tmp/parallel-install/share/zsh/site-functions +par_test_build_and_install parallel --shell-completion bash > /tmp/parallel-install/share/bash-completion/completions/parallel +par_test_build_and_install echo '#compdef parallel' > /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install echo '(( $+functions[_comp_parallel] )) ||' >> /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install echo ' eval "$(parallel --shell-completion auto)" &&' >> /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install echo ' _comp_parallel' >> /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' @@ -715,7 +724,16 @@ par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000/sr par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Nothing to be done for 'install-exec-am'. -par_test_build_and_install make[0]: Nothing to be done for 'install-data-am'. +par_test_build_and_install make install-data-hook +par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' +par_test_build_and_install mkdir -p /tmp/parallel-install/share/bash-completion/completions +par_test_build_and_install mkdir -p /tmp/parallel-install/share/zsh/site-functions +par_test_build_and_install parallel --shell-completion bash > /tmp/parallel-install/share/bash-completion/completions/parallel +par_test_build_and_install echo '#compdef parallel' > /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install echo '(( $+functions[_comp_parallel] )) ||' >> /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install echo ' eval "$(parallel --shell-completion auto)" &&' >> /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install echo ' _comp_parallel' >> /tmp/parallel-install/share/zsh/site-functions/_parallel +par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000' par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000'