diff --git a/NEWS b/NEWS index ca3f087d..04ae3b33 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,20 @@ +20231222 + +New in this release: + +* --combine-exec combines GNU Parallel with any executable. + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* The first GNU Parallel video that seems to be completely AI + generated: https://www.youtube.com/watch?v=Ie8eAhlEHOg + +* GNU Parallel and “fail immediately if any fails” problem + https://antiarchitect.medium.com/gnu-parallel-and-fail-immediately-if-any-fails-problem-481ddea690ba + + 20231122 New in this release: diff --git a/README b/README index 84cc19c8..ddb3d1c5 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20231122.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20231122.tar.bz2.sig - gpg parallel-20231122.tar.bz2.sig - bzip2 -dc parallel-20231122.tar.bz2 | tar xvf - - cd parallel-20231122 + wget https://ftpmirror.gnu.org/parallel/parallel-20231222.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20231222.tar.bz2.sig + gpg parallel-20231222.tar.bz2.sig + bzip2 -dc parallel-20231222.tar.bz2 | tar xvf - + cd parallel-20231222 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20231122.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20231122.tar.bz2.sig - gpg parallel-20231122.tar.bz2.sig - bzip2 -dc parallel-20231122.tar.bz2 | tar xvf - - cd parallel-20231122 + wget https://ftpmirror.gnu.org/parallel/parallel-20231222.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20231222.tar.bz2.sig + gpg parallel-20231222.tar.bz2.sig + bzip2 -dc parallel-20231222.tar.bz2 | tar xvf - + cd parallel-20231222 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2023, November 22). GNU Parallel 20231122 ('Grindavík'). - Zenodo. https://doi.org/10.5281/zenodo.10199085 + Tange, O. (2023, December 22). GNU Parallel 20231222 ('Sundhnúkagígur'). + Zenodo. https://doi.org/10.5281/zenodo.10428184 Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. diff --git a/configure b/configure index 427b788f..a28ffab2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for parallel 20231122. +# Generated by GNU Autoconf 2.71 for parallel 20231222. # # Report bugs to . # @@ -610,8 +610,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20231122' -PACKAGE_STRING='parallel 20231122' +PACKAGE_VERSION='20231222' +PACKAGE_STRING='parallel 20231222' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1246,7 +1246,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20231122 to adapt to many kinds of systems. +\`configure' configures parallel 20231222 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1313,7 +1313,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20231122:";; + short | recursive ) echo "Configuration of parallel 20231222:";; esac cat <<\_ACEOF @@ -1390,7 +1390,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20231122 +parallel configure 20231222 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -1427,7 +1427,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20231122, which was +It was created by parallel $as_me 20231222, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2379,7 +2379,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20231122' + VERSION='20231222' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -3026,7 +3026,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20231122, which was +This file was extended by parallel $as_me 20231222, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3090,7 +3090,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -parallel config.status 20231122 +parallel config.status 20231222 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index fb358323..eed9d11b 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel],[20231122],[bug-parallel@gnu.org]) +AC_INIT([parallel],[20231222],[bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 6ddcc77a..613425c1 100644 --- a/doc/haikus +++ b/doc/haikus @@ -15,6 +15,23 @@ Output is in sync write a haiku about gnu parallel and the :::+ option + Stop paralyzing start parallelizing + -- @harshgandhi100@YouTube + + GNU Parallel and Tmux are the two tools I rely on daily for my work. Indispensable. + -- @mitchelvalentino1569@YouTube + + The most glorious 15,000 lines of Perl ever written. + -- @nibblrrr7124@YouTube + + + parallel is frickin great for launching jobs on multiple + machines. Ansible and Jenkins and others may be good too but I was + able to jump right in with parallel. + -- dwhite21787@reddit + + Big fan of GNU parallel! + -- lovelyloafers@reddit I’m a big fan of GNU parallel! -- Scott Cain @scottjcain@twitter @@ -213,6 +230,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + Parallel is so damn good! You’ve got to use it. + -- @ThePrimeTimeagen@youtube.com + Got around to using GNU parallel for the first time from a suggestion by @jdwasmuth ... now I'm wishing I started using this years ago -- Stefan Gavriliuc @GavriliucStefan@twitter diff --git a/doc/release_new_version b/doc/release_new_version index 2ce23368..3a955c69 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -38,7 +38,7 @@ make short https://zenodo.org/deposit/new Do you already have a DOI for this upload? No -(Reserve DOI) +(Get a DOI now!) Resource type: Software GNU Parallel 20xxxx22 ('Xxxx') Add creator: Person @@ -54,8 +54,7 @@ gpl [Save] -10.5281/zenodo.4381888 -DOINO=3956817 +DOI=10.5281/zenodo.4381888 SPCTAG='Privacy Shield' # Update version @@ -274,27 +273,26 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20231122 ('Perry<<>>') released <<[stable]>> +Subject: GNU Parallel 20231222 ('Sundhnúkagígur') released -GNU Parallel 20231122 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20231222 ('Sundhnúkagígur') has been released. It is available for download at: lbry://@GnuParallel:4 Quote of the month: - Got around to using GNU parallel for the first time from a suggestion by @jdwasmuth ... now I'm wishing I started using this years ago - -- Stefan Gavriliuc @GavriliucStefan@twitter - + Parallel is so damn good! You’ve got to use it. + -- @ThePrimeTimeagen@youtube.com + New in this release: -* -a file1 -a +file2 will link file2 to file1 similar to ::::+ - -* --bar shows total time when all jobs are done. +* --combine-exec combines GNU Parallel with any executable. * Bug fixes and man page updates. News about GNU Parallel: -* Using GNU Parallel in bash scripts to optimize python processes - https://climate-cms.org/posts/2023-11-02-gnu-parallel.html +* The first video that seems to be completely AI generated: https://www.youtube.com/watch?v=Ie8eAhlEHOg + +* GNU Parallel and “fail immediately if any fails” problem https://antiarchitect.medium.com/gnu-parallel-and-fail-immediately-if-any-fails-problem-481ddea690ba GNU Parallel - For people who live life in the parallel lane. diff --git a/packager/obs/README b/packager/obs/README index 6b10bdf3..b96cb6e5 100644 --- a/packager/obs/README +++ b/packager/obs/README @@ -1,4 +1,6 @@ make -Edit home\:tange/parallel/parallel.spec -Then go to https://build.opensuse.org/package/files?package=parallel&project=home%3Atange -and add all the files +Check that the .spec looks ok: home\:tange/parallel/parallel.spec + +Then go to +https://build.opensuse.org/package/files?package=parallel&project=home%3Atange +for status. diff --git a/packager/obs/home-tange/parallel/.osc/_files b/packager/obs/home-tange/parallel/.osc/_files index 531b6386..9fd9554b 100644 --- a/packager/obs/home-tange/parallel/.osc/_files +++ b/packager/obs/home-tange/parallel/.osc/_files @@ -1,8 +1,8 @@ - + - - - - - + + + + + diff --git a/packager/obs/home-tange/parallel/.osc/parallel.spec b/packager/obs/home-tange/parallel/.osc/parallel.spec index 80b3e487..b69804d4 100644 --- a/packager/obs/home-tange/parallel/.osc/parallel.spec +++ b/packager/obs/home-tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20231122 +Version: 20231222 Release: 2.1 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/packager/releasescripts/updateversion b/packager/releasescripts/updateversion index 2bfc627d..8bd7cc9a 100755 --- a/packager/releasescripts/updateversion +++ b/packager/releasescripts/updateversion @@ -3,25 +3,31 @@ # SPDX-FileCopyrightText: 2021-2022 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. # SPDX-License-Identifier: GPL-3.0-or-later +# Set DOI and SPCTAG then run this + updater() { - export DOINO + export DOI export TAG export SPCTAG export YYYYMMDD=`yyyymmdd` - export DOI=10.5281/zenodo.$DOINO export YYYY=${YYYYMMDD:0:4} export YYYYMM=${YYYYMMDD:0:6} export YYYYMMDD=${YYYYMM}22 export MON=`date +%b` export MONTH=`date +%B` + # Space Tag => SpaceTag TAG=$(echo $SPCTAG | perl -pe 's/ //g') - echo Tag=$TAG=\"$SPCTAG\" Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI + # DOI=10.5281/zenodo. + # DOINO=10428184 + DOINO=$(echo $DOI | perl -pe 's/.*\.//') + export DOINO + echo Tag=$TAG=\"$SPCTAG\" Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI DOINO=$DOINO export TITLE="GNU Parallel $YYYYMMDD ('$SPCTAG')" - if [ -z "$DOINO" ] ; then - echo '*** Set DOINO and try again ***' - echo 'https://zenodo.org/deposit/new (Reserve DOI)' - echo '[Save]' + if [ -z "$DOI" ] ; then + echo '*** Set DOI and try again ***' + echo 'https://zenodo.org/deposit/new (Get a DOI now!)' + echo '[Save draft]' cat .last-doitag.txt echo . .last-doitag.txt return @@ -32,11 +38,12 @@ updater() { echo . .last-doitag.txt return fi - echo "export DOINO=$DOINO" > .last-doitag.txt + echo "export DOI=$DOI" > .last-doitag.txt + echo "export DOINO=$DOINO" >> .last-doitag.txt echo "export TAG='$TAG'" >> .last-doitag.txt echo "export SPCTAG='$SPCTAG'" >> .last-doitag.txt echo "export YYYYMMDD=$YYYYMMDD" >> .last-doitag.txt - echo 'echo "$DOINO $YYYYMMDD ($TAG)"' >> .last-doitag.txt + echo 'echo "$DOINO $DOI $YYYYMMDD ($TAG)"' >> .last-doitag.txt perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" configure.ac perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 47faf062..41515f2e 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -393,7 +393,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 8f69f993..108a0114 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -395,7 +395,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 599bdb8e..0be830c4 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -393,7 +393,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index cd56c55f..9c26405d 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -376,7 +376,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 5457697d..58fcea70 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -378,7 +378,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index b9dbc88d..b42cb917 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -393,7 +393,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 0385f611..75ba8542 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -368,7 +368,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20231122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20231222 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index 716e46f5..3c81c4f6 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20231122; +$Global::version = 20231222; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 2d111dd9..37b8d9d2 100755 --- a/src/parallel +++ b/src/parallel @@ -2794,7 +2794,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20231122; + $Global::version = 20231222; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -5810,8 +5810,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2023, November 22). GNU Parallel 20231122 ('Grindavík').", - " Zenodo. https://doi.org/10.5281/zenodo.10199085", + " Tange, O. (2023, December 22). GNU Parallel 20231222 ('Sundhnúkagígur').", + " Zenodo. https://doi.org/10.5281/zenodo.10428184", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice @@ -5843,8 +5843,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2023, November 22). GNU Parallel 20231122 ('Grindavík').", - " Zenodo. https://doi.org/10.5281/zenodo.10199085", + " Tange, O. (2023, December 22). GNU Parallel 20231222 ('Sundhnúkagígur').", + " Zenodo. https://doi.org/10.5281/zenodo.10428184", "", # Before changing these line, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and @@ -5971,20 +5971,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2023_10199085,", + "\@software{tange_2023_10428184,", " author = {Tange, Ole},", - " title = {GNU Parallel 20231122 ('Grindavík')},", - " month = Nov,", + " title = {GNU Parallel 20231222 ('Sundhnúkagígur')},", + " month = Dec,", " year = 2023,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.10199085},", - " url = {https://doi.org/10.5281/zenodo.10199085}", + " doi = {10.5281/zenodo.10428184},", + " url = {https://doi.org/10.5281/zenodo.10428184}", "}", "", - "(Feel free to use \\nocite{tange_2023_10199085})", + "(Feel free to use \\nocite{tange_2023_10428184})", "", # Before changing these lines, please read # https://www.gnu.org/software/parallel/parallel_design.html#citation-notice and diff --git a/src/parallel.pod b/src/parallel.pod index 629d5ef9..d88d8df0 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1799,10 +1799,10 @@ If the available memory falls below 2 * I, GNU B will suspend some of the running jobs. If the available memory falls below I, only one job will be running. -If a single job takes up at most I RAM, all jobs will complete -without running out of memory. If you have swap available, you can -usually lower I to around half the size of a single job - with -the slight risk of swapping a little. +If a single job fits in the given size, all jobs will complete without +running out of memory. If you have swap available, you can usually +lower I to around half the size of a single job - with the slight +risk of swapping a little. Jobs will be resumed when more RAM is available - typically when the oldest job completes. diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index bb7f2b7c..a217cdf8 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -4276,6 +4276,87 @@ https://github.com/SergioBenitez/sp (Last checked: 2023-10) +=head2 DIFFERENCES BETWEEN repeater AND GNU Parallel + +Summary (see legend above): + +=over + +=item - - - - - - - + +=item - - - - - - + +=item - O2 O3 N/A - O6 - x x ?O10 + +=item E1 - - - E5 - - + +=item - - - - - - - - - + +=item - - + +=back + +B runs the same job repeatedly. In other words: It does not +read arguments, thus is it an alternative for GNU B for only +quite limited applications. + +B has an overhead of around 0.23 ms/job. Compared to GNU +B's 2-3 ms this is fast. Compared to B's 0.05 +ms/job it is slow. + +=head3 Memory use and run time for large output + +Output takes O(n^2) time for output of size n. 10 MB takes ~1 second, +30 MB takes ~7 seconds, 100 MB takes ~60 seconds, 300 MB takes ~480 +seconds, 1000 GB takes + +100 MB of output takes around 1 GB of RAM. + + # Run time = 15 sec + # Memory use = 20 MB + # Output = 1 GB per job + \time -v parallel -j1 seq ::: 120000000 120000000 >/dev/null + + # Run time = 4.7 sec + # Memory use = 95 MB + # Output = 8 MB per job + \time -v repeater -w 1 -n 2 -reportFile ./run_output seq 1200000 >/dev/null + + # Run time = 42 sec + # Memory use = 277 MB + # Output = 27 MB per job + \time -v repeater -w 1 -n 2 -reportFile ./run_output seq 3600000 >/dev/null + + # Run time = 530 sec + # Memory use = 1000 MB + # Output = 97 MB per job + \time -v repeater -w 1 -n 2 -reportFile ./run_output seq 12000000 >/dev/null + + # Run time = 2h41m + # Memory use = 8.6 GB + # Output = 1 GB per job + \time -v repeater -w 1 -n 2 -reportFile ./run_output seq 120000000 >/dev/null + +For even just moderate sized outputs GNU B will be faster +and use less memory. + + +=head3 EXAMPLES + + 1$ repeater -n 100 -w 10 -reportFile ./run_output + -output REPORT_FILE -progress BOTH curl example.com + + 1$ seq 100 | parallel --joblog run.log --eta curl example.com > output + + 2$ repeater -n 100 -increment -progress HIDDEN -reportFile foo + echo "this is increment: " INC + 2$ seq 100 | parallel echo {} + 2$ seq 100 | parallel echo '{= $_ = ++$myvar =}' + +https://github.com/baalimago/repeater +(Last checked: 2023-12) + + =head2 Todo https://github.com/justanhduc/task-spooler diff --git a/src/parallel_examples.pod b/src/parallel_examples.pod index 5f800154..f0bccf48 100644 --- a/src/parallel_examples.pod +++ b/src/parallel_examples.pod @@ -1867,7 +1867,7 @@ If you have downloaded source and tried compiling it, you may have seen: Often it is not obvious which package you should install to get that file. Debian has `apt-file` to search for a file. `tracefile` from -https://gitlab.com/ole.tange/tangetools can tell which files a program +https://codeberg.org/tange/tangetools can tell which files a program tried to access. In this case we are interested in one of the last files: diff --git a/src/parsort b/src/parsort index b4cf7e92..8ed6a6fd 100755 --- a/src/parsort +++ b/src/parsort @@ -137,7 +137,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20231122; +$Global::version = 20231222; if($opt::version) { version(); exit 0; } # Remove -D and --parallel=N my @s = (grep { ! /^-D$|^--parallel=\S+$/ } diff --git a/src/sql b/src/sql index f429963f..74eae8e7 100755 --- a/src/sql +++ b/src/sql @@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20231122; + $Global::version = 20231222; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/wanted-results/parallel-local-300s b/testsuite/wanted-results/parallel-local-300s index dedd8e34..4080ba8e 100644 --- a/testsuite/wanted-results/parallel-local-300s +++ b/testsuite/wanted-results/parallel-local-300s @@ -315,6 +315,10 @@ par_test_build_and_install am__remove_distdir=: am__skip_length_check=: am_ par_test_build_and_install make[0]: Entering directory '~/privat/parallel/src' par_test_build_and_install make distdir-am par_test_build_and_install make[0]: Entering directory '~/privat/parallel/src' +par_test_build_and_install pod2pdf --output-file "."/parallel_alternatives.pdf "."/parallel_alternatives.pod --title "GNU Parallel alternatives" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_alternatives.pdf par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src' par_test_build_and_install make[0]: Leaving directory '~/privat/parallel/src' par_test_build_and_install test -n " \ @@ -345,6 +349,54 @@ par_test_build_and_install config.status: creating config.h par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000' par_test_build_and_install Making install in src par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000/src' +par_test_build_and_install pod2pdf --output-file "."/parallel.pdf "."/parallel.pod --title "GNU Parallel" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel.pdf +par_test_build_and_install pod2pdf --output-file "."/env_parallel.pdf "."/env_parallel.pod --title "GNU Parallel with environment" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old env_parallel.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old env_parallel.pdf +par_test_build_and_install pod2pdf --output-file "."/sem.pdf "."/sem.pod --title "GNU sem" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old sem.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old sem.pdf +par_test_build_and_install pod2pdf --output-file "."/sql.pdf "."/sql --title "GNU SQL" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old sql.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old sql.pdf +par_test_build_and_install pod2pdf --output-file "."/niceload.pdf "."/niceload.pod --title "GNU niceload" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old niceload.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old niceload.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_examples.pdf "."/parallel_examples.pod --title "GNU Parallel Examples" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_examples.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_examples.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_tutorial.pdf "."/parallel_tutorial.pod --title "GNU Parallel Tutorial" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_tutorial.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_tutorial.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_book.pdf "."/parallel_book.pod --title "GNU Parallel Book" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_book.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_book.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_design.pdf "."/parallel_design.pod --title "GNU Parallel Design" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_design.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_design.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_alternatives.pdf "."/parallel_alternatives.pod --title "GNU Parallel alternatives" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_alternatives.pdf +par_test_build_and_install pod2pdf --output-file "."/parcat.pdf "."/parcat.pod --title "GNU parcat" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parcat.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parcat.pdf +par_test_build_and_install pod2pdf --output-file "."/parset.pdf "."/parset.pod --title "GNU parset" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parset.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parset.pdf par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000/src' par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/bin' par_test_build_and_install /bin/install -c parallel sql niceload parcat parset parsort env_parallel env_parallel.ash env_parallel.bash env_parallel.csh env_parallel.dash env_parallel.fish env_parallel.ksh env_parallel.mksh env_parallel.pdksh env_parallel.sh env_parallel.tcsh env_parallel.zsh '/tmp/parallel-install/bin' @@ -353,6 +405,54 @@ par_test_build_and_install make[0]: Entering directory '/tmp/parallel-00000000/s par_test_build_and_install rm "/tmp/parallel-install/bin"/sem || true par_test_build_and_install ln -s parallel "/tmp/parallel-install/bin"/sem par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000/src' +par_test_build_and_install pod2pdf --output-file "."/parallel.pdf "."/parallel.pod --title "GNU Parallel" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel.pdf +par_test_build_and_install pod2pdf --output-file "."/env_parallel.pdf "."/env_parallel.pod --title "GNU Parallel with environment" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old env_parallel.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old env_parallel.pdf +par_test_build_and_install pod2pdf --output-file "."/sem.pdf "."/sem.pod --title "GNU sem" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old sem.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old sem.pdf +par_test_build_and_install pod2pdf --output-file "."/sql.pdf "."/sql --title "GNU SQL" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old sql.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old sql.pdf +par_test_build_and_install pod2pdf --output-file "."/niceload.pdf "."/niceload.pod --title "GNU niceload" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old niceload.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old niceload.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_examples.pdf "."/parallel_examples.pod --title "GNU Parallel Examples" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_examples.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_examples.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_tutorial.pdf "."/parallel_tutorial.pod --title "GNU Parallel Tutorial" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_tutorial.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_tutorial.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_book.pdf "."/parallel_book.pod --title "GNU Parallel Book" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_book.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_book.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_design.pdf "."/parallel_design.pod --title "GNU Parallel Design" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_design.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_design.pdf +par_test_build_and_install pod2pdf --output-file "."/parallel_alternatives.pdf "."/parallel_alternatives.pod --title "GNU Parallel alternatives" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parallel_alternatives.pdf +par_test_build_and_install pod2pdf --output-file "."/parcat.pdf "."/parcat.pod --title "GNU parcat" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parcat.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parcat.pdf +par_test_build_and_install pod2pdf --output-file "."/parset.pdf "."/parset.pod --title "GNU parset" \ +par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parset.pdf" +par_test_build_and_install /bin/bash: line 1: pod2pdf: command not found +par_test_build_and_install Warning: pod2pdf not found. Using old parset.pdf par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/doc/parallel' par_test_build_and_install /bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_examples.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parsort.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_examples.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parsort.texi parallel.rst env_parallel.rst sem.rst sql.rst niceload.rst parallel_examples.rst parallel_tutorial.rst parallel_book.rst parallel_design.rst parallel_alternatives.rst parcat.rst parset.rst parsort.rst parallel.pdf '/tmp/parallel-install/share/doc/parallel' par_test_build_and_install /bin/install -c -m 644 env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_examples.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf parsort.pdf parallel_cheat_bw.pdf parallel_options_map.pdf '/tmp/parallel-install/share/doc/parallel'