diff --git a/NEWS b/NEWS index d256cfb7..94404f03 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,37 @@ +20200522 + +New in this release: + +* While running a job $PARALLEL_JOBSLOT is the jobslot of the job. It + is equal to {%} unless the job is being retried. See {%} for + details. + +* While running a job $PARALLEL_SSHLOGIN is the sshlogin line with + number of cores removed. E.g. '4//usr/bin/specialssh user@host' + becomes: '/usr/bin/specialssh user@host' + +* While running a job $PARALLEL_SSHHOST is the host part of an + sshlogin line. E.g. '4//usr/bin/specialssh user@host' becomes: + 'host' + +* --plus activates the replacement strings {slot} = $PARALLEL_JOBSLOT, + {sshlogin} = $PARALLEL_SSHLOGIN, {host} = $PARALLEL_SSHHOST + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* Portable Batch System (PBS) & GNU Parallel - Running a Program + Multiple Times in Parallel + https://www.youtube.com/watch?v=6ccbWu6Befo + +* GNU Parallel przykÅ‚ad https://www.youtube.com/watch?v=gs_wG4Kt2G4 + +* Demo of LINUX APP - GNU PARALLEL - running multiple Gstreamer webcam + .sh scripts with only 1 command + https://www.youtube.com/watch?v=trQuA_wmWjg + + 20200422 This release celebrates GNU Parallel's 10 years as a GNU tool: diff --git a/README b/README index de8d24a6..2492bba9 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20200422.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20200422.tar.bz2.sig - gpg parallel-20200422.tar.bz2.sig - bzip2 -dc parallel-20200422.tar.bz2 | tar xvf - - cd parallel-20200422 + wget https://ftpmirror.gnu.org/parallel/parallel-20200522.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20200522.tar.bz2.sig + gpg parallel-20200522.tar.bz2.sig + bzip2 -dc parallel-20200522.tar.bz2 | tar xvf - + cd parallel-20200522 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20200422.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20200422.tar.bz2.sig - gpg parallel-20200422.tar.bz2.sig - bzip2 -dc parallel-20200422.tar.bz2 | tar xvf - - cd parallel-20200422 + wget https://ftpmirror.gnu.org/parallel/parallel-20200522.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20200522.tar.bz2.sig + gpg parallel-20200522.tar.bz2.sig + bzip2 -dc parallel-20200522.tar.bz2 | tar xvf - + cd parallel-20200522 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, - DOI https://doi.org/10.5281/zenodo.1146014 + Tange, O. (2020, May 22). GNU Parallel 20200522 ('Kraftwerk'). + Zenodo. https://doi.org/10.5281/zenodo.3840974 = New versions = diff --git a/configure b/configure index bf052562..bf03861f 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20200422. +# Generated by GNU Autoconf 2.69 for parallel 20200522. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20200422' -PACKAGE_STRING='parallel 20200422' +PACKAGE_VERSION='20200522' +PACKAGE_STRING='parallel 20200522' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20200422 to adapt to many kinds of systems. +\`configure' configures parallel 20200522 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20200422:";; + short | recursive ) echo "Configuration of parallel 20200522:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20200422 +parallel configure 20200522 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20200422, which was +It was created by parallel $as_me 20200522, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20200422' + VERSION='20200522' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20200422, which was +This file was extended by parallel $as_me 20200522, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20200422 +parallel config.status 20200522 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 1b558482..9469a878 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20200422], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20200522], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index b207706c..2a451206 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -12,19 +12,58 @@ Modified => alpha == Update version == +Get DOI: +https://zenodo.org/deposit/new (Reserve DOI) + configure.ac: AC_INIT([parallel], [20100422], [bug-parallel@gnu.org]) src/parallel: $Global::version = 20100422; README: parallel-20130222 +DOINO=3840974 +TAG=Kraftwerk YYYYMMDD=$(echo `yyyymmdd`-1 | bc) YYYYMMDD=$(echo `yyyymmdd`+1 | bc) YYYYMMDD=`yyyymmdd` -echo $YYYYMMDD -perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" configure.ac -perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parallel -perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql -perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/niceload -perl -i -pe "s/parallel-20\d\d\d\d\d\d/parallel-$YYYYMMDD/" README + +updater() { + export DOINO + export TAG + export YYYYMMDD + export DOI=10.5281/zenodo.$DOINO + export YYYY=${YYYYMMDD:0:4} + export MON=`date +%b` + export MONTH=`date +%B` + echo Tag=$TAG Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI + export TITLE="GNU Parallel $YYYYMMDD ('$TAG')" + + perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" configure.ac + perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql + perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/niceload + perl -i -pe "s/parallel-20\d\d\d\d\d\d/parallel-$YYYYMMDD/" README + perl -i -pe ' + # Update version 20209999 + /version/ and s/20\d\d\d\d\d\d/$ENV{YYYYMMDD}/; + # Update: 10.5281/zenodo.1146014 + s:10.5281/zenodo.\d+:$ENV{DOI}:; + # Update "@software{tange_2015_16303," + s:tange_\d+_\d+:tange_$ENV{YYYY}_$ENV{DOINO}:; + # Update month = mar, + s/(month\s+=\s+)\S+,",/$1$ENV{MON},",/; + # Update title = {GNU Parallel 20200522 ('Kraftwerk')},", + / title\s+= / and s/\{.*\}/{$ENV{TITLE}}/; + # Tange, O. (2020, May 22). GNU Parallel 20200522 ('Kraftwerk'). + s/(Tange, O. .).*(.. )(GNU.*[)])/$1$ENV{YYYY}, $ENV{MONTH} 22$2$ENV{TITLE}/; + ' src/parallel README + ( + ppar --help + ppar --citation + grep -i 'zenodo|tange' README + ) 2>&1 | grep -E '^ |^}|tange' + mv ~/.parallel/will-cite ~/.parallel/will-cite. + ppar ::: true + mv ~/.parallel/will-cite. ~/.parallel/will-cite +} +updater === Autoconf/automake === @@ -223,12 +262,23 @@ Quote of the month: New in this release: +* While running a job $PARALLEL_JOBSLOT is the jobslot of the job. It is equal to {%} unless the job is being retried. See {%} for details. + +* While running a job $PARALLEL_SSHLOGIN is the sshlogin line with number of cores removed. E.g. '4//usr/bin/specialssh user@host' becomes: '/usr/bin/specialssh user@host' + +* While running a job $PARALLEL_SSHHOST is the host part of an sshlogin line. E.g. '4//usr/bin/specialssh user@host' becomes: 'host' + +* --plus activates the replacement strings {slot} = $PARALLEL_JOBSLOT, {sshlogin} = $PARALLEL_SSHLOGIN, {host} = $PARALLEL_SSHHOST + * Bug fixes and man page updates. News about GNU Parallel: -* +* Portable Batch System (PBS) & GNU Parallel - Running a Program Multiple Times in Parallel https://www.youtube.com/watch?v=6ccbWu6Befo +* GNU Parallel przykÅ‚ad https://www.youtube.com/watch?v=gs_wG4Kt2G4 +* demo of LINUX APP - GNU PARALLEL - running multiple Gstreamer webcam .sh scripts with only 1 command https://www.youtube.com/watch?v=trQuA_wmWjg + Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html diff --git a/src/niceload b/src/niceload index ad298fa8..a6f3845e 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20200423; +$Global::version = 20200522; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 7cb1ce18..8b450ca6 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -114,11 +114,6 @@ E1 - - - E5 - - - -xjobs, prll, dxargs, mdm/middelman, xapply, paexec, ladon, jobflow, -ClusterSSH: TODO - Please file a bug-report if you know what features -they support (See REPORTING BUGS). - - =head2 DIFFERENCES BETWEEN xargs AND GNU Parallel Summary table (see legend above): @@ -520,9 +515,11 @@ using GNU B: 4$ parallel indent ::: *.c - 5$ find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' - + 5$ find ~ksb/bin -type f ! -perm -111 -print | \ + xapply -f -v 'chmod a+x' - - 5$ find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x + 5$ find ~ksb/bin -type f ! -perm -111 -print | \ + parallel -v chmod a+x 6$ find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' - @@ -532,15 +529,16 @@ using GNU B: 7$ find ... | xapply -f -5 -i /dev/tty 'vi' - - - - - - 7$ sh <(find ... |parallel -n5 echo vi) + 7$ sh <(find ... | parallel -n5 echo vi) - 7$ find ... |parallel -n5 -uj1 vi + 7$ find ... | parallel -n5 -uj1 vi 8$ xapply -fn "" /etc/passwd 8$ parallel -k echo < /etc/passwd - 9$ tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - - + 9$ tr ':' '\012' < /etc/passwd | \ + xapply -7 -nf 'chown %1 %6' - - - - - - - 9$ tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6} @@ -563,7 +561,7 @@ very much like GNU B. B does not run jobs in parallel. B does not use an argument separator (like B<:::>); instead the template must be the first argument. -=head3 EXAMPLES FROM IBM's KNOWLEDGE CENTER +=head3 EXAMPLES FROM IBM's KNOWLEDGE CENTER Here are the examples from IBM's Knowledge Center and the corresponding command using GNU B: @@ -649,65 +647,47 @@ https://github.com/cheusov/paexec =head2 DIFFERENCES BETWEEN map(sitaramc) AND GNU Parallel Summary table (see legend above): -I1 - - I4 - - - -M1 (M2) M3 M4 M5 - +I1 - - I4 - - (I7) +M1 (M2) M3 (M4) M5 M6 - O2 O3 - O5 - - N/A N/A O10 E1 - - - - - - - - - - - - - - - - - -(M2): Only if there is a single replacement string. +(I7): Only under special circumstances. See below. -B sees it as a feature to have less features and in doing so it -also handles corner cases incorrectly. A lot of GNU B's code -is to handle corner cases correctly on every platform, so you will not -get a nasty surprise if a user, for example, saves a file called: I +(M2+M4): Only if there is a single replacement string. -B's example showing how to deal with special characters fails on -special characters: +B rejects input with special characters: echo "The Cure" > My\ brother\'s\ 12\"\ records - ls | \ - map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' | - bc + ls | map 'echo %; wc %' It works with GNU B: - ls | \ - parallel \ - 'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc + ls | parallel 'echo {}; wc {}' -And you can even get the file name prepended: +Under some circumstances it also works with B: - ls | \ - parallel --tag \ - '(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc' + ls | map 'echo % works %' -B has no support for grouping. So this gives the wrong results -without any warnings: +But tiny changes make it reject the input with special characters: - parallel perl -e '\$a=\"1{}\"x10000000\;print\ \$a,\"\\n\"' '>' {} \ - ::: a b c d e f - ls -l a b c d e f - parallel -kP4 -n1 grep 1 > out.par ::: a b c d e f - map -p 4 'grep 1' a b c d e f > out.map-unbuf - map -p 4 'grep --line-buffered 1' a b c d e f > out.map-linebuf - map -p 1 'grep --line-buffered 1' a b c d e f > out.map-serial - ls -l out* - md5sum out* + ls | map 'echo % does not work "%"' -The documentation shows a workaround, but not only does that mix -stdout (standard output) with stderr (standard error) it also fails -completely for certain jobs (and may even be considered less readable): +This means that many UTF-8 characters will be rejected. This is by +design. From the web page: "As such, programs that I are doing their users a disservice." - parallel echo -n {} ::: 1 2 3 +B delays each job by 0.01 s. This can be emulated by using +B. - map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | \ - sort | cut -f2- -d: +B prints '+' on stderr when a job starts, and '-' when a job +finishes. This cannot be disabled. B has B<--bar> if you +need to see progress. -Bs replacement strings (% %D %B %E) can be simulated in GNU +B's replacement strings (% %D %B %E) can be simulated in GNU B by putting this in B<~/.parallel/config>: --rpl '%' @@ -730,25 +710,58 @@ context replace: map "echo 'BEGIN{'%'}END'" 1 2 3 -B requires Perl v5.10.0 making it harder to use on old systems. +B has no support for grouping. So this gives the wrong results: -To put a % in the command line B requires you to use %%: + parallel perl -e '\$a=\"1{}\"x10000000\;print\ \$a,\"\\n\"' '>' {} \ + ::: a b c d e f + ls -l a b c d e f + parallel -kP4 -n1 grep 1 ::: a b c d e f > out.par + map -n1 -p 4 'grep 1' a b c d e f > out.map-unbuf + map -n1 -p 4 'grep --line-buffered 1' a b c d e f > out.map-linebuf + map -n1 -p 1 'grep --line-buffered 1' a b c d e f > out.map-serial + ls -l out* + md5sum out* - seq 10 | map -n1 echo % +10%% {} +=head3 EXAMPLES FROM map's WEBSITE - seq 10 | parallel -I ,, echo ,, +10% {} +Here are the examples from B's web page with the equivalent using +GNU B: -GNU B has -I to specify another replacement string than -B<{}> if you use B<{}> in the command template. + 1$ ls *.gif | map convert % %B.png # default max-args: 1 -By design B is option incompatible with B, it does not -have remote job execution, a structured way of saving results, -multiple input sources, progress indicator, configurable record -delimiter (only field delimiter), logging of jobs run with possibility -to resume, keeping the output in the same order as input, --pipe -processing, and dynamically timeouts. + 1$ ls *.gif | parallel convert {} {.}.png -https://github.com/sitaramc/map (Last checked: 2020-04) + 2$ map "mkdir %B; tar -C %B -xf %" *.tgz # default max-args: 1 + + 2$ parallel 'mkdir {.}; tar -C {.} -xf {}' ::: *.tgz + + 3$ ls *.gif | map cp % /tmp # default max-args: 100 + + 3$ ls *.gif | parallel -X cp {} /tmp + + 4$ ls *.tar | map -n 1 tar -xf % + + 4$ ls *.tar | parallel tar -xf + + 5$ map "cp % /tmp" *.tgz + + 5$ parallel cp {} /tmp ::: *.tgz + + 6$ map "du -sm /home/%/mail" alice bob carol + + 6$ parallel "du -sm /home/{}/mail" ::: alice bob carol + or if you prefer running a single job with multiple args: + 6$ parallel -Xj1 "du -sm /home/{}/mail" ::: alice bob carol + + 7$ cat /etc/passwd | map -d: 'echo user %1 has shell %7' + + 7$ cat /etc/passwd | parallel --colsep : 'echo user {1} has shell {7}' + + 8$ export MAP_MAX_PROCS=$(( `nproc` / 2 )) + + 8$ export PARALLEL=-j50% + +https://github.com/sitaramc/map (Last checked: 2020-05) =head2 DIFFERENCES BETWEEN ladon AND GNU Parallel @@ -849,9 +862,11 @@ jobs. This can be emulated by GNU B using B's B: 3$ cat urls.txt | parallel -j32 wget {} - 4$ find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg + 4$ find . -name '*.bmp' | \ + jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg - 4$ find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg + 4$ find . -name '*.bmp' | \ + parallel -j8 bmp2jpeg {.}.bmp {.}.jpg https://github.com/rofl0r/jobflow @@ -877,9 +892,11 @@ Output to stderr (standard error) is changed if the command fails. 1$ seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}" - 2$ cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'" + 2$ cat t.txt | gargs --sep "\s+" \ + -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'" - 2$ cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'" + 2$ cat t.txt | parallel --colsep "\\s+" \ + -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'" https://github.com/brentp/gargs @@ -2633,7 +2650,7 @@ https://github.com/ExpectationMax/simple_gpu_scheduler seq 3 | parallel echo true >> gpu.queue # Add jobs echo "my_command_with | and stuff > logfile" >> gpu.queue - # Needed to flush output from completed jobs + # Needed to flush output from completed jobs seq 3 | parallel echo true >> gpu.queue diff --git a/src/sql b/src/sql index f510e0ee..51ff40d4 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20200423; + $Global::version = 20200522; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 56ac76cd..12451478 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -507,6 +507,15 @@ par_tmp_full() { stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy } +par_jobs_file() { + echo '### Test of -j filename - non-existent file' + stdout parallel -j no_such_file echo ::: 1 + + echo '### Test of -j filename' + echo 3 >/tmp/jobs_to_run1 + parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4 + # Should give 6 8 10 5 4 +} export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index e463ce3f..a60a6dc7 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -3317,4 +3317,6 @@ compgen -A function | grep par_ | LC_ALL=C sort -r | s/\d+ >= \d+/XXX >= XXX/; s/sh:? \d?\d\d:/sh: XXX:/; s/:\d?\d\d:/:XXX:/; - s/sh\[\d+\]/sh[XXX]/;' + s/sh\[\d+\]/sh[XXX]/; + s/.*(tange|zenodo).*//i; + ' diff --git a/testsuite/tests-to-run/parallel-local12.sh b/testsuite/tests-to-run/parallel-local12.sh index eab4e873..7548e4fc 100644 --- a/testsuite/tests-to-run/parallel-local12.sh +++ b/testsuite/tests-to-run/parallel-local12.sh @@ -22,7 +22,9 @@ cat >/tmp/parallel-script-for-script2 </tmp/parallel-script-for-script3 < ~/.parallel/runs-without-willing-to-cite diff --git a/testsuite/tests-to-run/parallel-local9.sh b/testsuite/tests-to-run/parallel-local9.sh index c80cbcc5..bbcd9684 100644 --- a/testsuite/tests-to-run/parallel-local9.sh +++ b/testsuite/tests-to-run/parallel-local9.sh @@ -100,14 +100,6 @@ echo "# --recend '' --files --halt-on-error" nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum -echo '### Test of -j filename - non-existent file'; - nice stdout parallel -j no_such_file echo ::: 1 - -echo '### Test of -j filename'; - echo 3 >/tmp/jobs_to_run1; - parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; - # Should give 6 8 10 5 4 - echo '### Test ::::' echo '### Change --arg-file-sep' $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 9cc1b032..ee426e3d 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -205,6 +205,51 @@ par_fifo_under_csh 1 par_fifo_under_csh 868832 par_fifo_under_csh 1 par_fifo_under_csh exit 22 +par_jobs_file ### Test of -j filename - non-existent file +par_jobs_file parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. +par_jobs_file Usage: +par_jobs_file +par_jobs_file parallel [options] [command [arguments]] < list_of_arguments +par_jobs_file parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))... +par_jobs_file cat ... | parallel --pipe [options] [command [arguments]] +par_jobs_file +par_jobs_file -j n Run n jobs in parallel +par_jobs_file -k Keep same order +par_jobs_file -X Multiple arguments with context replace +par_jobs_file --colsep regexp Split input on regexp for positional replacements +par_jobs_file {} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings +par_jobs_file {3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings +par_jobs_file With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = +par_jobs_file {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} +par_jobs_file +par_jobs_file -S sshlogin Example: foo@server.example.com +par_jobs_file --slf .. Use ~/.parallel/sshloginfile as the list of sshlogins +par_jobs_file --trc {}.bar Shorthand for --transfer --return {}.bar --cleanup +par_jobs_file --onall Run the given command with argument on all sshlogins +par_jobs_file --nonall Run the given command with no arguments on all sshlogins +par_jobs_file +par_jobs_file --pipe Split stdin (standard input) to multiple jobs. +par_jobs_file --recend str Record end separator for --pipe. +par_jobs_file --recstart str Record start separator for --pipe. +par_jobs_file +par_jobs_file See 'man parallel' for details +par_jobs_file +par_jobs_file Academic tradition requires you to cite works you base your article on. +par_jobs_file If you use programs that use GNU Parallel to process data for an article in a +par_jobs_file scientific publication, please cite: +par_jobs_file +par_jobs_file Tange, O. (2020, May 22). GNU Parallel 20200522 ('Kraftwerk'). +par_jobs_file Zenodo. https://doi.org/10.5281/zenodo.3840974 +par_jobs_file +par_jobs_file This helps funding further development; AND IT WON'T COST YOU A CENT. +par_jobs_file If you pay 10000 EUR you should feel free to use GNU Parallel without citing. +par_jobs_file +par_jobs_file ### Test of -j filename +par_jobs_file sleep 6 +par_jobs_file sleep 8 +par_jobs_file sleep 10 +par_jobs_file sleep 5 +par_jobs_file sleep 4 par_k ### Test -k par_k parallel: Warning: Only enough file handles to run X jobs in parallel. par_k parallel: Warning: Running 'parallel -jX -N X --pipe parallel -jX' or diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 513170f9..936998ff 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -445,8 +445,8 @@ par_tcsh_man Academic tradition requires you to cite works you base your article par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a par_tcsh_man par_tcsh_man scientific publication, please cite: par_tcsh_man par_tcsh_man par_tcsh_man -par_tcsh_man O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, par_tcsh_man -par_tcsh_man DOI https://doi.org/10.5281/zenodo.1146014 par_tcsh_man + + par_tcsh_man par_tcsh_man par_tcsh_man This helps funding further development; AND IT WON'T COST YOU A CENT. par_tcsh_man par_tcsh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing. par_tcsh_man @@ -1839,8 +1839,8 @@ par_csh_man par_csh_man par_csh_man par_csh_man {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} -par_csh_man DOI https://doi.org/10.5281/zenodo.1146014 -par_csh_man O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, + + par_csh_man ### From man env_parallel par_csh_man --colsep regexp Split input on regexp for positional replacements par_csh_man --nonall Run the given command with no arguments on all sshlogins diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 3b580721..98a6d5e5 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -4,8 +4,6 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, - DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. @@ -27,8 +25,6 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, - DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. @@ -39,22 +35,12 @@ https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice To silence this citation notice: run 'parallel --citation' once. 10 files to edit -[?2004h[?1049h[?1h=[?2004h[?12h[?12l[?25l"file1" [New File]½ [>c]10;?]11;?~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ 0,0-1All[?25h[?25lE173: 9 more files to edit[?2004h0,0-1All[?25h[?2004l[?2004l[?1l>[?1049lAcademic tradition requires you to cite works you base your article on. +Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: -@book{tange_ole_2018_1146014, - author = {Tange, Ole}, - title = {GNU Parallel 2018}, - publisher = {Ole Tange}, - month = Mar, - year = 2018, - ISBN = {9781387509881}, - doi = {10.5281/zenodo.1146014}, - url = {https://doi.org/10.5281/zenodo.1146014} } -(Feel free to use \nocite{tange_ole_2018_1146014}) This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. @@ -64,7 +50,6 @@ https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt -If you send a copy of your published article to tange@gnu.org, it will be mentioned in the release notes of next version of GNU Parallel. diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 8c9a3e92..13ba8568 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -126,53 +126,6 @@ echo "# --recend '' --files --halt-on-error" # --recend '' --files --halt-on-error nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - -echo '### Test of -j filename - non-existent file'; nice stdout parallel -j no_such_file echo ::: 1 -### Test of -j filename - non-existent file -parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. -Usage: - -parallel [options] [command [arguments]] < list_of_arguments -parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))... -cat ... | parallel --pipe [options] [command [arguments]] - --j n Run n jobs in parallel --k Keep same order --X Multiple arguments with context replace ---colsep regexp Split input on regexp for positional replacements -{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings -{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings -With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = - {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} - --S sshlogin Example: foo@server.example.com ---slf .. Use ~/.parallel/sshloginfile as the list of sshlogins ---trc {}.bar Shorthand for --transfer --return {}.bar --cleanup ---onall Run the given command with argument on all sshlogins ---nonall Run the given command with no arguments on all sshlogins - ---pipe Split stdin (standard input) to multiple jobs. ---recend str Record end separator for --pipe. ---recstart str Record start separator for --pipe. - -See 'man parallel' for details - -Academic tradition requires you to cite works you base your article on. -If you use programs that use GNU Parallel to process data for an article in a -scientific publication, please cite: - - O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, - DOI https://doi.org/10.5281/zenodo.1146014 - -This helps funding further development; AND IT WON'T COST YOU A CENT. -If you pay 10000 EUR you should feel free to use GNU Parallel without citing. - -echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 -### Test of -j filename -sleep 6 -sleep 8 -sleep 10 -sleep 5 -sleep 4 echo '### Test ::::' ### Test :::: echo '### Change --arg-file-sep'