diff --git a/Makefile.am b/Makefile.am index 1fa254b9..3f0df0ce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -168,4 +168,4 @@ pack_unpack_and_test_build: ./configure && make -j && sudo make -j install -EXTRA_DIST = CITATION CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt +EXTRA_DIST = CITATION CITATION.cff CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt diff --git a/Makefile.in b/Makefile.in index efd9da9f..052d57b1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -279,7 +279,7 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ SUBDIRS = src -EXTRA_DIST = CITATION CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt +EXTRA_DIST = CITATION CITATION.cff CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt all: config.h $(MAKE) $(AM_MAKEFLAGS) all-recursive diff --git a/doc/haikus b/doc/haikus index d1269668..1676c0a3 100644 --- a/doc/haikus +++ b/doc/haikus @@ -4,6 +4,14 @@ Quote of the month: + + GNU parallel すごいな、シェルスクリプトを make -jX みたいにして並列 + に走らせたいときに超便利 + -- びーる @debugordie@twitter + + GNU parallel いいよ、大量のデータファイルをスクリプトで処理する人とかにマジおすすめ + -- びーる @debugordie@twitter + Igual @GnuParallel te puede echar una mano. Te permite pensar y programar en serie pero ejecutar en paralelo usando cores de una máquina o máquinas remotas. Una vez que lo has usando te lamentas no diff --git a/doc/release_new_version b/doc/release_new_version index 7c0c7ec5..97fb57dc 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -254,32 +254,32 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20220122 ('20 years') released [stable] +Subject: GNU Parallel 20220222 ('Ukraine/Mali') released [stable] -GNU Parallel 20220122 ('20 years') [stable] has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20220222 ('<<>>') [stable] has been released. It is available for download at: lbry://@GnuParallel:4 No new functionality was introduced so this is a good candidate for a stable release. Quote of the month: - I'm going to make GNU parallel my new superpower this year. - -- Christian G. Warden @xn@twitter + <<>> New in this release: -* GNU Parallel turned 20 on the 2022-01-06 https://www.gnu.org/software/parallel/20th-birthday.html +* Encoding of remote commands has changed. This limits the length of remote commands to less than half. It also requires $REMOTE_TMPDIR to be writable. -* Bug fixes and man page updates. +* Time functions hh_mm_ss() hhmmss() hh_mm() hhmm() are now available for {= =}. + +<<>> + +* Bug fixes and man page updates: Many options now have a 'See also' section. News about GNU Parallel: -* Save Time with Modern Search Techniques https://www.eventbrite.com/e/ohio-information-security-forum-january-2022-monthly-meeting-tickets-240632737997 +https://blog.csdn.net/qq_38197157/article/details/112596389 +https://www.freelancer.com/projects/linux/gnu-parallel-processing-for-data/?ngsw-bypass=&w=f -* Bash "Fail immediately if any fails" https://morioh.com/p/a01b883dc57c - -* Re-encoding the EmacsConf videos with FFmpeg and GNU Parallel https://sachachua.com/blog/2021/12/re-encoding-the-emacsconf-videos-with-ffmpeg-and-gnu-parallel/ - -* Bash 平行 https://learntutorials.net/ja/bash/topic/10778/%E5%B9%B3%E8%A1%8C +<<>> Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 40063a11..6de04cd8 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 011a4670..f4e1fbc5 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 281649f4..9ab35ac5 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 6b379b46..2943deae 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -363,7 +363,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 523a419a..62ae8e0b 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -365,7 +365,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 2095802a..b8e81f3a 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -390,7 +390,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index e6c97904..c1267bd8 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -355,7 +355,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20220122 (GNU parallel `parallel --minversion 1`)" + echo "parset 20220123 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/parallel b/src/parallel index 60c956b2..a6a7d5f3 100755 --- a/src/parallel +++ b/src/parallel @@ -13052,6 +13052,14 @@ sub total_jobs() { # ISO8601 2038-01-19 ::strftime("%Y-%m-%d", localtime(time())); } + sub hh_mm_ss() { + # ISO8601 03:14:08 + ::strftime("%H:%M:%S", localtime(time())); + } + sub hh_mm() { + # ISO8601 03:14 + ::strftime("%H:%M", localtime(time())); + } sub yyyymmddhhmmss() { # ISO8601 20380119031408 ::strftime("%Y%m%d%H%M%S", localtime(time())); @@ -13064,6 +13072,14 @@ sub total_jobs() { # ISO8601 20380119 ::strftime("%Y%m%d", localtime(time())); } + sub hhmmss() { + # ISO8601 031408 + ::strftime("%H%M%S", localtime(time())); + } + sub hhmm() { + # ISO8601 0314 + ::strftime("%H%M", localtime(time())); + } sub replace($$$$) { # Calculates the corresponding value for a given perl expression diff --git a/src/parsort b/src/parsort index 5414ffd7..bc60af1e 100755 --- a/src/parsort +++ b/src/parsort @@ -122,7 +122,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20220122; +$Global::version = 20220123; if($opt::version) { version(); exit 0; } @Global::sortoptions = grep { ! /^-D$/ } shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); diff --git a/src/sql b/src/sql index 66a906f9..70d22194 100755 --- a/src/sql +++ b/src/sql @@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20220122; + $Global::version = 20220123; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-android.sh b/testsuite/tests-to-run/parallel-android.sh index 43761d2e..9e5fa1f6 100755 --- a/testsuite/tests-to-run/parallel-android.sh +++ b/testsuite/tests-to-run/parallel-android.sh @@ -9,7 +9,7 @@ torsocks bash <<'EOF' . `which env_parallel.bash` env_parallel --session -host=$(parallel -j0 --halt now,success=1 ssh {} echo {} ::: koditor huator fairtor 2>/dev/null) +host=$(parallel -j0 --halt now,success=1 ssh {} echo {} ::: kodi koditor huator fairtor 2>/dev/null) if [ -z "$host" ] ; then echo Error: no android host working else diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 7d3837eb..dbb927c3 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -60,7 +60,8 @@ EOF par_crnl() { echo '### Give a warning if input is DOS-ascii' - printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | stdout parallel -k echo {}a + printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | + stdout parallel -k 'sleep 0.01; echo {}a' echo This should give no warning because -d is set printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | parallel -k -d '\r\n' echo {}a echo This should give no warning because line2 has newline only @@ -649,6 +650,14 @@ par_tee() { seq 300000 | parallel 'grep {1} | LC_ALL=C wc {2}' ::: {1..5} ::: -l -c } +par_parset_tee() { + . $(which env_parallel.bash) + export PARALLEL='-k --tee --pipe --tag' + parset a,b 'grep {}|wc' ::: 1 5 < <(seq 10000) + echo $a + echo $b +} + par_tagstring_pipe() { echo 'bug #50228: --pipe --tagstring broken' seq 3000 | parallel -j4 --pipe -N1000 -k --tagstring {%} LC_ALL=C wc @@ -1052,9 +1061,13 @@ par_hash_and_time_functions() { yyyy_mm_dd_hh_mm_ss(), yyyy_mm_dd_hh_mm(), yyyy_mm_dd(), + hh_mm_ss(), + hh_mm(), yyyymmddhhmmss(), yyyymmddhhmm(), - yyyymmdd()) =}' ::: 1 | + yyyymmdd(), + hhmmss(), + hhmm()) =}' ::: 1 | perl -pe 's/\d/9/g' parallel echo '{= $_=hash($_) =}' ::: 1 | perl -pe 's/[a-f0-9]+/X/g' diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 979b9b1d..a8fbcbba 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -187,8 +187,8 @@ par_remote_load() { par_remote_nice() { echo '### Test --nice remote' - stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | - perl -pe 's/\S*parallel-server\S*/one-server/;s:="[0-9]+":="XXXXX":i;' + parallel --nice 5 -S lo 'ps -l $$|field 8; true' ::: a + nice -n5 parallel -S lo 'ps -l $$|field 8; true' ::: a } par_hgrp_agrp() { diff --git a/testsuite/tests-to-run/parallel-local-ssh5.sh b/testsuite/tests-to-run/parallel-local-ssh5.sh index 702318fb..3378e780 100644 --- a/testsuite/tests-to-run/parallel-local-ssh5.sh +++ b/testsuite/tests-to-run/parallel-local-ssh5.sh @@ -44,6 +44,8 @@ par_lsh() { echo '### --ssh lsh' parallel --ssh 'lsh -c aes256-ctr' -S lo echo ::: OK echo OK | parallel --ssh 'lsh -c aes256-ctr' --pipe -S csh@lo cat + parallel --ssh lsh -S lo echo ::: OK + echo OK | parallel --ssh lsh --pipe -S csh@lo cat # Todo rsync/trc csh@lo # Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler } diff --git a/testsuite/tests-to-run/parallel-local-ssh8.sh b/testsuite/tests-to-run/parallel-local-ssh8.sh index 32418155..5eedc70c 100644 --- a/testsuite/tests-to-run/parallel-local-ssh8.sh +++ b/testsuite/tests-to-run/parallel-local-ssh8.sh @@ -8,7 +8,7 @@ par_transfer_special_char_names() { echo '### Test --return of weirdly named file' - ssh parallel@lo rm 'aa*b' + ssh parallel@lo rm -f 'aa*b' rm -f 'aa<${#}" b' stdout parallel --return {} -S parallel@lo echo '>'{} ::: 'aa<${#}" b' ls 'aa<${#}" b' @@ -73,13 +73,13 @@ _EOS par_keep_order() { echo '### Test --keep-order' seq 0 2 | - parallel --keep-order -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})' + parallel --keep-order -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})' } par_keeporder() { echo '### Test --keeporder' seq 0 2 | - parallel --keeporder -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})' + parallel --keeporder -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})' } par_load_csh() { diff --git a/testsuite/tests-to-run/parallel-local15.sh b/testsuite/tests-to-run/parallel-local15.sh index 35b40cdb..2d140b0f 100755 --- a/testsuite/tests-to-run/parallel-local15.sh +++ b/testsuite/tests-to-run/parallel-local15.sh @@ -22,7 +22,7 @@ echo '### Check that we can have more input than max procs (-j 0) - touch' perl -e 'print map {"more_than_5000-$_\n" } (4000..9999)' | parallel -vj 0 touch | LC_ALL=C sort | tail echo '### rm' perl -e 'print map {"more_than_5000-$_\n" } (4000..9900)' | parallel -j 0 rm | LC_ALL=C sort -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 | egrep -v 'parallel: Warning: Starting|parallel: Warning: Consider' +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 | egrep -v 'parallel: Warning: (Starting|Consider|Only|Try)' ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_=pQ($_)=}"\n\"}"' | LC_ALL=C sort ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_=pQ($_)=}"\n\"}"' | LC_ALL=C sort find . -type f | parallel --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | LC_ALL=C sort diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 6eea9382..07dcbf8c 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -224,7 +224,7 @@ par_halt_one_job 1 par_halt_one_job parallel: This job failed: par_halt_one_job echo 1;exit 1 par_hash_and_time_functions ### Functions for replacement string -par_hash_and_time_functions 9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99999999999999 999999999999 99999999 +par_hash_and_time_functions 9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99:99:99 99:99 99999999999999 999999999999 99999999 999999 9999 par_hash_and_time_functions XxX par_help ### Test --help and -h: Help output (just check we get the same amount of lines) par_help Output from -h and --help @@ -560,6 +560,8 @@ par_parcat_args_stdin OK2 par_parcat_rm bug #51691: parcat --rm remove fifo when opened par_parcat_rm OK1 par_parcat_rm OK file removed +par_parset_tee 1 3440 3440 16910 +par_parset_tee 5 3439 3439 16904 par_perlexpr_with_newline Perl expression spanning 2 lines par_pipe_N1_regexp bug #55131: --regexp --recstart hangs par_pipe_N1_regexp These should give the same diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index f915633c..9d3d06b3 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -76,8 +76,10 @@ par_remote_load parallel: Warning: This job was killed because it timed out: par_remote_load parallel: Warning: perl -e 'while(1){ }' par_remote_load OK par_remote_nice ### Test --nice remote -par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$ENV{"PARALLEL_JOBSLOT"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;$termseq="";do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}my$parent=getppid;do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid!=$parent);if(not$done){my@term_seq=split/,/,$termseq;if(not@term_seq){@term_seq=("TERM",200,"TERM",100,"TERM",50,"KILL",25);}while(@term_seq&&kill(0,-$pid)){kill(shift@term_seq,-$pid);select(undef,undef,undef,(shift@term_seq)/1000);}}wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; -par_remote_nice a b +par_remote_nice NI +par_remote_nice 5 +par_remote_nice NI +par_remote_nice 5 par_ssh ### use --ssh par_ssh Run through BARSSH? par_ssh BARSSH diff --git a/testsuite/wanted-results/parallel-local-ssh5 b/testsuite/wanted-results/parallel-local-ssh5 index 9caf18fe..266565fc 100644 --- a/testsuite/wanted-results/parallel-local-ssh5 +++ b/testsuite/wanted-results/parallel-local-ssh5 @@ -8,6 +8,8 @@ par_pipe_retries parallel: Error: --retries cannot be combined with --roundrobin par_lsh ### --ssh lsh par_lsh OK par_lsh OK +par_lsh OK +par_lsh OK par_input_loss_pipe ### bug #36595: silent loss of input with --pipe and --sshlogin par_input_loss_pipe 1 10000 48894 par_env_parallel_onall bug #54352: env_parallel -Slo --nonall myfunc broken in 20180722 diff --git a/testsuite/wanted-results/parallel-local15 b/testsuite/wanted-results/parallel-local15 index 7098a476..c037f9fb 100644 --- a/testsuite/wanted-results/parallel-local15 +++ b/testsuite/wanted-results/parallel-local15 @@ -171,8 +171,6 @@ ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_= 4 ab 4 b 6 a -parallel: Warning: Only enough file handles to run 252 jobs in parallel. -parallel: Warning: Try running 'parallel -j0 -N 252 --pipe parallel -j0' parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`) parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf parallel: Warning: or increasing /proc/sys/fs/file-max @@ -285,8 +283,6 @@ ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\ 4 ab 4 b 6 a -parallel: Warning: Only enough file handles to run 252 jobs in parallel. -parallel: Warning: Try running 'parallel -j0 -N 252 --pipe parallel -j0' parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`) parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf parallel: Warning: or increasing /proc/sys/fs/file-max