parallel: Implemented hh_mm_ss() hh_mm() hhmmss() hhmm() for {= perl expr =}.

2024-10-18 04:49:40 +00:00 · 2022-02-17 13:19:55 +01:00 · 2022-02-17 13:19:55 +01:00 · 486b7bf83f
parent 3461b32dea
commit 486b7bf83f
24 changed files with 80 additions and 39 deletions
--- a/Makefile.am
+++ b/Makefile.am
@ -168,4 +168,4 @@ pack_unpack_and_test_build:
 	./configure && make -j && sudo make -j install


-EXTRA_DIST = CITATION CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt
+EXTRA_DIST = CITATION CITATION.cff CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt
--- a/Makefile.in
+++ b/Makefile.in
@ -279,7 +279,7 @@ top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 SUBDIRS = src
-EXTRA_DIST = CITATION CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt
+EXTRA_DIST = CITATION CITATION.cff CREDITS LICENSES/CC-BY-SA-4.0.txt LICENSES/GFDL-1.3-or-later.txt LICENSES/GPL-3.0-or-later.txt
 all: config.h
 	$(MAKE) $(AM_MAKEFLAGS) all-recursive

--- a/doc/haikus
+++ b/doc/haikus
@ -4,6 +4,14 @@
 
 Quote of the month:

+
+  GNU parallel すごいな、シェルスクリプトを make -jX みたいにして並列
+  に走らせたいときに超便利
+    -- びーる @debugordie@twitter
+
+  GNU parallel いいよ、大量のデータファイルをスクリプトで処理する人とかにマジおすすめ
+    -- びーる @debugordie@twitter
+
  Igual @GnuParallel te puede echar una mano. Te permite pensar y
  programar en serie pero ejecutar en paralelo usando cores de una
  máquina o máquinas remotas. Una vez que lo has usando te lamentas no
--- a/doc/release_new_version
+++ b/doc/release_new_version
@ -254,32 +254,32 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>

-Subject: GNU Parallel 20220122 ('20 years') released [stable]
+Subject: GNU Parallel 20220222 ('Ukraine/Mali') released [stable]

-GNU Parallel 20220122 ('20 years') [stable] has been released. It is available for download at: lbry://@GnuParallel:4
+GNU Parallel 20220222 ('<<>>') [stable] has been released. It is available for download at: lbry://@GnuParallel:4

 No new functionality was introduced so this is a good candidate for a stable release.

 Quote of the month:

-  I'm going to make GNU parallel my new superpower this year.
-    -- Christian G. Warden @xn@twitter
+  <<>>
  
 New in this release:

-* GNU Parallel turned 20 on the 2022-01-06 https://www.gnu.org/software/parallel/20th-birthday.html
+* Encoding of remote commands has changed. This limits the length of remote commands to less than half. It also requires $REMOTE_TMPDIR to be writable.

-* Bug fixes and man page updates.
+* Time functions hh_mm_ss() hhmmss() hh_mm() hhmm() are now available for {= =}.
+
+<<>>
+
+* Bug fixes and man page updates: Many options now have a 'See also' section.

 News about GNU Parallel:

-* Save Time with Modern Search Techniques https://www.eventbrite.com/e/ohio-information-security-forum-january-2022-monthly-meeting-tickets-240632737997
+https://blog.csdn.net/qq_38197157/article/details/112596389
+https://www.freelancer.com/projects/linux/gnu-parallel-processing-for-data/?ngsw-bypass=&w=f

-* Bash "Fail immediately if any fails" https://morioh.com/p/a01b883dc57c
-
-* Re-encoding the EmacsConf videos with FFmpeg and GNU Parallel https://sachachua.com/blog/2021/12/re-encoding-the-emacsconf-videos-with-ffmpeg-and-gnu-parallel/
-
-* Bash 平行 https://learntutorials.net/ja/bash/topic/10778/%E5%B9%B3%E8%A1%8C
+<<>>

 Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

--- a/src/env_parallel.ash
+++ b/src/env_parallel.ash
@ -385,7 +385,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/env_parallel.bash
+++ b/src/env_parallel.bash
@ -385,7 +385,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/env_parallel.dash
+++ b/src/env_parallel.dash
@ -385,7 +385,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/env_parallel.ksh
+++ b/src/env_parallel.ksh
@ -363,7 +363,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/env_parallel.mksh
+++ b/src/env_parallel.mksh
@ -365,7 +365,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/env_parallel.sh
+++ b/src/env_parallel.sh
@ -390,7 +390,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/env_parallel.zsh
+++ b/src/env_parallel.zsh
@ -355,7 +355,7 @@ _parset_main() {
 	return 255
    fi
    if [ "$_parset_NAME" = "--version" ] ; then
-	echo "parset 20220122 (GNU parallel `parallel --minversion 1`)"
+	echo "parset 20220123 (GNU parallel `parallel --minversion 1`)"
 	echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
 	echo "Foundation, Inc."
 	echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
--- a/src/parallel
+++ b/src/parallel
@ -13052,6 +13052,14 @@ sub total_jobs() {
 	# ISO8601 2038-01-19
 	::strftime("%Y-%m-%d", localtime(time()));
    }
+    sub hh_mm_ss() {
+	# ISO8601 03:14:08
+	::strftime("%H:%M:%S", localtime(time()));
+    }
+    sub hh_mm() {
+	# ISO8601 03:14
+	::strftime("%H:%M", localtime(time()));
+    }
    sub yyyymmddhhmmss() {
 	# ISO8601 20380119031408
 	::strftime("%Y%m%d%H%M%S", localtime(time()));
@ -13064,6 +13072,14 @@ sub total_jobs() {
 	# ISO8601 20380119
 	::strftime("%Y%m%d", localtime(time()));
    }
+    sub hhmmss() {
+	# ISO8601 031408
+	::strftime("%H%M%S", localtime(time()));
+    }
+    sub hhmm() {
+	# ISO8601 0314
+	::strftime("%H%M", localtime(time()));
+    }

    sub replace($$$$) {
 	# Calculates the corresponding value for a given perl expression
--- a/src/parsort
+++ b/src/parsort
@ -122,7 +122,7 @@ GetOptions(
    "help" => \$opt::dummy,
    ) || exit(255);
 $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
-$Global::version = 20220122;
+$Global::version = 20220123;
 if($opt::version) { version(); exit 0; }
@Global::sortoptions = grep { ! /^-D$/ }
    shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]);
--- a/src/sql
+++ b/src/sql
@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);

 sub parse_options {
-    $Global::version = 20220122;
+    $Global::version = 20220123;
    $Global::progname = 'sql';

    # This must be done first as this may exec myself
--- a/testsuite/tests-to-run/parallel-android.sh
+++ b/testsuite/tests-to-run/parallel-android.sh
@ -9,7 +9,7 @@ torsocks bash <<'EOF'
 . `which env_parallel.bash`
 env_parallel --session

-host=$(parallel -j0 --halt now,success=1 ssh {} echo {} ::: koditor huator fairtor 2>/dev/null)
+host=$(parallel -j0 --halt now,success=1 ssh {} echo {} ::: kodi koditor huator fairtor 2>/dev/null)
 if [ -z "$host" ] ; then
    echo Error: no android host working
 else    
--- a/testsuite/tests-to-run/parallel-local-0.3s.sh
+++ b/testsuite/tests-to-run/parallel-local-0.3s.sh
@ -60,7 +60,8 @@ EOF

 par_crnl() {
    echo '### Give a warning if input is DOS-ascii'
-    printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | stdout parallel -k echo {}a
+    printf "b\r\nc\r\nd\r\ne\r\nf\r\n" |
+	stdout parallel -k 'sleep 0.01; echo {}a'
    echo This should give no warning because -d is set
    printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | parallel -k -d '\r\n' echo {}a
    echo This should give no warning because line2 has newline only
@ -649,6 +650,14 @@ par_tee() {
    seq 300000 | parallel 'grep {1} | LC_ALL=C wc {2}' ::: {1..5} ::: -l -c
 }

+par_parset_tee() {
+    . $(which env_parallel.bash)
+    export PARALLEL='-k --tee --pipe --tag'
+    parset a,b 'grep {}|wc' ::: 1 5 < <(seq 10000)
+    echo $a
+    echo $b
+}
+
 par_tagstring_pipe() {
    echo 'bug #50228: --pipe --tagstring broken'
    seq 3000 | parallel -j4 --pipe -N1000 -k --tagstring {%} LC_ALL=C wc
@ -1052,9 +1061,13 @@ par_hash_and_time_functions() {
                              yyyy_mm_dd_hh_mm_ss(),
                              yyyy_mm_dd_hh_mm(),
                              yyyy_mm_dd(),
+                              hh_mm_ss(),
+                              hh_mm(),
                              yyyymmddhhmmss(),
 			      yyyymmddhhmm(),
-			      yyyymmdd()) =}' ::: 1 |
+			      yyyymmdd(),
+                              hhmmss(),
+                              hhmm()) =}' ::: 1 |
 	perl -pe 's/\d/9/g'
    parallel echo '{= $_=hash($_) =}' ::: 1 |
 	perl -pe 's/[a-f0-9]+/X/g'
--- a/testsuite/tests-to-run/parallel-local-ssh1.sh
+++ b/testsuite/tests-to-run/parallel-local-ssh1.sh
@ -187,8 +187,8 @@ par_remote_load() {

 par_remote_nice() {
    echo '### Test --nice remote'
-    stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo  \$PAR {}"' ::: b |
-	perl -pe 's/\S*parallel-server\S*/one-server/;s:="[0-9]+":="XXXXX":i;'
+    parallel --nice 5 -S lo 'ps -l $$|field 8; true' ::: a
+    nice -n5 parallel -S lo 'ps -l $$|field 8; true' ::: a
 }

 par_hgrp_agrp() {
--- a/testsuite/tests-to-run/parallel-local-ssh5.sh
+++ b/testsuite/tests-to-run/parallel-local-ssh5.sh
@ -44,6 +44,8 @@ par_lsh() {
    echo '### --ssh lsh'
    parallel --ssh 'lsh -c aes256-ctr' -S lo echo ::: OK
    echo OK | parallel --ssh 'lsh -c aes256-ctr' --pipe -S csh@lo cat
+    parallel --ssh lsh -S lo echo ::: OK
+    echo OK | parallel --ssh lsh --pipe -S csh@lo cat
    # Todo rsync/trc csh@lo
    # Test gl. parallel med --ssh lsh: Hvilke fejler? brug dem. Også hvis de fejler
 }
--- a/testsuite/tests-to-run/parallel-local-ssh8.sh
+++ b/testsuite/tests-to-run/parallel-local-ssh8.sh
@ -8,7 +8,7 @@

 par_transfer_special_char_names() {
    echo '### Test --return of weirdly named file'
-    ssh parallel@lo rm 'aa*b'
+    ssh parallel@lo rm -f 'aa*b'
    rm -f 'aa<${#}" b'
    stdout parallel --return {} -S parallel@lo echo '>'{} ::: 'aa<${#}" b'
    ls 'aa<${#}" b'
@ -73,13 +73,13 @@ _EOS
 par_keep_order() {
    echo '### Test --keep-order'
    seq 0 2 |
-    parallel --keep-order -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})'
+	parallel --keep-order -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})'
 }

 par_keeporder() {
    echo '### Test --keeporder'
    seq 0 2 |
-    parallel --keeporder -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})'
+	parallel --keeporder -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})'
 }

 par_load_csh() {
--- a/testsuite/tests-to-run/parallel-local15.sh
+++ b/testsuite/tests-to-run/parallel-local15.sh
@ -22,7 +22,7 @@ echo '### Check that we can have more input than max procs (-j 0) - touch'
 perl -e 'print map {"more_than_5000-$_\n" } (4000..9999)' | parallel -vj 0 touch | LC_ALL=C sort | tail
 echo '### rm'
 perl -e 'print map {"more_than_5000-$_\n" } (4000..9900)' | parallel -j 0 rm | LC_ALL=C sort
-cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 | egrep -v 'parallel: Warning: Starting|parallel: Warning: Consider'
+cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 | egrep -v 'parallel: Warning: (Starting|Consider|Only|Try)'
 ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_=pQ($_)=}"\n\"}"' | LC_ALL=C sort
 ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_=pQ($_)=}"\n\"}"' | LC_ALL=C sort
 find . -type f | parallel --group  "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | LC_ALL=C sort
--- a/testsuite/wanted-results/parallel-local-0.3s
+++ b/testsuite/wanted-results/parallel-local-0.3s
@ -224,7 +224,7 @@ par_halt_one_job	1
 par_halt_one_job	parallel: This job failed:
 par_halt_one_job	echo 1;exit 1
 par_hash_and_time_functions	### Functions for replacement string
-par_hash_and_time_functions	9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99999999999999 999999999999 99999999
+par_hash_and_time_functions	9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99:99:99 99:99 99999999999999 999999999999 99999999 999999 9999
 par_hash_and_time_functions	XxX
 par_help	### Test --help and -h: Help output (just check we get the same amount of lines)
 par_help	Output from -h and --help
@ -560,6 +560,8 @@ par_parcat_args_stdin	OK2
 par_parcat_rm	bug #51691: parcat --rm remove fifo when opened
 par_parcat_rm	OK1
 par_parcat_rm	OK file removed
+par_parset_tee	1 3440 3440 16910
+par_parset_tee	5 3439 3439 16904
 par_perlexpr_with_newline	Perl expression spanning 2 lines
 par_pipe_N1_regexp	bug #55131: --regexp --recstart hangs
 par_pipe_N1_regexp	These should give the same
--- a/testsuite/wanted-results/parallel-local-ssh1
+++ b/testsuite/wanted-results/parallel-local-ssh1
@ -76,8 +76,10 @@ par_remote_load	parallel: Warning: This job was killed because it timed out:
 par_remote_load	parallel: Warning: perl -e 'while(1){ }'
 par_remote_load	OK
 par_remote_nice	### Test --nice remote
-par_remote_nice	ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$ENV{"PARALLEL_JOBSLOT"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo  \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;$termseq="";do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}my$parent=getppid;do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid!=$parent);if(not$done){my@term_seq=split/,/,$termseq;if(not@term_seq){@term_seq=("TERM",200,"TERM",100,"TERM",50,"KILL",25);}while(@term_seq&&kill(0,-$pid)){kill(shift@term_seq,-$pid);select(undef,undef,undef,(shift@term_seq)/1000);}}wait;exit($?&127?128+($?&127):1+$?>>8)'"'";
-par_remote_nice	a b
+par_remote_nice	NI
+par_remote_nice	5
+par_remote_nice	NI
+par_remote_nice	5
 par_ssh	### use --ssh
 par_ssh	Run through BARSSH?
 par_ssh	BARSSH
--- a/testsuite/wanted-results/parallel-local-ssh5
+++ b/testsuite/wanted-results/parallel-local-ssh5
@ -8,6 +8,8 @@ par_pipe_retries	parallel: Error: --retries cannot be combined with --roundrobin
 par_lsh	### --ssh lsh
 par_lsh	OK
 par_lsh	OK
+par_lsh	OK
+par_lsh	OK
 par_input_loss_pipe	### bug #36595: silent loss of input with --pipe and --sshlogin
 par_input_loss_pipe	      1   10000   48894
 par_env_parallel_onall	bug #54352: env_parallel -Slo --nonall myfunc broken in 20180722
--- a/testsuite/wanted-results/parallel-local15
+++ b/testsuite/wanted-results/parallel-local15
@ -171,8 +171,6 @@ ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" "{=$_=
 4  ab
 4 b
 6 a
-parallel: Warning: Only enough file handles to run 252 jobs in parallel.
-parallel: Warning: Try running 'parallel -j0 -N 252 --pipe parallel -j0'
 parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`)
 parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf
 parallel: Warning: or increasing /proc/sys/fs/file-max
@ -285,8 +283,6 @@ ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\
 4  ab
 4 b
 6 a
-parallel: Warning: Only enough file handles to run 252 jobs in parallel.
-parallel: Warning: Try running 'parallel -j0 -N 252 --pipe parallel -j0'
 parallel: Warning: or increasing 'ulimit -n' (try: ulimit -n `ulimit -Hn`)
 parallel: Warning: or increasing 'nofile' in /etc/security/limits.conf
 parallel: Warning: or increasing /proc/sys/fs/file-max