diff --git a/NEWS b/NEWS index 8d9fa300..4394f7c3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +20190222 + +* Bug fixes and man page updates. + + 20190122 * 'env_parallel --end-session' makes it possible to nest 'env_parallel @@ -17,13 +22,17 @@ * Simpler job scheduling flow. -* MPGI Workshop: Job Parallelization with Task Arrays and GNU Parallel https://www.msi.umn.edu/tutorials/mpgi-workshop-module-4 +* MPGI Workshop: Job Parallelization with Task Arrays and GNU Parallel + https://www.msi.umn.edu/tutorials/mpgi-workshop-module-4 -* CONCOCT uses GNU Parallel https://media.readthedocs.org/pdf/concoct/latest/concoct.pdf +* CONCOCT uses GNU Parallel + https://media.readthedocs.org/pdf/concoct/latest/concoct.pdf -* If you use SLURM read https://wiki.hpc.uconn.edu/index.php/Advanced_SLURM +* If you use SLURM read + https://wiki.hpc.uconn.edu/index.php/Advanced_SLURM -* すぐ忘れてしまう、仕事で使う技 https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c +* すぐ忘れてしまう、仕事で使う技 + https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c * Bug fixes and man page updates. diff --git a/doc/haikus b/doc/haikus index 65620ab8..6fef5a2d 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,5 +1,8 @@ Quote of the month: +Y'all need some GNU parallel in your lives +-- ChaKu @ChaiLovesChai@twitter + It's really powerful for smallish data processing scripts -- João Veiga @jcsvveiga@twitter diff --git a/doc/release_new_version b/doc/release_new_version index fb0b9be2..3efc2e33 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -207,7 +207,7 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20190222 ('') released <<[stable]>> +Subject: GNU Parallel 20190222 ('Al-Baghuz Fawqani') released <<[stable]>> GNU Parallel 20190222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ @@ -220,6 +220,12 @@ Quote of the month: New in this release: +https://medium.freecodecamp.org/how-to-supercharge-your-bash-workflows-with-gnu-parallel-53aab0aea141 + +* GNU Parallel is available in Termux https://github.com/termux/termux-packages/tree/master/packages/parallel + +https://www.youtube.com/watch?v=LHb29uW_KyI + * blog.cogneurostats.com/?p=148 https://www.neurotrivial.com/2019/01/24/freesurfer-recon-all/ diff --git a/src/parallel b/src/parallel index d806e11b..6b2520cc 100755 --- a/src/parallel +++ b/src/parallel @@ -5406,6 +5406,13 @@ sub memfree_recompute() { q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ]. q[ { sum += \$2} END { print sum }' ]. q[ /proc/meminfo } ], + # Android uses same code as GNU/Linux + "android" => + q[ print 1024 * qx{ ]. + q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ]. + q[ { sum += \$2} END { print sum }' ]. + q[ /proc/meminfo } ], + # $ vmstat 1 1 # procs memory page faults cpu # r b w avm free re at pi po fr de sr in sy cs us sy id @@ -5800,8 +5807,9 @@ sub loadavg_too_high($) { awk '{print $2,$1}' }; $dummy="echo S COMMAND;echo R dummy"; %ps=( - # TODO Find better code for AIX + # TODO Find better code for AIX/Android 'aix' => "uptime", + 'android' => "uptime", 'cygwin' => $cygwin, 'darwin' => $bsd, 'dec_osf' => $sysv2, @@ -5814,13 +5822,13 @@ sub loadavg_too_high($) { 'minix' => "ps el|awk '{print \$1,\$11}'", 'mirbsd' => $bsd, 'msys' => $cygwin, - 'MSWin32' => $sysv, 'netbsd' => $bsd, 'nto' => $dummy, 'openbsd' => $bsd, 'solaris' => $sysv, 'svr5' => $psel, 'ultrix' => "ps -ax | awk '{print \$3,\$5}'", + 'MSWin32' => $sysv, ); print `$ps{$^O}`; }); @@ -6408,6 +6416,8 @@ sub socket_core_thread() { if ($^O eq 'linux') { $cpu = sct_gnu_linux(); + } elsif ($^O eq 'android') { + $cpu = sct_android(); } elsif ($^O eq 'freebsd') { $cpu = sct_freebsd(); } elsif ($^O eq 'netbsd') { @@ -6433,8 +6443,9 @@ sub socket_core_thread() { } elsif ($^O eq 'dec_osf') { $cpu = sct_tru64(); } else { - # Try all methods until you find something that works + # Try all methods until we find something that works $cpu = (sct_gnu_linux() + || sct_android() || sct_freebsd() || sct_netbsd() || sct_openbsd() @@ -6535,6 +6546,16 @@ sub sct_gnu_linux() { } } +sub sct_android() { + # Returns: + # { 'sockets' => #sockets + # 'cores' => #cores + # 'threads' => #threads + # 'active' => #taskset_threads } + # Use GNU/Linux + return sct_gnu_linux(); +} + sub sct_freebsd() { # Returns: # { 'sockets' => #sockets diff --git a/src/parallel.pod b/src/parallel.pod index e2278f10..b3c8330c 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -3286,18 +3286,15 @@ This also works if the input file is a file with columns: GNU B makes all combinations when given two lists. To make all combinations in a single list with unique values, you -repeat the list and use replacement string with a Perl expression that -skips the job if the value from input source 1 is greater than or -equal to the value from input source 2: +repeat the list and use replacement string B<{choose_k}>: - parallel echo {= 'if($arg[1] ge $arg[2]) { skip() }' =} \ - ::: A B C D ::: A B C D + parallel --plus echo {choose_k} ::: A B C D ::: A B C D -Or more generally: + parallel --plus echo arg2 {2choose_k} arg1 {1choose_k} ::: A B C D ::: A B C D - parallel echo \ - '{= for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } } =}' \ - ::: A B C D ::: A B C D ::: A B C D +B works for any number of input sources: + + parallel --plus echo {choose_k} ::: A B C D ::: A B C D ::: A B C D =head1 EXAMPLE: From a to b and b to c diff --git a/src/parcat b/src/parcat index 3ea4add1..721da3d4 100755 --- a/src/parcat +++ b/src/parcat @@ -122,7 +122,9 @@ do { # Nothing read, but would not block: # This file is done $s->remove($infh); - syswrite($Global::fh{$outfd},"@{$buffer{$infh}{$outfd}}"); + for(@{$buffer{$infh}{$outfd}}) { + syswrite($Global::fh{$outfd},$_); + } delete $buffer{$infh}; # Closing the $infh causes it to block # close $infh; diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 56ee03a8..64bea043 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -264,6 +264,7 @@ par_too_long_line_X() { } par_test_cpu_detection() { + # Xeon 8 core server in Germany cpuinfo1=" processor : 0 vendor_id : GenuineIntel @@ -465,6 +466,7 @@ cache_alignment : 64 address sizes : 38 bits physical, 48 bits virtual power management: "; + # Core i7 Acer laptop cpuinfo2=" processor : 0 vendor_id : GenuineIntel @@ -682,6 +684,7 @@ cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: "; + # Core i5 laptop firewall cpuinfo3=" processor : 0 vendor_id : GenuineIntel @@ -791,6 +794,7 @@ cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: "; + # cpuinfo4=" processor : 0 vendor_id : AuthenticAMD @@ -2083,6 +2087,7 @@ cache_alignment : 64 address sizes : 46 bits physical, 48 bits virtual power management: "; + # HP Laptop Compaq 6530b cpuinfo6=" processor : 0 vendor_id : GenuineIntel @@ -2138,6 +2143,81 @@ cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: +"; + # Huawei P Smart Octa-core (4x2.36 GHz Cortex-A53 & 4x1.7 GHz Cortex-A53) + cpuinfo7=" +processor : 0 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 1 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 2 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 3 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 4 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 5 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 6 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + +processor : 7 +BogoMIPS : 3.84 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x0 +CPU part : 0xd03 +CPU revision : 4 + "; test_one() { export PARALLEL_CPUINFO="$1" @@ -2148,8 +2228,8 @@ power management: } export -f test_one ppar -j0 -0 -k --tagstring {2} test_one {1} \ - ::: "$cpuinfo1" "$cpuinfo2" "$cpuinfo3" "$cpuinfo4" "$cpuinfo5" "$cpuinfo6" \ - :::+ "2-8-8-8" "1-4-8-4" "1-2-4-2" "1-2-2-2" "2-24-48-24" "1-2-2-2" + ::: "$cpuinfo1" "$cpuinfo2" "$cpuinfo3" "$cpuinfo4" "$cpuinfo5" "$cpuinfo6" "$cpuinfo7" \ + :::+ "2-8-8-8" "1-4-8-4" "1-2-4-2" "1-2-2-2" "2-24-48-24" "1-2-2-2" "1-8-8-8" } export -f $(compgen -A function | grep par_) diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 7ba40a48..de83dd90 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -625,6 +625,7 @@ par_test_cpu_detection 1-2-4-2 1 2 4 2 par_test_cpu_detection 1-2-2-2 1 2 2 2 par_test_cpu_detection 2-24-48-24 2 24 48 24 par_test_cpu_detection 1-2-2-2 1 2 2 2 +par_test_cpu_detection 1-8-8-8 1 8 8 8 par_too_long_line_X bug #54869: Long lines break par_too_long_line_X 1 29302 131012 par_too_long_line_X 1 12698 63490 diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index ed3cb136..ac2bb97f 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -186,7 +186,8 @@ par_shellquote mksh ' par_shellquote mksh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' par_shellquote posh ' par_shellquote posh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' -par_shellquote rzsh zsh:1: /bin/echo: restricted +par_shellquote rzsh ' +par_shellquote rzsh  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' par_shellquote sash ' par_shellquote sash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~' par_shellquote sh '