From 94ac9fd72cb4c0f4c9d9d2de6a5166a7a397a7ea Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 21 May 2018 10:04:31 +0200 Subject: [PATCH] parallel: --results test.csv bug incl. testing. --- doc/release_new_version | 9 +++++++- src/parallel | 2 +- src/parallel.pod | 21 ++++++++++++------- src/parallel_design.pod | 19 +++++++++++++++++ src/parallel_tutorial.pod | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 12 +++++++++-- testsuite/wanted-results/parallel-install | 6 ++++++ testsuite/wanted-results/parallel-local-0.3s | 6 +++++- 8 files changed, 64 insertions(+), 13 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 45f6e24f..552c9512 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -199,7 +199,7 @@ to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20180522 ('Korea') released <<[stable]>> +Subject: GNU Parallel 20180522 ('Korea/The Great March of Return') released <<[stable]>> GNU Parallel 20180522 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ @@ -210,6 +210,13 @@ Quote of the month: <<>> New in this release: +https://comquest.in/training +https://bash-prompt.net/guides/parallell-bash/ +https://blog.uta.edu/xus/2017/09/29/an-easy-way-to-run-parallel-jobs-on-stampede2/ + +https://opensource.com/article/18/5/gnu-parallel +https://blog.desdelinux.net/gnu-parallel-haz-mas-cosas-simultaneamente-en-el-terminal/ +https://www.sans.org/summit-archives/file/summit-archive-1524582079.pdf 「今日から使い切る」ための GNU Parallelによる並列処理入門 diff --git a/src/parallel b/src/parallel index 6358331c..2176c7d3 100755 --- a/src/parallel +++ b/src/parallel @@ -1876,7 +1876,7 @@ sub open_csv { ::die_bug("Can't >/dev/null in csv: $!"); $Global::fd{1} = $fd; $Global::fd{2} = $fd; - } elsif($Global::csv) { + } elsif($Global::csvsep) { if(not open($Global::csv_fh,">",$opt::results)) { ::error("Cannot open results file `$opt::results': ". "$!."); diff --git a/src/parallel.pod b/src/parallel.pod index 1eda13dd..97598813 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -52,11 +52,15 @@ often be used as a substitute for B or B. =head2 Reader's guide -Start by watching the intro videos for a quick introduction: +If you prefer reading a book buy B at +http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html +or download it at: https://doi.org/10.5281/zenodo.1146014 + +Otherwise start by watching the intro videos for a quick introduction: http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 Then look at the Bs after the list of B (Use -B). That will give you an idea of what +B). That will give you an idea of what GNU B is capable of. Then spend an hour walking through the tutorial (B -Quote I. This will quote the command line so special -characters are not interpreted by the shell. See the section -QUOTING. Most people will never need this. Quoting is disabled by -default. +Quote I. The command must be a simple command (see B) without redirections and without variable assignments. This +will quote the command line and arguments so special characters are +not interpreted by the shell. See the section QUOTING. Most people +will never need this. Quoting is disabled by default. =item B<--no-run-if-empty> @@ -4323,7 +4328,8 @@ using -q: Prints: B -This is particularly useful if you have lots of quoting. If you want to run a perl script like this: +This is particularly useful if you have lots of quoting. If you want +to run a perl script like this: perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' file @@ -4351,6 +4357,7 @@ If you get errors like: sh: Syntax error: Unterminated quoted string sh: -c: line 0: unexpected EOF while looking for matching `'' sh: -c: line 1: syntax error: unexpected end of file + zsh:1: no matches found: then you might try using B<-q>. diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 790afb5e..ed415d15 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -840,6 +840,25 @@ B<--sqlmaster> be the part that shuffles the jobs. The B<--sqlworker>s simply executes according to Seq number. +=head --csv + +B<--pipepart> is incompatible with B<--csv> because you can have +records like: + + a,b,c + a," + a,b,c + a,b,c + a,b,c + ",c + a,b,c + +Here the second record contains a multi-line field that looks like +records. Since B<--pipepart> does not read then whole file when +searching for record endings, it may start reading in this multi-line +field, which would be wrong. + + =head2 Buffering on disk GNU B buffers output, because if output is not buffered you diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 102ad527..61c934ce 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -13,7 +13,7 @@ Start by watching the intro videos for a quick introduction: http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 Then look at the Bs after the list of B in B (Use B). That will give you +parallel> (Use B). That will give you an idea of what GNU B is capable of. Then spend a couple of hours walking through this tutorial (B&1' diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index 0ebbdb3d..336a1277 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -7,6 +7,12 @@ test -d "parallel-00000000" || mkdir "parallel-00000000" (cd src && make top_distdir=../parallel-00000000 distdir=../parallel-00000000/src \ am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir) make[0]: Entering directory '~/privat/parallel/src' +pod2man --release='00000000' --center='parallel' \ + --section=7 ./parallel_design.pod > ./parallel_design.7n \ +&& mv ./parallel_design.7n ./parallel_design.7 \ +|| echo "Warning: pod2man not found. Using old parallel_design.7" +/bin/bash: pod2man: command not found +Warning: pod2man not found. Using old parallel_design.7 make[0]: Leaving directory '~/privat/parallel/src' test -n "" \ || find "parallel-00000000" -type d ! -perm -755 \ diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 4c0dceda..933ffeb0 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1369,6 +1369,7 @@ par_csv 2"x3" board-Value with ,-Column 3 par_csv_pipe Only pass full records to tail par_csv_pipe Too small block size par_csv_pipe parallel: Warning: A record was longer than 10000. Increasing to --blocksize 13001. +par_csv_pipe parallel: Warning: A record was longer than 13001. Increasing to --blocksize 16903. par_csv_pipe 2000" par_csv_pipe 3000" par_csv_pipe 4000" @@ -1377,7 +1378,6 @@ par_csv_pipe 6000" par_csv_pipe 7000" par_csv_pipe 8000" par_csv_pipe 9000" -par_csv_pipe parallel: Warning: A record was longer than 13001. Increasing to --blocksize 16903. par_csv_pipe 10000" par_csv_pipe 11000" par_csv_pipe More records in single block @@ -1527,6 +1527,10 @@ par_pipepart_recend_recstart 8 par_pipepart_recend_recstart 9 par_pipepart_recend_recstart 10 par_python_children ### bug #49970: Python child process dies if --env is used +par_results ### --results test.csv +par_results a +par_results b +par_results c par_retries_replacement_string 11 par_retries_replacement_string 22 par_retries_replacement_string 22