From 1e15645faaa9c80806f9c5b054226bf1098a58bf Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 22 Apr 2018 16:28:01 +0200 Subject: [PATCH] parallel: --pipe --csv: Pass only full CSV-records. --- README | 12 +- configure.ac | 2 +- doc/release_new_version | 23 +++- src/niceload | 2 +- src/parallel | 39 ++++-- src/parallel.pod | 2 + src/sql | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 113 +++-------------- testsuite/tests-to-run/parallel-local-1s.sh | 93 ++++++++++++++ testsuite/wanted-results/parallel-local-0.3s | 119 +++--------------- testsuite/wanted-results/parallel-local-1s | 102 +++++++++++++++ testsuite/wanted-results/parallel-local-ssh2 | 6 +- testsuite/wanted-results/parallel-local-ssh7 | 12 +- testsuite/wanted-results/parallel-local12 | 6 +- testsuite/wanted-results/parallel-local22 | 10 +- testsuite/wanted-results/parallel-local9 | 2 +- 16 files changed, 305 insertions(+), 240 deletions(-) diff --git a/README b/README index 4e6c6282..25946aeb 100644 --- a/README +++ b/README @@ -44,9 +44,9 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20180322.tar.bz2 - bzip2 -dc parallel-20180322.tar.bz2 | tar xvf - - cd parallel-20180322 + wget https://ftpmirror.gnu.org/parallel/parallel-20180422.tar.bz2 + bzip2 -dc parallel-20180422.tar.bz2 | tar xvf - + cd parallel-20180422 ./configure && make && sudo make install @@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20180322.tar.bz2 - bzip2 -dc parallel-20180322.tar.bz2 | tar xvf - - cd parallel-20180322 + wget https://ftpmirror.gnu.org/parallel/parallel-20180422.tar.bz2 + bzip2 -dc parallel-20180422.tar.bz2 | tar xvf - + cd parallel-20180422 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure.ac b/configure.ac index 8f49e259..7dacbfd2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20180322], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20180422], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 08500022..bb4895f3 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -207,18 +207,29 @@ GNU Parallel 20180422 ('Trèbes') <<[stable]>> has been released. It is availabl Quote of the month: -<<>> - -- +Today I discovered GNU Parallel, and I don’t know what to do with all this spare time. + --Ryan Booker New in this release: -Launch at DIKU +* --csv makes GNU Parallel parse the input sources as CSV. When used with --pipe it only passes full CSV-records. -https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently +* Time in --bar is printed as 1d02h03m04s. -http://fliplinux.com/gnu-parallel-6.html +* Optimization of --tee: It spawns a process less per value. + +* The GNU Parallel 2018 book is now available: http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html + +* Modern pentest tricks for faster, wider, greater engagement (15) https://conference.hitb.org/hitbsecconf2018ams/materials/D1%20COMMSEC%20-%20Thomas%20Debize%20-%20Modern%20Pentest%20Tricks%20for%20Faster,%20Wider,%20Greater%20Engagements.pdf + +* Running many serial jobs efficiently https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently + +* GNU Parallel: как сохранить результаты нескольких команд для переменной? http://fliplinux.com/gnu-parallel-6.html + +* Running Multiple Commands in Parallel on a GNU Linux https://www.youtube.com/watch?v=sHpTywpb4_4 + +* Klaatu covers the documentation of GNU parallel in episode 12x15 http://gnuworldorder.info/ -https://www.youtube.com/watch?v=sHpTywpb4_4 <> diff --git a/src/niceload b/src/niceload index f8245118..a42bfb7e 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20180323; +$Global::version = 20180422; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 26c5c3d9..5ece4501 100755 --- a/src/parallel +++ b/src/parallel @@ -392,7 +392,8 @@ sub find_split_positions { } } else { # If match $recend$recstart => Record position - # TODO optimize to only look at the appended $dd_block_size + len $recendrecstart + # TODO optimize to only look at the appended + # $dd_block_size + len $recendrecstart # TODO increase $dd_block_size to optimize for longer records my $i = index64(\$buf,$recendrecstart); if($i != -1) { @@ -434,7 +435,7 @@ sub cat_partial { sysseek(STDIN,shift,0) || die; $left = shift; while($read = - sysread(STDIN,$buf, $left > 1048524 ? 1048524 : $left)){ + sysread(STDIN,$buf, $left > 131072 ? 131072 : $left)){ $left -= $read; syswrite(STDOUT,$buf); } @@ -500,6 +501,7 @@ sub spreadstdin { # Read n-line records my $n_lines = $buf =~ tr/\n/\n/; my $last_newline_pos = rindex64(\$buf,"\n"); + # Go backwards until there are full n-line records while($n_lines % $Global::max_lines) { $n_lines--; $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1); @@ -533,6 +535,21 @@ sub spreadstdin { $recstart,$recend,length $1); } } + } elsif($opt::csv) { + # Read a full CSV record + # even number of " + end of line + my $last_newline_pos = length $buf; + do { + # find last EOL + $last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1); + # While uneven " + } while((substr($buf,0,$last_newline_pos) =~ y/"/"/)%2 + and $last_newline_pos >= 0); + # Chop at $last_newline_pos as that is where CSV record ends + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$last_newline_pos+1); + shorten(\$buf,$last_newline_pos+1); } else { if($Global::max_number_of_args) { # -N => (start..*?end){n} @@ -1427,7 +1444,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20180323; + $Global::version = 20180422; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -10817,7 +10834,7 @@ sub read_arg_from_fh { my $fh = shift; my $prepend; my $arg; - my $double_quotes = 0; + my $half_record = 0; do {{ # This makes 10% faster if(not defined ($arg = <$fh>)) { @@ -10829,16 +10846,20 @@ sub read_arg_from_fh { } if($opt::csv) { # We need to read a full CSV line. - $double_quotes += ($arg =~ y/"/"/); - if($double_quotes % 2) { - # CSV halflines with quoting: + if(($arg =~ y/"/"/) % 2 ) { + # The number of " on the line is uneven: + # If we were in a half_record => we have a full record now + # If we were ouside a half_record => we are in a half record now + $half_record = not $half_record; + } + if($half_record) { + # CSV half-record with quoting: # col1,"col2 2""x3"" board newline <-this one # cont",col3 $prepend .= $arg; redo; } else { - # Now we have a full CSV line - $double_quotes = 0; + # Now we have a full CSV record } } # Remove delimiter diff --git a/src/parallel.pod b/src/parallel.pod index 7406ea02..5b0fa49f 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -607,6 +607,8 @@ Even quoted newlines are parsed correctly: echo 'Line 2 in field 1";value 2') | parallel --csv --colsep ';' echo Field 1: {1} Field 2: {2} +When used with B<--pipe> only pass full CSV-records. + =item B<--delimiter> I =item B<-d> I diff --git a/src/sql b/src/sql index a91218ed..bc93f72f 100755 --- a/src/sql +++ b/src/sql @@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20180323; + $Global::version = 20180422; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index d7afdb3a..17143c0c 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -521,23 +521,6 @@ par_pipe_record_size_in_lines() { seq 10 | parallel -k --pipe -l 4 cat\;echo bug 34958-2 } -par_pipe_compress_blocks() { - echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations" - seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat - echo echo 1-4 + 1-4 - seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo - echo 4 times wc to stderr to stdout - (seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null - echo 1 2 3 4 - seq 4 | parallel -k -j3 --compress echo - echo 1 2 3 4 - seq 4 | parallel -k -j1 --compress echo - echo 1 2 - seq 2 | parallel -k -j1 --compress echo - echo 1 2 3 - seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat -} - par_pipe_no_command() { echo '### --pipe without command' @@ -570,18 +553,6 @@ par_tricolonplus() { parallel -k echo :::: <(seq 3) :::: <(seq 21 23) :::+ a b c ::: aa bb cc } -par_header_parens() { - echo 'bug #49538: --header and {= =}' - - parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O - parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O - parallel --header : echo {var/.} ::: var sub/dir/file.ext - parallel --header : echo {var//} ::: var sub/dir/file.ext - parallel --header : echo {var/.} ::: var sub/dir/file.ext - parallel --header : echo {var/} ::: var sub/dir/file.ext - parallel --header : echo {var.} ::: var sub/dir/file.ext -} - par_colsep_0() { echo 'bug --colsep 0' @@ -656,21 +627,6 @@ par_link_files_as_only_arg() { parallel -k echo ::::+ <(seq 10) <(seq 3) <(seq 4) } -par_macron() { - print_it() { - parallel ::: "echo $1" - parallel echo ::: "$1" - parallel echo "$1" ::: "$1" - parallel echo \""$1"\" ::: "$1" - parallel -q echo ::: "$1" - parallel -q echo "$1" ::: "$1" - parallel -q echo \""$1"\" ::: "$1" - } - print_it "$(perl -e 'print "\257"')" - print_it "$(perl -e 'print "\257\256"')" - print_it "$(perl -e 'print "\257<\257<\257>\257>"')" -} - par_basic_halt() { cpuburn=$(tempfile) cpuburn2=$(tempfile) @@ -747,51 +703,6 @@ par_linebuffer_files() { stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l } -par_parset() { - . `which env_parallel.bash` - echo '### parset into array' - parset arr1 echo ::: foo bar baz - echo ${arr1[0]} ${arr1[1]} ${arr1[2]} - - echo '### parset into vars with comma' - parset comma3,comma2,comma1 echo ::: baz bar foo - echo $comma1 $comma2 $comma3 - - echo '### parset into vars with space' - parset 'space3 space2 space1' echo ::: baz bar foo - echo $space1 $space2 $space3 - - echo '### parset with newlines' - parset 'newline3 newline2 newline1' seq ::: 3 2 1 - echo "$newline1" - echo "$newline2" - echo "$newline3" - - echo '### parset into indexed array vars' - parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo - echo ${myarray[*]} - echo ${myarray[4]} ${myarray[5]} ${myarray[5]} - - echo '### env_parset' - alias myecho='echo myecho "$myvar" "${myarr[1]}"' - myvar="myvar" - myarr=("myarr 0" "myarr 1" "myarr 2") - mynewline="`echo newline1;echo newline2;`" - env_parset arr1 myecho ::: foo bar baz - echo "${arr1[0]} ${arr1[1]} ${arr1[2]}" - env_parset comma3,comma2,comma1 myecho ::: baz bar foo - echo "$comma1 $comma2 $comma3" - env_parset 'space3 space2 space1' myecho ::: baz bar foo - echo "$space1 $space2 $space3" - env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1 - echo "$newline1" - echo "$newline2" - echo "$newline3" - env_parset 'myarray[6],myarray[5],myarray[4]' myecho ::: baz bar foo - echo "${myarray[*]}" - echo "${myarray[4]} ${myarray[5]} ${myarray[5]}" -} - par_halt_one_job() { echo '# Halt soon if there is a single job' echo should run 0 1 = job 1 2 @@ -823,13 +734,6 @@ par_pipepart_recend_recstart() { rm $tmp1 2>/dev/null } -par_parset_v() { - echo 'bug #52507: parset arr1 -v echo ::: fails' - . `which env_parallel.bash` - parset arr1 -v seq ::: 1 2 3 - echo "${arr1[2]}" -} - par_pipe_tag_v() { echo 'pipe with --tag -v' seq 3 | parallel -v --pipe --tagstring foo cat @@ -861,6 +765,23 @@ par_csv() { parallel --csv echo {1}-{2}-{3} } +par_csv_pipe() { + echo 'Only pass full records to tail' + echo 'Too small block size' + perl -e 'for $b(1..10) { + print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000; + print "\n" + }' | + stdout parallel --pipe --csv -k --block 10k tail -n1 + + echo 'More records in single block' + perl -e 'for $b(1..10) { + print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000; + print "\n" + }' | + stdout parallel --pipe --csv -k --block 100k tail -n1 +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 829e53e2..76c118bc 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -237,6 +237,99 @@ _EOF ps aux | grep parallel[-]-lb-test } +par_macron() { + print_it() { + parallel ::: "echo $1" + parallel echo ::: "$1" + parallel echo "$1" ::: "$1" + parallel echo \""$1"\" ::: "$1" + parallel -q echo ::: "$1" + parallel -q echo "$1" ::: "$1" + parallel -q echo \""$1"\" ::: "$1" + } + print_it "$(perl -e 'print "\257"')" + print_it "$(perl -e 'print "\257\256"')" + print_it "$(perl -e 'print "\257<\257<\257>\257>"')" +} + +par_header_parens() { + echo 'bug #49538: --header and {= =}' + + parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O + parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O + parallel --header : echo {var/.} ::: var sub/dir/file.ext + parallel --header : echo {var//} ::: var sub/dir/file.ext + parallel --header : echo {var/.} ::: var sub/dir/file.ext + parallel --header : echo {var/} ::: var sub/dir/file.ext + parallel --header : echo {var.} ::: var sub/dir/file.ext +} + +par_parset2() { + . `which env_parallel.bash` + echo '### parset into array' + parset arr1 echo ::: foo bar baz + echo ${arr1[0]} ${arr1[1]} ${arr1[2]} + + echo '### parset into vars with comma' + parset comma3,comma2,comma1 echo ::: baz bar foo + echo $comma1 $comma2 $comma3 + + echo '### parset into vars with space' + parset 'space3 space2 space1' echo ::: baz bar foo + echo $space1 $space2 $space3 + + echo '### parset with newlines' + parset 'newline3 newline2 newline1' seq ::: 3 2 1 + echo "$newline1" + echo "$newline2" + echo "$newline3" + + echo '### parset into indexed array vars' + parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo + echo ${myarray[*]} + echo ${myarray[4]} ${myarray[5]} ${myarray[5]} + + echo '### env_parset' + alias myecho='echo myecho "$myvar" "${myarr[1]}"' + myvar="myvar" + myarr=("myarr 0" "myarr 1" "myarr 2") + mynewline="`echo newline1;echo newline2;`" + env_parset arr1 myecho ::: foo bar baz + echo "${arr1[0]} ${arr1[1]} ${arr1[2]}" + env_parset comma3,comma2,comma1 myecho ::: baz bar foo + echo "$comma1 $comma2 $comma3" + env_parset 'space3 space2 space1' myecho ::: baz bar foo + echo "$space1 $space2 $space3" + env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1 + echo "$newline1" + echo "$newline2" + echo "$newline3" + env_parset 'myarray[6],myarray[5],myarray[4]' myecho ::: baz bar foo + echo "${myarray[*]}" + echo "${myarray[4]} ${myarray[5]} ${myarray[5]}" + + echo 'bug #52507: parset arr1 -v echo ::: fails' + parset arr1 -v seq ::: 1 2 3 + echo "${arr1[2]}" +} + +par_pipe_compress_blocks() { + echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations" + seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat + echo echo 1-4 + 1-4 + seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo + echo 4 times wc to stderr to stdout + (seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null + echo 1 2 3 4 + seq 4 | parallel -k -j3 --compress echo + echo 1 2 3 4 + seq 4 | parallel -k -j1 --compress echo + echo 1 2 + seq 2 | parallel -k -j1 --compress echo + echo 1 2 3 + seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index fdad3146..025eac24 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1366,6 +1366,23 @@ par_csv col1"x3"-new par_csv line col2-new2 par_csv line col3-col 4 par_csv 2"x3" board-Value with ,-Column 3 +par_csv_pipe Only pass full records to tail +par_csv_pipe Too small block size +par_csv_pipe parallel: Warning: A record was longer than 10000. Increasing to --blocksize 13001. +par_csv_pipe 2000" +par_csv_pipe 3000" +par_csv_pipe 4000" +par_csv_pipe 5000" +par_csv_pipe 6000" +par_csv_pipe 7000" +par_csv_pipe 8000" +par_csv_pipe 9000" +par_csv_pipe parallel: Warning: A record was longer than 13001. Increasing to --blocksize 16903. +par_csv_pipe 10000" +par_csv_pipe 11000" +par_csv_pipe More records in single block +par_csv_pipe 9000" +par_csv_pipe 11000" par_dryrun_append_joblog --dry-run should not append to joblog par_dryrun_append_joblog 1 par_dryrun_append_joblog 2 @@ -1408,14 +1425,6 @@ par_halt_one_job 1 par_halt_one_job parallel: This job failed: par_halt_one_job echo 1;exit 1 par_halt_one_job parallel: Starting no more jobs. Waiting for 0 jobs to finish. -par_header_parens bug #49538: --header and {= =} -par_header_parens OK -par_header_parens OK -par_header_parens file -par_header_parens sub/dir -par_header_parens file -par_header_parens file.ext -par_header_parens sub/dir/file par_inefficient_L bug #37325: Inefficiency of --pipe -L par_inefficient_L 276 276 996 par_inefficient_L FOO @@ -1441,28 +1450,6 @@ par_link_files_as_only_arg bug #50685: single ::::+ does not work par_link_files_as_only_arg 1 1 1 par_link_files_as_only_arg 2 2 2 par_link_files_as_only_arg 3 3 3 -par_macron -par_macron -par_macron -par_macron -par_macron -par_macron -par_macron "" -par_macron -par_macron -par_macron -par_macron -par_macron -par_macron -par_macron "" -par_macron /bin/bash: -c: line 0: syntax error near unexpected token `newline' -par_macron /bin/bash: -c: line 0: `echo <<>>' -par_macron <<>> -par_macron /bin/bash: : No such file or directory -par_macron <<>> <<>> -par_macron <<>> -par_macron <<>> <<>> -par_macron "<<>>" <<>> par_newline_in_command Command with newline and positional replacement strings par_newline_in_command O K par_no_command_given ### Test bugfix if no command given @@ -1475,78 +1462,6 @@ par_parcat_args_stdin OK2 par_parcat_rm bug #51691: parcat --rm remove fifo when opened par_parcat_rm OK1 par_parcat_rm OK file removed -par_parset ### parset into array -par_parset foo bar baz -par_parset ### parset into vars with comma -par_parset foo bar baz -par_parset ### parset into vars with space -par_parset foo bar baz -par_parset ### parset with newlines -par_parset 1 -par_parset 1 -par_parset 2 -par_parset 1 -par_parset 2 -par_parset 3 -par_parset ### parset into indexed array vars -par_parset foo bar baz -par_parset foo bar bar -par_parset ### env_parset -par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz -par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz -par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz -par_parset newline1 -par_parset newline2 -par_parset 1 -par_parset newline1 -par_parset newline2 -par_parset 1 -par_parset 2 -par_parset newline1 -par_parset newline2 -par_parset 1 -par_parset 2 -par_parset 3 -par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz -par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 bar -par_parset_v bug #52507: parset arr1 -v echo ::: fails -par_parset_v seq 3 -par_parset_v 1 -par_parset_v 2 -par_parset_v 3 -par_pipe_compress_blocks ### bug #41482: --pipe --compress blocks at different -j/seq combinations -par_pipe_compress_blocks 1 -par_pipe_compress_blocks echo 1-4 + 1-4 -par_pipe_compress_blocks echo 1 -par_pipe_compress_blocks 1 -par_pipe_compress_blocks echo 2 -par_pipe_compress_blocks 2 -par_pipe_compress_blocks echo 3 -par_pipe_compress_blocks 3 -par_pipe_compress_blocks echo 4 -par_pipe_compress_blocks 4 -par_pipe_compress_blocks 4 times wc to stderr to stdout -par_pipe_compress_blocks 1 1 2 -par_pipe_compress_blocks 1 1 2 -par_pipe_compress_blocks 1 1 2 -par_pipe_compress_blocks 1 1 2 -par_pipe_compress_blocks 1 2 3 4 -par_pipe_compress_blocks 1 -par_pipe_compress_blocks 2 -par_pipe_compress_blocks 3 -par_pipe_compress_blocks 4 -par_pipe_compress_blocks 1 2 3 4 -par_pipe_compress_blocks 1 -par_pipe_compress_blocks 2 -par_pipe_compress_blocks 3 -par_pipe_compress_blocks 4 -par_pipe_compress_blocks 1 2 -par_pipe_compress_blocks 1 -par_pipe_compress_blocks 2 -par_pipe_compress_blocks 1 2 3 -par_pipe_compress_blocks 1 -par_pipe_compress_blocks 2 -par_pipe_compress_blocks 3 par_pipe_no_command ### --pipe without command par_pipe_no_command parallel: Error: --pipe/--pipepart must have a command to pipe into (e.g. 'cat'). par_pipe_record_size_in_lines bug #34958: --pipe with record size measured in lines diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 9b7f3a36..d29485be 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -153,6 +153,14 @@ par_fifo_under_csh 1 par_fifo_under_csh 868832 par_fifo_under_csh 1 par_fifo_under_csh exit 22 +par_header_parens bug #49538: --header and {= =} +par_header_parens OK +par_header_parens OK +par_header_parens file +par_header_parens sub/dir +par_header_parens file +par_header_parens file.ext +par_header_parens sub/dir/file par_incomplete_linebuffer bug #51337: --lb does not kill jobs at sigpipe par_incomplete_linebuffer 1 par_incomplete_linebuffer 2 @@ -261,6 +269,28 @@ par_linebuffer_files lrz --files par_linebuffer_files lrz 1 par_linebuffer_files lrz --results par_linebuffer_files lrz 1 +par_macron +par_macron +par_macron +par_macron +par_macron +par_macron +par_macron "" +par_macron +par_macron +par_macron +par_macron +par_macron +par_macron +par_macron "" +par_macron /bin/bash: -c: line 0: syntax error near unexpected token `newline' +par_macron /bin/bash: -c: line 0: `echo <<>>' +par_macron <<>> +par_macron /bin/bash: : No such file or directory +par_macron <<>> <<>> +par_macron <<>> +par_macron <<>> <<>> +par_macron "<<>>" <<>> par_max_length_len_128k ### BUG: The length for -X is not close to max (131072) par_max_length_len_128k 1 12817 131016 par_max_length_len_128k 1 10946 131032 @@ -322,6 +352,78 @@ par_parset 9 par_parset Commands with newline require -0 par_parset line1 par_parset line2 +par_parset2 ### parset into array +par_parset2 foo bar baz +par_parset2 ### parset into vars with comma +par_parset2 foo bar baz +par_parset2 ### parset into vars with space +par_parset2 foo bar baz +par_parset2 ### parset with newlines +par_parset2 1 +par_parset2 1 +par_parset2 2 +par_parset2 1 +par_parset2 2 +par_parset2 3 +par_parset2 ### parset into indexed array vars +par_parset2 foo bar baz +par_parset2 foo bar bar +par_parset2 ### env_parset +par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz +par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz +par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz +par_parset2 newline1 +par_parset2 newline2 +par_parset2 1 +par_parset2 newline1 +par_parset2 newline2 +par_parset2 1 +par_parset2 2 +par_parset2 newline1 +par_parset2 newline2 +par_parset2 1 +par_parset2 2 +par_parset2 3 +par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz +par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 bar +par_parset2 bug #52507: parset arr1 -v echo ::: fails +par_parset2 seq 3 +par_parset2 1 +par_parset2 2 +par_parset2 3 +par_pipe_compress_blocks ### bug #41482: --pipe --compress blocks at different -j/seq combinations +par_pipe_compress_blocks 1 +par_pipe_compress_blocks echo 1-4 + 1-4 +par_pipe_compress_blocks echo 1 +par_pipe_compress_blocks 1 +par_pipe_compress_blocks echo 2 +par_pipe_compress_blocks 2 +par_pipe_compress_blocks echo 3 +par_pipe_compress_blocks 3 +par_pipe_compress_blocks echo 4 +par_pipe_compress_blocks 4 +par_pipe_compress_blocks 4 times wc to stderr to stdout +par_pipe_compress_blocks 1 1 2 +par_pipe_compress_blocks 1 1 2 +par_pipe_compress_blocks 1 1 2 +par_pipe_compress_blocks 1 1 2 +par_pipe_compress_blocks 1 2 3 4 +par_pipe_compress_blocks 1 +par_pipe_compress_blocks 2 +par_pipe_compress_blocks 3 +par_pipe_compress_blocks 4 +par_pipe_compress_blocks 1 2 3 4 +par_pipe_compress_blocks 1 +par_pipe_compress_blocks 2 +par_pipe_compress_blocks 3 +par_pipe_compress_blocks 4 +par_pipe_compress_blocks 1 2 +par_pipe_compress_blocks 1 +par_pipe_compress_blocks 2 +par_pipe_compress_blocks 1 2 3 +par_pipe_compress_blocks 1 +par_pipe_compress_blocks 2 +par_pipe_compress_blocks 3 par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory par_pxz_complains can not seek in input: Illegal seek diff --git a/testsuite/wanted-results/parallel-local-ssh2 b/testsuite/wanted-results/parallel-local-ssh2 index 43d38e4b..828abba6 100644 --- a/testsuite/wanted-results/parallel-local-ssh2 +++ b/testsuite/wanted-results/parallel-local-ssh2 @@ -90,14 +90,14 @@ par_tee_ssh 2 par_tee_ssh 3 par_tee_ssh 3 par_wd_no_such_dir ### --wd no-such-dir - csh -par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied +par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir par_wd_no_such_dir Exit code 1 par_wd_no_such_dir ### --wd no-such-dir - tcsh -par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied +par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir par_wd_no_such_dir Exit code 1 par_wd_no_such_dir ### --wd no-such-dir - bash -par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied +par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir par_wd_no_such_dir Exit code 1 diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 1d9b153f..559eebc3 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -315,7 +315,7 @@ par_tcsh_man Academic tradition requires you to cite works you base your article par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a par_tcsh_man scientific publication, please cite: par_tcsh_man -par_tcsh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, +par_tcsh_man O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, par_tcsh_man DOI https://doi.org/10.5281/zenodo.1146014 par_tcsh_man par_tcsh_man This helps funding further development; AND IT WON'T COST YOU A CENT. @@ -754,15 +754,15 @@ par_fish_underscore aliases and arrays in functions work par_fish_underscore aliases functions work par_fish_underscore aliases functions work par_fish_underscore ^ -par_fish_underscore in function “myfunc” +par_fish_underscore in function 'myfunc' par_fish_underscore called on standard input -par_fish_underscore with parameter list “work” +par_fish_underscore with parameter list 'work' par_fish_underscore par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho par_fish_underscore ^ -par_fish_underscore in function “myfunc” +par_fish_underscore in function 'myfunc' par_fish_underscore called on standard input -par_fish_underscore with parameter list “work” +par_fish_underscore with parameter list 'work' par_fish_underscore par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho par_fish_underscore ^ @@ -1020,7 +1020,7 @@ par_csh_man par_csh_man par_csh_man {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} par_csh_man DOI https://doi.org/10.5281/zenodo.1146014 -par_csh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, +par_csh_man O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, par_csh_man ### From man env_parallel par_csh_man --colsep regexp Split input on regexp for positional replacements par_csh_man --nonall Run the given command with no arguments on all sshlogins diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 2a55cc3d..3b1874c9 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -4,7 +4,7 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, + O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. @@ -27,7 +27,7 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, + O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT. @@ -47,7 +47,7 @@ scientific publication, please cite: author = {Tange, Ole}, title = {GNU Parallel 2018}, publisher = {Ole Tange}, - month = Apr, + month = Mar, year = 2018, ISBN = {9781387509881}, doi = {10.5281/zenodo.1146014}, diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 4d6980f7..b60f0dd7 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -75,11 +75,11 @@ parallel: Error: --pipepart is incompatible with --max-replace-args, --max-lines echo '### bug #42893: --block should not cause decimals in cat_partial' ### bug #42893: --block should not cause decimals in cat_partial seq 100000 >/tmp/parallel-decimal; parallel --dry-run -kvv --pipepart --block 0.12345M -a /tmp/parallel-decimal true; rm /tmp/parallel-decimal -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true) -1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true) +131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true) +131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true) +131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true) +131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true) +131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true) echo '### bug #42902: profiles containing arguments with space' ### bug #42902: profiles containing arguments with space echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH; parallel -JFULLPATH echo FULLPATH ::: $0 diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 4d691de7..62425b4c 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -171,7 +171,7 @@ Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: - O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881, + O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881, DOI https://doi.org/10.5281/zenodo.1146014 This helps funding further development; AND IT WON'T COST YOU A CENT.