mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: --pipe --csv: Pass only full CSV-records.
This commit is contained in:
Ole Tange
parent
bbcef1032c
commit
1e15645faa
12
README
12
README
|
|
@ -44,9 +44,9 @@ document.
|
|||
|
||||
Full installation of GNU Parallel is as simple as:
|
||||
|
||||
wget https://ftpmirror.gnu.org/parallel/parallel-20180322.tar.bz2
|
||||
bzip2 -dc parallel-20180322.tar.bz2 | tar xvf -
|
||||
cd parallel-20180322
|
||||
wget https://ftpmirror.gnu.org/parallel/parallel-20180422.tar.bz2
|
||||
bzip2 -dc parallel-20180422.tar.bz2 | tar xvf -
|
||||
cd parallel-20180422
|
||||
./configure && make && sudo make install
|
||||
|
||||
|
||||
|
|
@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as:
|
|||
If you are not root you can add ~/bin to your path and install in
|
||||
~/bin and ~/share:
|
||||
|
||||
wget https://ftpmirror.gnu.org/parallel/parallel-20180322.tar.bz2
|
||||
bzip2 -dc parallel-20180322.tar.bz2 | tar xvf -
|
||||
cd parallel-20180322
|
||||
wget https://ftpmirror.gnu.org/parallel/parallel-20180422.tar.bz2
|
||||
bzip2 -dc parallel-20180422.tar.bz2 | tar xvf -
|
||||
cd parallel-20180422
|
||||
./configure --prefix=$HOME && make && make install
|
||||
|
||||
Or if your system lacks 'make' you can simply copy src/parallel
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
AC_INIT([parallel], [20180322], [bug-parallel@gnu.org])
|
||||
AC_INIT([parallel], [20180422], [bug-parallel@gnu.org])
|
||||
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
|
||||
AC_CONFIG_HEADERS([config.h])
|
||||
AC_CONFIG_FILES([
|
||||
|
|
|
|||
|
|
@ -207,18 +207,29 @@ GNU Parallel 20180422 ('Trèbes') <<[stable]>> has been released. It is availabl
|
|||
|
||||
Quote of the month:
|
||||
|
||||
<<>>
|
||||
--
|
||||
Today I discovered GNU Parallel, and I don’t know what to do with all this spare time.
|
||||
--Ryan Booker
|
||||
|
||||
New in this release:
|
||||
|
||||
Launch at DIKU
|
||||
* --csv makes GNU Parallel parse the input sources as CSV. When used with --pipe it only passes full CSV-records.
|
||||
|
||||
https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently
|
||||
* Time in --bar is printed as 1d02h03m04s.
|
||||
|
||||
http://fliplinux.com/gnu-parallel-6.html
|
||||
* Optimization of --tee: It spawns a process less per value.
|
||||
|
||||
* The GNU Parallel 2018 book is now available: http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html
|
||||
|
||||
* Modern pentest tricks for faster, wider, greater engagement (15) https://conference.hitb.org/hitbsecconf2018ams/materials/D1%20COMMSEC%20-%20Thomas%20Debize%20-%20Modern%20Pentest%20Tricks%20for%20Faster,%20Wider,%20Greater%20Engagements.pdf
|
||||
|
||||
* Running many serial jobs efficiently https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently
|
||||
|
||||
* GNU Parallel: как сохранить результаты нескольких команд для переменной? http://fliplinux.com/gnu-parallel-6.html
|
||||
|
||||
* Running Multiple Commands in Parallel on a GNU Linux https://www.youtube.com/watch?v=sHpTywpb4_4
|
||||
|
||||
* Klaatu covers the documentation of GNU parallel in episode 12x15 http://gnuworldorder.info/
|
||||
|
||||
https://www.youtube.com/watch?v=sHpTywpb4_4
|
||||
|
||||
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20180323;
|
||||
$Global::version = 20180422;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
39
src/parallel
39
src/parallel
|
|
@ -392,7 +392,8 @@ sub find_split_positions {
|
|||
}
|
||||
} else {
|
||||
# If match $recend$recstart => Record position
|
||||
# TODO optimize to only look at the appended $dd_block_size + len $recendrecstart
|
||||
# TODO optimize to only look at the appended
|
||||
# $dd_block_size + len $recendrecstart
|
||||
# TODO increase $dd_block_size to optimize for longer records
|
||||
my $i = index64(\$buf,$recendrecstart);
|
||||
if($i != -1) {
|
||||
|
|
@ -434,7 +435,7 @@ sub cat_partial {
|
|||
sysseek(STDIN,shift,0) || die;
|
||||
$left = shift;
|
||||
while($read =
|
||||
sysread(STDIN,$buf, $left > 1048524 ? 1048524 : $left)){
|
||||
sysread(STDIN,$buf, $left > 131072 ? 131072 : $left)){
|
||||
$left -= $read;
|
||||
syswrite(STDOUT,$buf);
|
||||
}
|
||||
|
|
@ -500,6 +501,7 @@ sub spreadstdin {
|
|||
# Read n-line records
|
||||
my $n_lines = $buf =~ tr/\n/\n/;
|
||||
my $last_newline_pos = rindex64(\$buf,"\n");
|
||||
# Go backwards until there are full n-line records
|
||||
while($n_lines % $Global::max_lines) {
|
||||
$n_lines--;
|
||||
$last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1);
|
||||
|
|
@ -533,6 +535,21 @@ sub spreadstdin {
|
|||
$recstart,$recend,length $1);
|
||||
}
|
||||
}
|
||||
} elsif($opt::csv) {
|
||||
# Read a full CSV record
|
||||
# even number of " + end of line
|
||||
my $last_newline_pos = length $buf;
|
||||
do {
|
||||
# find last EOL
|
||||
$last_newline_pos = rindex64(\$buf,"\n",$last_newline_pos-1);
|
||||
# While uneven "
|
||||
} while((substr($buf,0,$last_newline_pos) =~ y/"/"/)%2
|
||||
and $last_newline_pos >= 0);
|
||||
# Chop at $last_newline_pos as that is where CSV record ends
|
||||
$anything_written +=
|
||||
write_record_to_pipe($chunk_number++,\$header,\$buf,
|
||||
$recstart,$recend,$last_newline_pos+1);
|
||||
shorten(\$buf,$last_newline_pos+1);
|
||||
} else {
|
||||
if($Global::max_number_of_args) {
|
||||
# -N => (start..*?end){n}
|
||||
|
|
@ -1427,7 +1444,7 @@ sub check_invalid_option_combinations {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20180323;
|
||||
$Global::version = 20180422;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -10817,7 +10834,7 @@ sub read_arg_from_fh {
|
|||
my $fh = shift;
|
||||
my $prepend;
|
||||
my $arg;
|
||||
my $double_quotes = 0;
|
||||
my $half_record = 0;
|
||||
do {{
|
||||
# This makes 10% faster
|
||||
if(not defined ($arg = <$fh>)) {
|
||||
|
|
@ -10829,16 +10846,20 @@ sub read_arg_from_fh {
|
|||
}
|
||||
if($opt::csv) {
|
||||
# We need to read a full CSV line.
|
||||
$double_quotes += ($arg =~ y/"/"/);
|
||||
if($double_quotes % 2) {
|
||||
# CSV halflines with quoting:
|
||||
if(($arg =~ y/"/"/) % 2 ) {
|
||||
# The number of " on the line is uneven:
|
||||
# If we were in a half_record => we have a full record now
|
||||
# If we were ouside a half_record => we are in a half record now
|
||||
$half_record = not $half_record;
|
||||
}
|
||||
if($half_record) {
|
||||
# CSV half-record with quoting:
|
||||
# col1,"col2 2""x3"" board newline <-this one
|
||||
# cont",col3
|
||||
$prepend .= $arg;
|
||||
redo;
|
||||
} else {
|
||||
# Now we have a full CSV line
|
||||
$double_quotes = 0;
|
||||
# Now we have a full CSV record
|
||||
}
|
||||
}
|
||||
# Remove delimiter
|
||||
|
|
|
|||
|
|
@ -607,6 +607,8 @@ Even quoted newlines are parsed correctly:
|
|||
echo 'Line 2 in field 1";value 2') |
|
||||
parallel --csv --colsep ';' echo Field 1: {1} Field 2: {2}
|
||||
|
||||
When used with B<--pipe> only pass full CSV-records.
|
||||
|
||||
=item B<--delimiter> I<delim>
|
||||
|
||||
=item B<-d> I<delim>
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20180323;
|
||||
$Global::version = 20180422;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -521,23 +521,6 @@ par_pipe_record_size_in_lines() {
|
|||
seq 10 | parallel -k --pipe -l 4 cat\;echo bug 34958-2
|
||||
}
|
||||
|
||||
par_pipe_compress_blocks() {
|
||||
echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
|
||||
seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat
|
||||
echo echo 1-4 + 1-4
|
||||
seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo
|
||||
echo 4 times wc to stderr to stdout
|
||||
(seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
|
||||
echo 1 2 3 4
|
||||
seq 4 | parallel -k -j3 --compress echo
|
||||
echo 1 2 3 4
|
||||
seq 4 | parallel -k -j1 --compress echo
|
||||
echo 1 2
|
||||
seq 2 | parallel -k -j1 --compress echo
|
||||
echo 1 2 3
|
||||
seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat
|
||||
}
|
||||
|
||||
par_pipe_no_command() {
|
||||
echo '### --pipe without command'
|
||||
|
||||
|
|
@ -570,18 +553,6 @@ par_tricolonplus() {
|
|||
parallel -k echo :::: <(seq 3) :::: <(seq 21 23) :::+ a b c ::: aa bb cc
|
||||
}
|
||||
|
||||
par_header_parens() {
|
||||
echo 'bug #49538: --header and {= =}'
|
||||
|
||||
parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
|
||||
parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
|
||||
parallel --header : echo {var/.} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var//} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var/.} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var/} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var.} ::: var sub/dir/file.ext
|
||||
}
|
||||
|
||||
par_colsep_0() {
|
||||
echo 'bug --colsep 0'
|
||||
|
||||
|
|
@ -656,21 +627,6 @@ par_link_files_as_only_arg() {
|
|||
parallel -k echo ::::+ <(seq 10) <(seq 3) <(seq 4)
|
||||
}
|
||||
|
||||
par_macron() {
|
||||
print_it() {
|
||||
parallel ::: "echo $1"
|
||||
parallel echo ::: "$1"
|
||||
parallel echo "$1" ::: "$1"
|
||||
parallel echo \""$1"\" ::: "$1"
|
||||
parallel -q echo ::: "$1"
|
||||
parallel -q echo "$1" ::: "$1"
|
||||
parallel -q echo \""$1"\" ::: "$1"
|
||||
}
|
||||
print_it "$(perl -e 'print "\257"')"
|
||||
print_it "$(perl -e 'print "\257\256"')"
|
||||
print_it "$(perl -e 'print "\257<\257<\257>\257>"')"
|
||||
}
|
||||
|
||||
par_basic_halt() {
|
||||
cpuburn=$(tempfile)
|
||||
cpuburn2=$(tempfile)
|
||||
|
|
@ -747,51 +703,6 @@ par_linebuffer_files() {
|
|||
stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
|
||||
}
|
||||
|
||||
par_parset() {
|
||||
. `which env_parallel.bash`
|
||||
echo '### parset into array'
|
||||
parset arr1 echo ::: foo bar baz
|
||||
echo ${arr1[0]} ${arr1[1]} ${arr1[2]}
|
||||
|
||||
echo '### parset into vars with comma'
|
||||
parset comma3,comma2,comma1 echo ::: baz bar foo
|
||||
echo $comma1 $comma2 $comma3
|
||||
|
||||
echo '### parset into vars with space'
|
||||
parset 'space3 space2 space1' echo ::: baz bar foo
|
||||
echo $space1 $space2 $space3
|
||||
|
||||
echo '### parset with newlines'
|
||||
parset 'newline3 newline2 newline1' seq ::: 3 2 1
|
||||
echo "$newline1"
|
||||
echo "$newline2"
|
||||
echo "$newline3"
|
||||
|
||||
echo '### parset into indexed array vars'
|
||||
parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo
|
||||
echo ${myarray[*]}
|
||||
echo ${myarray[4]} ${myarray[5]} ${myarray[5]}
|
||||
|
||||
echo '### env_parset'
|
||||
alias myecho='echo myecho "$myvar" "${myarr[1]}"'
|
||||
myvar="myvar"
|
||||
myarr=("myarr 0" "myarr 1" "myarr 2")
|
||||
mynewline="`echo newline1;echo newline2;`"
|
||||
env_parset arr1 myecho ::: foo bar baz
|
||||
echo "${arr1[0]} ${arr1[1]} ${arr1[2]}"
|
||||
env_parset comma3,comma2,comma1 myecho ::: baz bar foo
|
||||
echo "$comma1 $comma2 $comma3"
|
||||
env_parset 'space3 space2 space1' myecho ::: baz bar foo
|
||||
echo "$space1 $space2 $space3"
|
||||
env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1
|
||||
echo "$newline1"
|
||||
echo "$newline2"
|
||||
echo "$newline3"
|
||||
env_parset 'myarray[6],myarray[5],myarray[4]' myecho ::: baz bar foo
|
||||
echo "${myarray[*]}"
|
||||
echo "${myarray[4]} ${myarray[5]} ${myarray[5]}"
|
||||
}
|
||||
|
||||
par_halt_one_job() {
|
||||
echo '# Halt soon if there is a single job'
|
||||
echo should run 0 1 = job 1 2
|
||||
|
|
@ -823,13 +734,6 @@ par_pipepart_recend_recstart() {
|
|||
rm $tmp1 2>/dev/null
|
||||
}
|
||||
|
||||
par_parset_v() {
|
||||
echo 'bug #52507: parset arr1 -v echo ::: fails'
|
||||
. `which env_parallel.bash`
|
||||
parset arr1 -v seq ::: 1 2 3
|
||||
echo "${arr1[2]}"
|
||||
}
|
||||
|
||||
par_pipe_tag_v() {
|
||||
echo 'pipe with --tag -v'
|
||||
seq 3 | parallel -v --pipe --tagstring foo cat
|
||||
|
|
@ -861,6 +765,23 @@ par_csv() {
|
|||
parallel --csv echo {1}-{2}-{3}
|
||||
}
|
||||
|
||||
par_csv_pipe() {
|
||||
echo 'Only pass full records to tail'
|
||||
echo 'Too small block size'
|
||||
perl -e 'for $b(1..10) {
|
||||
print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000;
|
||||
print "\n"
|
||||
}' |
|
||||
stdout parallel --pipe --csv -k --block 10k tail -n1
|
||||
|
||||
echo 'More records in single block'
|
||||
perl -e 'for $b(1..10) {
|
||||
print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000;
|
||||
print "\n"
|
||||
}' |
|
||||
stdout parallel --pipe --csv -k --block 100k tail -n1
|
||||
}
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
compgen -A function | grep par_ | sort |
|
||||
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'
|
||||
|
|
|
|||
|
|
@ -237,6 +237,99 @@ _EOF
|
|||
ps aux | grep parallel[-]-lb-test
|
||||
}
|
||||
|
||||
par_macron() {
|
||||
print_it() {
|
||||
parallel ::: "echo $1"
|
||||
parallel echo ::: "$1"
|
||||
parallel echo "$1" ::: "$1"
|
||||
parallel echo \""$1"\" ::: "$1"
|
||||
parallel -q echo ::: "$1"
|
||||
parallel -q echo "$1" ::: "$1"
|
||||
parallel -q echo \""$1"\" ::: "$1"
|
||||
}
|
||||
print_it "$(perl -e 'print "\257"')"
|
||||
print_it "$(perl -e 'print "\257\256"')"
|
||||
print_it "$(perl -e 'print "\257<\257<\257>\257>"')"
|
||||
}
|
||||
|
||||
par_header_parens() {
|
||||
echo 'bug #49538: --header and {= =}'
|
||||
|
||||
parallel --header : echo '{=v2=}{=v1 $_=Q($_)=}' ::: v1 K ::: v2 O
|
||||
parallel --header : echo '{2}{=1 $_=Q($_)=}' ::: v2 K ::: v1 O
|
||||
parallel --header : echo {var/.} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var//} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var/.} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var/} ::: var sub/dir/file.ext
|
||||
parallel --header : echo {var.} ::: var sub/dir/file.ext
|
||||
}
|
||||
|
||||
par_parset2() {
|
||||
. `which env_parallel.bash`
|
||||
echo '### parset into array'
|
||||
parset arr1 echo ::: foo bar baz
|
||||
echo ${arr1[0]} ${arr1[1]} ${arr1[2]}
|
||||
|
||||
echo '### parset into vars with comma'
|
||||
parset comma3,comma2,comma1 echo ::: baz bar foo
|
||||
echo $comma1 $comma2 $comma3
|
||||
|
||||
echo '### parset into vars with space'
|
||||
parset 'space3 space2 space1' echo ::: baz bar foo
|
||||
echo $space1 $space2 $space3
|
||||
|
||||
echo '### parset with newlines'
|
||||
parset 'newline3 newline2 newline1' seq ::: 3 2 1
|
||||
echo "$newline1"
|
||||
echo "$newline2"
|
||||
echo "$newline3"
|
||||
|
||||
echo '### parset into indexed array vars'
|
||||
parset 'myarray[6],myarray[5],myarray[4]' echo ::: baz bar foo
|
||||
echo ${myarray[*]}
|
||||
echo ${myarray[4]} ${myarray[5]} ${myarray[5]}
|
||||
|
||||
echo '### env_parset'
|
||||
alias myecho='echo myecho "$myvar" "${myarr[1]}"'
|
||||
myvar="myvar"
|
||||
myarr=("myarr 0" "myarr 1" "myarr 2")
|
||||
mynewline="`echo newline1;echo newline2;`"
|
||||
env_parset arr1 myecho ::: foo bar baz
|
||||
echo "${arr1[0]} ${arr1[1]} ${arr1[2]}"
|
||||
env_parset comma3,comma2,comma1 myecho ::: baz bar foo
|
||||
echo "$comma1 $comma2 $comma3"
|
||||
env_parset 'space3 space2 space1' myecho ::: baz bar foo
|
||||
echo "$space1 $space2 $space3"
|
||||
env_parset 'newline3 newline2 newline1' 'echo "$mynewline";seq' ::: 3 2 1
|
||||
echo "$newline1"
|
||||
echo "$newline2"
|
||||
echo "$newline3"
|
||||
env_parset 'myarray[6],myarray[5],myarray[4]' myecho ::: baz bar foo
|
||||
echo "${myarray[*]}"
|
||||
echo "${myarray[4]} ${myarray[5]} ${myarray[5]}"
|
||||
|
||||
echo 'bug #52507: parset arr1 -v echo ::: fails'
|
||||
parset arr1 -v seq ::: 1 2 3
|
||||
echo "${arr1[2]}"
|
||||
}
|
||||
|
||||
par_pipe_compress_blocks() {
|
||||
echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
|
||||
seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat
|
||||
echo echo 1-4 + 1-4
|
||||
seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo
|
||||
echo 4 times wc to stderr to stdout
|
||||
(seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
|
||||
echo 1 2 3 4
|
||||
seq 4 | parallel -k -j3 --compress echo
|
||||
echo 1 2 3 4
|
||||
seq 4 | parallel -k -j1 --compress echo
|
||||
echo 1 2
|
||||
seq 2 | parallel -k -j1 --compress echo
|
||||
echo 1 2 3
|
||||
seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat
|
||||
}
|
||||
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
compgen -A function | grep par_ | sort |
|
||||
|
|
|
|||
|
|
@ -1366,6 +1366,23 @@ par_csv col1"x3"-new
|
|||
par_csv line col2-new2
|
||||
par_csv line col3-col 4
|
||||
par_csv 2"x3" board-Value with ,-Column 3
|
||||
par_csv_pipe Only pass full records to tail
|
||||
par_csv_pipe Too small block size
|
||||
par_csv_pipe parallel: Warning: A record was longer than 10000. Increasing to --blocksize 13001.
|
||||
par_csv_pipe 2000"
|
||||
par_csv_pipe 3000"
|
||||
par_csv_pipe 4000"
|
||||
par_csv_pipe 5000"
|
||||
par_csv_pipe 6000"
|
||||
par_csv_pipe 7000"
|
||||
par_csv_pipe 8000"
|
||||
par_csv_pipe 9000"
|
||||
par_csv_pipe parallel: Warning: A record was longer than 13001. Increasing to --blocksize 16903.
|
||||
par_csv_pipe 10000"
|
||||
par_csv_pipe 11000"
|
||||
par_csv_pipe More records in single block
|
||||
par_csv_pipe 9000"
|
||||
par_csv_pipe 11000"
|
||||
par_dryrun_append_joblog --dry-run should not append to joblog
|
||||
par_dryrun_append_joblog 1
|
||||
par_dryrun_append_joblog 2
|
||||
|
|
@ -1408,14 +1425,6 @@ par_halt_one_job 1
|
|||
par_halt_one_job parallel: This job failed:
|
||||
par_halt_one_job echo 1;exit 1
|
||||
par_halt_one_job parallel: Starting no more jobs. Waiting for 0 jobs to finish.
|
||||
par_header_parens bug #49538: --header and {= =}
|
||||
par_header_parens OK
|
||||
par_header_parens OK
|
||||
par_header_parens file
|
||||
par_header_parens sub/dir
|
||||
par_header_parens file
|
||||
par_header_parens file.ext
|
||||
par_header_parens sub/dir/file
|
||||
par_inefficient_L bug #37325: Inefficiency of --pipe -L
|
||||
par_inefficient_L 276 276 996
|
||||
par_inefficient_L FOO
|
||||
|
|
@ -1441,28 +1450,6 @@ par_link_files_as_only_arg bug #50685: single ::::+ does not work
|
|||
par_link_files_as_only_arg 1 1 1
|
||||
par_link_files_as_only_arg 2 2 2
|
||||
par_link_files_as_only_arg 3 3 3
|
||||
par_macron ¯
|
||||
par_macron ¯
|
||||
par_macron ¯ ¯
|
||||
par_macron ¯ ¯
|
||||
par_macron ¯
|
||||
par_macron ¯ ¯
|
||||
par_macron "¯" ¯
|
||||
par_macron ¯®
|
||||
par_macron ¯®
|
||||
par_macron ¯® ¯®
|
||||
par_macron ¯® ¯®
|
||||
par_macron ¯®
|
||||
par_macron ¯® ¯
|
||||
par_macron "¯®" ¯
|
||||
par_macron /bin/bash: -c: line 0: syntax error near unexpected token `newline'
|
||||
par_macron /bin/bash: -c: line 0: `echo ¯<¯<¯>¯>'
|
||||
par_macron ¯<¯<¯>¯>
|
||||
par_macron /bin/bash: ¯: No such file or directory
|
||||
par_macron ¯<¯<¯>¯> ¯<¯<¯>¯>
|
||||
par_macron ¯<¯<¯>¯>
|
||||
par_macron ¯<¯<¯>¯> ¯<¯<¯>¯>
|
||||
par_macron "¯<¯<¯>¯>" ¯<¯<¯>¯>
|
||||
par_newline_in_command Command with newline and positional replacement strings
|
||||
par_newline_in_command O K
|
||||
par_no_command_given ### Test bugfix if no command given
|
||||
|
|
@ -1475,78 +1462,6 @@ par_parcat_args_stdin OK2
|
|||
par_parcat_rm bug #51691: parcat --rm remove fifo when opened
|
||||
par_parcat_rm OK1
|
||||
par_parcat_rm OK file removed
|
||||
par_parset ### parset into array
|
||||
par_parset foo bar baz
|
||||
par_parset ### parset into vars with comma
|
||||
par_parset foo bar baz
|
||||
par_parset ### parset into vars with space
|
||||
par_parset foo bar baz
|
||||
par_parset ### parset with newlines
|
||||
par_parset 1
|
||||
par_parset 1
|
||||
par_parset 2
|
||||
par_parset 1
|
||||
par_parset 2
|
||||
par_parset 3
|
||||
par_parset ### parset into indexed array vars
|
||||
par_parset foo bar baz
|
||||
par_parset foo bar bar
|
||||
par_parset ### env_parset
|
||||
par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset newline1
|
||||
par_parset newline2
|
||||
par_parset 1
|
||||
par_parset newline1
|
||||
par_parset newline2
|
||||
par_parset 1
|
||||
par_parset 2
|
||||
par_parset newline1
|
||||
par_parset newline2
|
||||
par_parset 1
|
||||
par_parset 2
|
||||
par_parset 3
|
||||
par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 bar
|
||||
par_parset_v bug #52507: parset arr1 -v echo ::: fails
|
||||
par_parset_v seq 3
|
||||
par_parset_v 1
|
||||
par_parset_v 2
|
||||
par_parset_v 3
|
||||
par_pipe_compress_blocks ### bug #41482: --pipe --compress blocks at different -j/seq combinations
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks echo 1-4 + 1-4
|
||||
par_pipe_compress_blocks echo 1
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks echo 2
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks echo 3
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_compress_blocks echo 4
|
||||
par_pipe_compress_blocks 4
|
||||
par_pipe_compress_blocks 4 times wc to stderr to stdout
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 2 3 4
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_compress_blocks 4
|
||||
par_pipe_compress_blocks 1 2 3 4
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_compress_blocks 4
|
||||
par_pipe_compress_blocks 1 2
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 1 2 3
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_no_command ### --pipe without command
|
||||
par_pipe_no_command parallel: Error: --pipe/--pipepart must have a command to pipe into (e.g. 'cat').
|
||||
par_pipe_record_size_in_lines bug #34958: --pipe with record size measured in lines
|
||||
|
|
|
|||
|
|
@ -153,6 +153,14 @@ par_fifo_under_csh 1
|
|||
par_fifo_under_csh 868832
|
||||
par_fifo_under_csh 1
|
||||
par_fifo_under_csh exit 22
|
||||
par_header_parens bug #49538: --header and {= =}
|
||||
par_header_parens OK
|
||||
par_header_parens OK
|
||||
par_header_parens file
|
||||
par_header_parens sub/dir
|
||||
par_header_parens file
|
||||
par_header_parens file.ext
|
||||
par_header_parens sub/dir/file
|
||||
par_incomplete_linebuffer bug #51337: --lb does not kill jobs at sigpipe
|
||||
par_incomplete_linebuffer 1
|
||||
par_incomplete_linebuffer 2
|
||||
|
|
@ -261,6 +269,28 @@ par_linebuffer_files lrz --files
|
|||
par_linebuffer_files lrz 1
|
||||
par_linebuffer_files lrz --results
|
||||
par_linebuffer_files lrz 1
|
||||
par_macron ¯
|
||||
par_macron ¯
|
||||
par_macron ¯ ¯
|
||||
par_macron ¯ ¯
|
||||
par_macron ¯
|
||||
par_macron ¯ ¯
|
||||
par_macron "¯" ¯
|
||||
par_macron ¯®
|
||||
par_macron ¯®
|
||||
par_macron ¯® ¯®
|
||||
par_macron ¯® ¯®
|
||||
par_macron ¯®
|
||||
par_macron ¯® ¯
|
||||
par_macron "¯®" ¯
|
||||
par_macron /bin/bash: -c: line 0: syntax error near unexpected token `newline'
|
||||
par_macron /bin/bash: -c: line 0: `echo ¯<¯<¯>¯>'
|
||||
par_macron ¯<¯<¯>¯>
|
||||
par_macron /bin/bash: ¯: No such file or directory
|
||||
par_macron ¯<¯<¯>¯> ¯<¯<¯>¯>
|
||||
par_macron ¯<¯<¯>¯>
|
||||
par_macron ¯<¯<¯>¯> ¯<¯<¯>¯>
|
||||
par_macron "¯<¯<¯>¯>" ¯<¯<¯>¯>
|
||||
par_max_length_len_128k ### BUG: The length for -X is not close to max (131072)
|
||||
par_max_length_len_128k 1 12817 131016
|
||||
par_max_length_len_128k 1 10946 131032
|
||||
|
|
@ -322,6 +352,78 @@ par_parset 9
|
|||
par_parset Commands with newline require -0
|
||||
par_parset line1
|
||||
par_parset line2
|
||||
par_parset2 ### parset into array
|
||||
par_parset2 foo bar baz
|
||||
par_parset2 ### parset into vars with comma
|
||||
par_parset2 foo bar baz
|
||||
par_parset2 ### parset into vars with space
|
||||
par_parset2 foo bar baz
|
||||
par_parset2 ### parset with newlines
|
||||
par_parset2 1
|
||||
par_parset2 1
|
||||
par_parset2 2
|
||||
par_parset2 1
|
||||
par_parset2 2
|
||||
par_parset2 3
|
||||
par_parset2 ### parset into indexed array vars
|
||||
par_parset2 foo bar baz
|
||||
par_parset2 foo bar bar
|
||||
par_parset2 ### env_parset
|
||||
par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset2 newline1
|
||||
par_parset2 newline2
|
||||
par_parset2 1
|
||||
par_parset2 newline1
|
||||
par_parset2 newline2
|
||||
par_parset2 1
|
||||
par_parset2 2
|
||||
par_parset2 newline1
|
||||
par_parset2 newline2
|
||||
par_parset2 1
|
||||
par_parset2 2
|
||||
par_parset2 3
|
||||
par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 baz
|
||||
par_parset2 myecho myvar myarr 1 foo myecho myvar myarr 1 bar myecho myvar myarr 1 bar
|
||||
par_parset2 bug #52507: parset arr1 -v echo ::: fails
|
||||
par_parset2 seq 3
|
||||
par_parset2 1
|
||||
par_parset2 2
|
||||
par_parset2 3
|
||||
par_pipe_compress_blocks ### bug #41482: --pipe --compress blocks at different -j/seq combinations
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks echo 1-4 + 1-4
|
||||
par_pipe_compress_blocks echo 1
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks echo 2
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks echo 3
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_compress_blocks echo 4
|
||||
par_pipe_compress_blocks 4
|
||||
par_pipe_compress_blocks 4 times wc to stderr to stdout
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 1 2
|
||||
par_pipe_compress_blocks 1 2 3 4
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_compress_blocks 4
|
||||
par_pipe_compress_blocks 1 2 3 4
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 3
|
||||
par_pipe_compress_blocks 4
|
||||
par_pipe_compress_blocks 1 2
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 1 2 3
|
||||
par_pipe_compress_blocks 1
|
||||
par_pipe_compress_blocks 2
|
||||
par_pipe_compress_blocks 3
|
||||
par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway
|
||||
par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory
|
||||
par_pxz_complains can not seek in input: Illegal seek
|
||||
|
|
|
|||
|
|
@ -90,14 +90,14 @@ par_tee_ssh 2
|
|||
par_tee_ssh 3
|
||||
par_tee_ssh 3
|
||||
par_wd_no_such_dir ### --wd no-such-dir - csh
|
||||
par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied
|
||||
par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied
|
||||
par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir
|
||||
par_wd_no_such_dir Exit code 1
|
||||
par_wd_no_such_dir ### --wd no-such-dir - tcsh
|
||||
par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied
|
||||
par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied
|
||||
par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir
|
||||
par_wd_no_such_dir Exit code 1
|
||||
par_wd_no_such_dir ### --wd no-such-dir - bash
|
||||
par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied
|
||||
par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied
|
||||
par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir
|
||||
par_wd_no_such_dir Exit code 1
|
||||
|
|
|
|||
|
|
@ -315,7 +315,7 @@ par_tcsh_man Academic tradition requires you to cite works you base your article
|
|||
par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a
|
||||
par_tcsh_man scientific publication, please cite:
|
||||
par_tcsh_man
|
||||
par_tcsh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
|
||||
par_tcsh_man O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
|
||||
par_tcsh_man DOI https://doi.org/10.5281/zenodo.1146014
|
||||
par_tcsh_man
|
||||
par_tcsh_man This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
|
|
@ -754,15 +754,15 @@ par_fish_underscore aliases and arrays in functions work
|
|||
par_fish_underscore aliases functions work
|
||||
par_fish_underscore aliases functions work
|
||||
par_fish_underscore ^
|
||||
par_fish_underscore in function “myfunc”
|
||||
par_fish_underscore in function 'myfunc'
|
||||
par_fish_underscore called on standard input
|
||||
par_fish_underscore with parameter list “work”
|
||||
par_fish_underscore with parameter list 'work'
|
||||
par_fish_underscore
|
||||
par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho
|
||||
par_fish_underscore ^
|
||||
par_fish_underscore in function “myfunc”
|
||||
par_fish_underscore in function 'myfunc'
|
||||
par_fish_underscore called on standard input
|
||||
par_fish_underscore with parameter list “work”
|
||||
par_fish_underscore with parameter list 'work'
|
||||
par_fish_underscore
|
||||
par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho
|
||||
par_fish_underscore ^
|
||||
|
|
@ -1020,7 +1020,7 @@ par_csh_man
|
|||
par_csh_man
|
||||
par_csh_man {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
|
||||
par_csh_man DOI https://doi.org/10.5281/zenodo.1146014
|
||||
par_csh_man O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
|
||||
par_csh_man O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
|
||||
par_csh_man ### From man env_parallel
|
||||
par_csh_man --colsep regexp Split input on regexp for positional replacements
|
||||
par_csh_man --nonall Run the given command with no arguments on all sshlogins
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Academic tradition requires you to cite works you base your article on.
|
|||
If you use programs that use GNU Parallel to process data for an article in a
|
||||
scientific publication, please cite:
|
||||
|
||||
O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
|
||||
O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
|
||||
DOI https://doi.org/10.5281/zenodo.1146014
|
||||
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
|
|
@ -27,7 +27,7 @@ Academic tradition requires you to cite works you base your article on.
|
|||
If you use programs that use GNU Parallel to process data for an article in a
|
||||
scientific publication, please cite:
|
||||
|
||||
O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
|
||||
O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
|
||||
DOI https://doi.org/10.5281/zenodo.1146014
|
||||
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
|
|
@ -47,7 +47,7 @@ scientific publication, please cite:
|
|||
author = {Tange, Ole},
|
||||
title = {GNU Parallel 2018},
|
||||
publisher = {Ole Tange},
|
||||
month = Apr,
|
||||
month = Mar,
|
||||
year = 2018,
|
||||
ISBN = {9781387509881},
|
||||
doi = {10.5281/zenodo.1146014},
|
||||
|
|
|
|||
|
|
@ -75,11 +75,11 @@ parallel: Error: --pipepart is incompatible with --max-replace-args, --max-lines
|
|||
echo '### bug #42893: --block should not cause decimals in cat_partial'
|
||||
### bug #42893: --block should not cause decimals in cat_partial
|
||||
seq 100000 >/tmp/parallel-decimal; parallel --dry-run -kvv --pipepart --block 0.12345M -a /tmp/parallel-decimal true; rm /tmp/parallel-decimal
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>1048524?1048524:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 129450 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 258900 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 388350 129450 |(true)
|
||||
</tmp/parallel-decimal perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 517800 71095 |(true)
|
||||
echo '### bug #42902: profiles containing arguments with space'
|
||||
### bug #42902: profiles containing arguments with space
|
||||
echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH; parallel -JFULLPATH echo FULLPATH ::: $0
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ Academic tradition requires you to cite works you base your article on.
|
|||
If you use programs that use GNU Parallel to process data for an article in a
|
||||
scientific publication, please cite:
|
||||
|
||||
O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
|
||||
O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
|
||||
DOI https://doi.org/10.5281/zenodo.1146014
|
||||
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
|
|
|
|||
Loading…
Reference in a new issue