diff --git a/README b/README index 97c42335..33e6d43e 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20140622.tar.bz2 - bzip2 -dc parallel-20140622.tar.bz2 | tar xvf - - cd parallel-20140622 + wget http://ftpmirror.gnu.org/parallel/parallel-20140711.tar.bz2 + bzip2 -dc parallel-20140711.tar.bz2 | tar xvf - + cd parallel-20140711 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20140622.tar.bz2 - bzip2 -dc parallel-20140622.tar.bz2 | tar xvf - - cd parallel-20140622 + wget http://ftpmirror.gnu.org/parallel/parallel-20140711.tar.bz2 + bzip2 -dc parallel-20140711.tar.bz2 | tar xvf - + cd parallel-20140711 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 8b9e7683..98ebe44b 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20140622. +# Generated by GNU Autoconf 2.69 for parallel 20140711. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20140622' -PACKAGE_STRING='parallel 20140622' +PACKAGE_VERSION='20140711' +PACKAGE_STRING='parallel 20140711' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1197,7 +1197,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20140622 to adapt to many kinds of systems. +\`configure' configures parallel 20140711 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1263,7 +1263,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20140622:";; + short | recursive ) echo "Configuration of parallel 20140711:";; esac cat <<\_ACEOF @@ -1337,7 +1337,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20140622 +parallel configure 20140711 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1354,7 +1354,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20140622, which was +It was created by parallel $as_me 20140711, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2169,7 +2169,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20140622' + VERSION='20140711' cat >>confdefs.h <<_ACEOF @@ -2741,7 +2741,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20140622, which was +This file was extended by parallel $as_me 20140711, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2803,7 +2803,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20140622 +parallel config.status 20140711 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index c418ade1..a69d17f3 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20140622], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20140711], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index c54d2b3f..d77e7fe7 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -216,9 +216,11 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20140722 ('') released +Subject: GNU Parallel 20140722 ('Eyal Gilad Naftali') released -GNU Parallel 20140722 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20140722 ('Eyal Gilad Naftali') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ + +This release contains a major change in central parts of the code and should be considered alpha quality. As always it passes the testsuite, so most functionality clearly works. Haiku of the month: @@ -229,8 +231,28 @@ Haiku of the month: New in this release: +* {= perl expression =} can be used as replacement string. The expression should modify $_. E.g. {= s/\.gz$// =} to remove .gz from the string. This makes replacement strings extremely flexible. + +* Positional perl expressions (similar to {2}) are given as {=2 perl expression=} where 2 is the position. + +* Replacement strings can be defined using --rpl. E.g. parallel --rpl {.gz} 's/\.gz$//' echo {.gz} ::: *.gz + +* The parenthesis around {= perl expression =} can be changed with --parens. + +* GNU Parallel was cited in: bammds: A tool for assessing the ancestry of low depth whole genome data using multidimensional scaling (MDS) http://bioinformatics.oxfordjournals.org/content/early/2014/06/28/bioinformatics.btu410.abstract + +* GNU Parallel was cited in: Molecular ferroelectric contributions to anomalous hysteresis in hybrid perovskite solar cells http://people.bath.ac.uk/aw558/publications/2014/arxiv_hysteresis_14.pdf + +* GNU Parallel was cited in: Energy Sorghum-a genetic model for the design of C4 grass bioenergy crops http://jxb.oxfordjournals.org/content/early/2014/06/20/jxb.eru229.short + * GNU Parallel was cited in: Web-scale Content Reuse Detection www.isi.edu/~johnh/PAPERS/Ardi14a.pdf +* Webcast at 2014-08-20 covering GNU Parallel: Data Science at the Command Line http://www.oreilly.com/pub/e/3115 + +* Paralelizace běžných činností v konzoli pomocí GNU Parallel http://www.abclinuxu.cz/clanky/paralelizace-beznych-cinnosti-v-konzoli-pomoci-gnu-parallel + +* [原] Ubuntu 下使用 parallel 命令的注意事项 http://blog.ailms.me/2014/06/28/ubuntu-with-parallel.html + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel.pdf b/src/parallel.pdf index fd865837..bc3f7e84 100644 Binary files a/src/parallel.pdf and b/src/parallel.pdf differ diff --git a/src/parallel.pod b/src/parallel.pod index 74394b2d..ee58a328 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2373,6 +2373,48 @@ B This will run 1.5 job per core, and give 1000 arguments to B. +=head1 EXAMPLE: Grepping n lines for m regular expressions. + +The simplest solution to grep a big file for a lot of regexps is: + + grep -f regexps.txt bigfile + +Or if the regexps are fixed strings: + + grep -F -f regexps.txt bigfile + +There are 2 limiting factors: CPU and disk I/O. CPU is easy to +measure: If the grep takes >90% CPU (e.g. when running top), then the +CPU is a limiting factor, and parallelization will speed this up. If +not, then disk I/O is the limiting factor, and depending on the disk +system it may be faster or slower to parallelize. The only way to know +for certain is to measure. + +If the CPU is the limiting factor parallelization should be done on the regexps: + + cat regexp.txt | parallel --pipe -L1000 --round-robin grep -f - bigfile + +This will start one grep per CPU and read bigfile one time per CPU, +but as that is done in parallel, all reads except the first will be +cached in RAM. Depending on the size of regexp.txt it may be faster to +use --block 10m instead of -L1000. If regexp.txt is too big to fit in +RAM, remove --round-robin and adjust -L1000. This will cause bigfile +to be read more times. + +Some storage systems perform better when reading multiple chunks in +parallel. This is true for some RAID systems and for some network file +systems. To parallelize the reading of bigfile: + + parallel --pipepart --block 100M -a bigfile grep -f regexp.txt + +This will split bigfile into 100MB chunks and run grep on each of +these chunks. To parallelize both reading of bigfile and regexp.txt +combine the two using --fifo: + + parallel --pipepart --block 100M -a bigfile --fifo cat regexp.txt \ + \| parallel --pipe -L1000 --round-robin grep -f - {} + + =head1 EXAMPLE: Using remote computers To run commands on a remote computer SSH needs to be set up and you diff --git a/src/parallel.texi b/src/parallel.texi index 5abf177b..2d2ac45c 100644 --- a/src/parallel.texi +++ b/src/parallel.texi @@ -38,6 +38,7 @@ * EXAMPLE@asis{:} Tag output lines:: * EXAMPLE@asis{:} Keep order of output same as order of input:: * EXAMPLE@asis{:} Parallel grep:: +* EXAMPLE@asis{:} Grepping n lines for m regular expressions.:: * EXAMPLE@asis{:} Using remote computers:: * EXAMPLE@asis{:} Transferring of files:: * EXAMPLE@asis{:} Distributing work to local and remote computers:: @@ -2604,6 +2605,57 @@ GNU @strong{parallel} can often speed this up. This will run 1.5 job per core, and give 1000 arguments to @strong{grep}. +@node EXAMPLE: Grepping n lines for m regular expressions. +@chapter EXAMPLE: Grepping n lines for m regular expressions. + +The simplest solution to grep a big file for a lot of regexps is: + +@verbatim + grep -f regexps.txt bigfile +@end verbatim + +Or if the regexps are fixed strings: + +@verbatim + grep -F -f regexps.txt bigfile +@end verbatim + +There are 2 limiting factors: CPU and disk I/O. CPU is easy to +measure: If the grep takes >90% CPU (e.g. when running top), then the +CPU is a limiting factor, and parallelization will speed this up. If +not, then disk I/O is the limiting factor, and depending on the disk +system it may be faster or slower to parallelize. The only way to know +for certain is to measure. + +If the CPU is the limiting factor parallelization should be done on the regexps: + +@verbatim + cat regexp.txt | parallel --pipe -L1000 --round-robin grep -f - bigfile +@end verbatim + +This will start one grep per CPU and read bigfile one time per CPU, +but as that is done in parallel, all reads except the first will be +cached in RAM. Depending on the size of regexp.txt it may be faster to +use --block 10m instead of -L1000. If regexp.txt is too big to fit in +RAM, remove --round-robin and adjust -L1000. This will cause bigfile +to be read more times. + +Some storage systems perform better when reading multiple chunks in +parallel. This is true for some RAID systems and for some network file +systems. To parallelize the reading of bigfile: + +@verbatim + parallel --pipepart --block 100M -a bigfile grep -f regexp.txt +@end verbatim + +This will split bigfile into 100MB chunks and run grep on each of +these chunks. To parallelize both reading of bigfile and regexp.txt +combine the two using --fifo: + +@verbatim + parallel --pipepart --block 100M -a bigfile --fifo cat regexp.txt \| parallel --pipe -L1000 --round-robin grep -f - {} +@end verbatim + @node EXAMPLE: Using remote computers @chapter EXAMPLE: Using remote computers diff --git a/testsuite/Makefile b/testsuite/Makefile index dd558251..41cee044 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -51,13 +51,15 @@ startvm: stopvm: VBoxManage controlvm CentOS3-root:centos3 savestate VBoxManage controlvm RedHat9-root:redhat9 savestate + VBoxManage controlvm OracleXE savestate installparallel: ../src/parallel cd .. && make -j && sudo make -j install startdb: - echo shutdown abort | sudo su - oracle -c "sqlplus / as sysdba" + #echo shutdown abort | sudo su - oracle -c "sqlplus / as sysdba" sudo parallel /etc/init.d/{} restart ::: postgresql mysql oracle-xe + VBoxManage startvm OracleXE || true clean: rm -rf input-files/random_dirs_no_newline diff --git a/testsuite/tests-to-run/niceload01.sh b/testsuite/tests-to-run/niceload01.sh index 9a5d7a1d..52df77ef 100755 --- a/testsuite/tests-to-run/niceload01.sh +++ b/testsuite/tests-to-run/niceload01.sh @@ -21,7 +21,7 @@ freepl >/dev/null & # niceload -q -l 5 perl -e '$a=join"",<>; while(1){push @a,$a}' & -cat <<'EOF' | stdout parallel -j0 -k -L1 +cat <<'EOF' | stdout parallel -vj0 -k -L1 echo '### --rm and --runmem' niceload -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm niceload -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem @@ -34,13 +34,13 @@ EOF # force load > 10 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done -cat <<'EOF' | stdout parallel -j0 -k -L1 +cat <<'EOF' | stdout parallel -vj0 -k -L1 echo '### -H and --hard' niceload -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9.9 niceload --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9 EOF -cat <<'EOF' | stdout parallel -j0 -L1 +cat <<'EOF' | stdout parallel -vj0 -L1 echo '### -f and --factor' niceload -H --factor 10 -l6 echo factor 10 finish last niceload -H -f 0.01 -l6 echo f 0.1 finish first diff --git a/testsuite/tests-to-run/niceload03.sh b/testsuite/tests-to-run/niceload03.sh index 745b759b..274d01fb 100755 --- a/testsuite/tests-to-run/niceload03.sh +++ b/testsuite/tests-to-run/niceload03.sh @@ -8,7 +8,7 @@ int() { } export -f int -cat <<'EOF' | stdout parallel -kj0 -L1 +cat <<'EOF' | stdout parallel -k -vj0 -L1 # The seq 10000000 should take > 1 cpu sec to run. echo '### --soft -f and test if child is actually suspended and thus takes longer' niceload --soft -f 0.5 'seq 20000000 | wc;echo This should finish last' & diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index 75c6b039..ef4f3690 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -9,7 +9,7 @@ ping -c 1 freebsd7.tange.dk >/dev/null 2>&1 ssh freebsd7.tange.dk touch .parallel/will-cite scp -q .*/src/{parallel,sem,sql,niceload} freebsd7.tange.dk:bin/ -cat <<'EOF' | sed -e 's/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -S freebsd7.tange.dk -j9 +cat <<'EOF' | sed -e 's/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -S freebsd7.tange.dk -vj9 echo 'bug #40136: FreeBSD: No more processes' # Long line due to FreeBSD's /bin/sh stupidity sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' && sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' && sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' && sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' && sem --wait --id my_id diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 995c3562..9976d7c1 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -3,7 +3,7 @@ # Simple jobs that never fails # Each should be taking 0.3-1s and be possible to run in parallel # I.e.: No race conditions, no logins -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -vj4 -L1 echo '### Test exit val - true'; echo true | parallel; echo $? diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index f1edb27e..b23c833c 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -3,7 +3,7 @@ # Simple jobs that never fails # Each should be taking >100s and be possible to run in parallel # I.e.: No race conditions, no logins -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -vj4 -L1 echo '### Test if we can deal with output > 4 GB' ## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 843738c8..01a0dac6 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -3,5 +3,5 @@ # Simple jobs that never fails # Each should be taking 10-30s and be possible to run in parallel # I.e.: No race conditions, no logins -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -vj4 -L1 EOF diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 1614db57..d9717314 100644 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -10,7 +10,7 @@ mkdir -p $SHM sudo umount -l $SHM sudo mount -t tmpfs -o size=10% none $SHM -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -vj4 -L1 echo '### Test race condition on 8 CPU (my laptop)'; seq 1 5000000 > /tmp/parallel_test; seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index 77e06d2d..0633881e 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -3,7 +3,7 @@ # Simple jobs that never fails # Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -vj4 -L1 echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)' parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456; ls parallel_test_dir/1/ diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index ee48e6f0..6ed33739 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -4,7 +4,7 @@ rm -rf tmp mkdir tmp cd tmp -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j5 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj5 -k -L1 echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & diff --git a/testsuite/tests-to-run/parallel-local-ssh2.sh b/testsuite/tests-to-run/parallel-local-ssh2.sh index 720a1b23..d6dbdd0c 100644 --- a/testsuite/tests-to-run/parallel-local-ssh2.sh +++ b/testsuite/tests-to-run/parallel-local-ssh2.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -j8 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj8 -k -L1 echo '### --filter-hosts --slf <()' parallel --nonall --filter-hosts --slf <(echo localhost) echo OK diff --git a/testsuite/tests-to-run/parallel-local1.sh b/testsuite/tests-to-run/parallel-local1.sh index 9c882366..fd64a712 100644 --- a/testsuite/tests-to-run/parallel-local1.sh +++ b/testsuite/tests-to-run/parallel-local1.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | parallel -j0 -k +cat <<'EOF' | parallel -vj0 -k echo "### Test --basenamereplace" parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b diff --git a/testsuite/tests-to-run/parallel-local114.sh b/testsuite/tests-to-run/parallel-local114.sh index 86abc817..ae200834 100755 --- a/testsuite/tests-to-run/parallel-local114.sh +++ b/testsuite/tests-to-run/parallel-local114.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | parallel -vj0 -k -L1 echo "### --line-buffer" seq 10 | parallel -j20 --line-buffer 'seq {} 10 | pv -qL 10' > /tmp/parallel_l$$; seq 10 | parallel -j20 'seq {} 10 | pv -qL 10' > /tmp/parallel_$$; diff --git a/testsuite/tests-to-run/parallel-local13.sh b/testsuite/tests-to-run/parallel-local13.sh index e91c4283..2e7b3c99 100644 --- a/testsuite/tests-to-run/parallel-local13.sh +++ b/testsuite/tests-to-run/parallel-local13.sh @@ -5,7 +5,7 @@ echo '### Tests from xargs' rsync -Ha --delete input-files/xargs-inputs/ tmp/ cd tmp -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k -L1 echo '### -0 -n3 echo < files0.xi' stdout xargs -0 -n3 echo < files0.xi stdout parallel -k -0 -n3 echo < files0.xi diff --git a/testsuite/tests-to-run/parallel-local15.sh b/testsuite/tests-to-run/parallel-local15.sh index 5af13cf3..129a1ec0 100755 --- a/testsuite/tests-to-run/parallel-local15.sh +++ b/testsuite/tests-to-run/parallel-local15.sh @@ -17,7 +17,7 @@ echo '### Check that we can have more input than max procs (-j 0) - touch' perl -e 'print map {"more_than_5000-$_\n" } (4000..9999)' | parallel -vj 0 touch | sort | tail echo '### rm' perl -e 'print map {"more_than_5000-$_\n" } (4000..9900)' | parallel -j 0 rm | sort -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout nice parallel -k -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout nice parallel -vj0 -k -L1 ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort find . -type f | parallel --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort diff --git a/testsuite/tests-to-run/parallel-local150.sh b/testsuite/tests-to-run/parallel-local150.sh index 707fc5c2..9f1073c2 100644 --- a/testsuite/tests-to-run/parallel-local150.sh +++ b/testsuite/tests-to-run/parallel-local150.sh @@ -4,7 +4,7 @@ rsync -Ha --delete input-files/segfault/ tmp/ cd tmp # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j+0 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 echo '### bug #41565: Print happens in blocks - not after each job complete' echo 'The timing here is important: 2 3 4 5 6' ping -c 7 lo | parallel -j3 'echo {#}' | timestamp -dd | perl -pe '$_=int($_+0.2)."\n"' diff --git a/testsuite/tests-to-run/parallel-local164.sh b/testsuite/tests-to-run/parallel-local164.sh index 3ef0085a..4e981b70 100644 --- a/testsuite/tests-to-run/parallel-local164.sh +++ b/testsuite/tests-to-run/parallel-local164.sh @@ -1,7 +1,7 @@ #!/bin/bash # -L1 will join lines ending in ' ' -cat <<'EOF' | parallel -j10 -k -L1 +cat <<'EOF' | parallel -vj10 -k -L1 echo '### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name' echo "echo echo from ./local_test_profile" > local_test_profile; parallel --profile local_test_profile echo ::: 1; diff --git a/testsuite/tests-to-run/parallel-local19.sh b/testsuite/tests-to-run/parallel-local19.sh index ebd803f1..0df5463a 100755 --- a/testsuite/tests-to-run/parallel-local19.sh +++ b/testsuite/tests-to-run/parallel-local19.sh @@ -28,7 +28,7 @@ ls | parallel -kv rm -- {.}/abc-{.}-{} 2>&1 #test05.sh:find . -type d -print0 | perl -0 -pe 's:^./::' | parallel -0 -v rmdir -- {} 2>&1 \ # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | nice parallel -j0 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | nice parallel -vj0 -k -L1 echo '### Test compress' seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10' diff --git a/testsuite/tests-to-run/parallel-local2.sh b/testsuite/tests-to-run/parallel-local2.sh index 6aeb9983..fc684d80 100644 --- a/testsuite/tests-to-run/parallel-local2.sh +++ b/testsuite/tests-to-run/parallel-local2.sh @@ -16,7 +16,7 @@ highload () highload 2>/dev/null & sleep 1 -cat <<'EOF' | parallel -j0 -k -L1 +cat <<'EOF' | parallel -vj0 -k -L1 echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first job" /usr/bin/time -f %e parallel --load 100% true ::: a 2>&1 | perl -ne '$_ > 1 and print "More than 1 secs wall clock: OK\n"' diff --git a/testsuite/tests-to-run/parallel-local22.sh b/testsuite/tests-to-run/parallel-local22.sh index 7126ac5a..abeaefd8 100755 --- a/testsuite/tests-to-run/parallel-local22.sh +++ b/testsuite/tests-to-run/parallel-local22.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k -L1 echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; diff --git a/testsuite/tests-to-run/parallel-local23.sh b/testsuite/tests-to-run/parallel-local23.sh index 8779ddb1..782c1a07 100755 --- a/testsuite/tests-to-run/parallel-local23.sh +++ b/testsuite/tests-to-run/parallel-local23.sh @@ -3,7 +3,7 @@ rm -rf tmp 2>/dev/null cp -a input-files/testdir2 tmp -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k -L1 echo '### bug #42329: --line-buffer gives wrong output'; parallel --line-buffer --tag seq ::: 10000000 | wc -c; parallel --line-buffer seq ::: 10000000 | wc -c diff --git a/testsuite/tests-to-run/parallel-local4.sh b/testsuite/tests-to-run/parallel-local4.sh index 52d95b54..e1a8fcb8 100644 --- a/testsuite/tests-to-run/parallel-local4.sh +++ b/testsuite/tests-to-run/parallel-local4.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | parallel -j0 -vk +cat <<'EOF' | parallel -vj0 -vk echo '### bug #36595: silent loss of input with --pipe and --sshlogin' seq 10000 | xargs | parallel --pipe -S 10/localhost cat | wc diff --git a/testsuite/tests-to-run/parallel-local5.sh b/testsuite/tests-to-run/parallel-local5.sh index 62268b15..53387d26 100644 --- a/testsuite/tests-to-run/parallel-local5.sh +++ b/testsuite/tests-to-run/parallel-local5.sh @@ -5,7 +5,7 @@ echo '### Test --pipe' seq 1 1000000 >/tmp/parallel-seq shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j2 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj2 -k -L1 echo '### Test 200M records with too small block'; ( echo start; diff --git a/testsuite/tests-to-run/parallel-local9.sh b/testsuite/tests-to-run/parallel-local9.sh index 10860a8b..3e5ad02e 100644 --- a/testsuite/tests-to-run/parallel-local9.sh +++ b/testsuite/tests-to-run/parallel-local9.sh @@ -5,7 +5,7 @@ export PAR XAP="nice nice parallel --xapply" export XAP -cat <<'EOF' | sed -e 's/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1 +cat <<'EOF' | sed -e 's/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k -L1 echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index ec2248e8..e5be7fd0 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -13,7 +13,7 @@ echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 chmod 755 /tmp/myssh1 /tmp/myssh2 seq 1 100 | parallel --sshdelay 0.05 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -j2 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj2 -k -L1 echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname diff --git a/testsuite/tests-to-run/test15.sh b/testsuite/tests-to-run/test15.sh index 3c262a00..80d9bc09 100755 --- a/testsuite/tests-to-run/test15.sh +++ b/testsuite/tests-to-run/test15.sh @@ -33,7 +33,7 @@ send "y\n" expect "opt--interactive 3" _EOF echo -cat <<'EOF' | parallel -j0 -k -L1 +cat <<'EOF' | parallel -vj0 -k -L1 echo '### Test killing children with --timeout and exit value (failed if timed out)' pstree | grep sleep | grep -v anacron | grep -v screensave | wc; parallel --timeout 3 'true {} ; for i in `seq 100 120`; do bash -c "(sleep $i)" & sleep $i & done; wait; echo No good' ::: 1000000000 1000000001 ; diff --git a/testsuite/tests-to-run/test30.sh b/testsuite/tests-to-run/test30.sh index 71d430e6..054c1460 100644 --- a/testsuite/tests-to-run/test30.sh +++ b/testsuite/tests-to-run/test30.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | parallel -j0 -k +cat <<'EOF' | parallel -vj0 -k echo '### Test of --eta' seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l diff --git a/testsuite/tests-to-run/test45.sh b/testsuite/tests-to-run/test45.sh index 591f0d53..17f29a46 100644 --- a/testsuite/tests-to-run/test45.sh +++ b/testsuite/tests-to-run/test45.sh @@ -4,7 +4,7 @@ SERVER1=parallel-server3 SERVER2=parallel-server1 # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 echo "### BUG: The length for -X is not close to max (131072)"; seq 1 60000 | nice parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc seq 1 60000 | nice parallel -X echo a{}b{}c |head -n 1 |wc diff --git a/testsuite/tests-to-run/test60.sh b/testsuite/tests-to-run/test60.sh index 7c33b76b..93578ac4 100644 --- a/testsuite/tests-to-run/test60.sh +++ b/testsuite/tests-to-run/test60.sh @@ -6,7 +6,7 @@ SSHLOGIN1=parallel@$SERVER1 SSHLOGIN2=parallel@$SERVER2 # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -j0 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj0 -k -L1 echo '### Test --onall'; parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 diff --git a/testsuite/tests-to-run/test61.sh b/testsuite/tests-to-run/test61.sh index 3d1b1b7e..f1e2dddd 100644 --- a/testsuite/tests-to-run/test61.sh +++ b/testsuite/tests-to-run/test61.sh @@ -3,7 +3,7 @@ SERVER1=parallel-server3 SERVER2=parallel-server2 -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k echo '### Test --return of weirdly named file' stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b' diff --git a/testsuite/tests-to-run/test65.sh b/testsuite/tests-to-run/test65.sh index 2c1e928b..937dd9fd 100644 --- a/testsuite/tests-to-run/test65.sh +++ b/testsuite/tests-to-run/test65.sh @@ -1,7 +1,7 @@ #!/bin/bash # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k -L1 echo "### Test memory consumption stays (almost) the same for 30 and 300 jobs" out30=$( stdout memusg parallel -j2 true :::: <(perl -e '$a="x"x100000;for(1..30){print $a,"\n"}') ); out300=$( stdout memusg parallel -j2 true :::: <(perl -e '$a="x"x100000;for(1..300){print $a,"\n"}') ); diff --git a/testsuite/wanted-results/niceload01 b/testsuite/wanted-results/niceload01 index ae04f23c..3962f0eb 100644 --- a/testsuite/wanted-results/niceload01 +++ b/testsuite/wanted-results/niceload01 @@ -1,15 +1,27 @@ ### Test niceload -q This works +echo '### --rm and --runmem' ### --rm and --runmem + niceload -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm OK--rm + niceload -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem OK--runmem +echo '### -N and --noswap. Must give 0' ### -N and --noswap. Must give 0 + niceload -H -N vmstat 1 2 | tail -n1 | awk '{print "-N " $7*$8}' -N 0 + niceload -H --noswap vmstat 1 2 | tail -n1 | awk '{print "--noswap " $7*$8}' --noswap 0 +echo '### -H and --hard' ### -H and --hard + niceload -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9.9 OK-l9.9 + niceload --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9 OK-l9 +echo '### -f and --factor' ### -f and --factor + niceload -H -f 0.01 -l6 echo f 0.1 finish first f 0.1 finish first + niceload -H --factor 10 -l6 echo factor 10 finish last factor 10 finish last diff --git a/testsuite/wanted-results/niceload03 b/testsuite/wanted-results/niceload03 index 6160319e..a70408fc 100644 --- a/testsuite/wanted-results/niceload03 +++ b/testsuite/wanted-results/niceload03 @@ -1,21 +1,36 @@ +# The seq 10000000 should take > 1 cpu sec to run. +echo '### --soft -f and test if child is actually suspended and thus takes longer' ### --soft -f and test if child is actually suspended and thus takes longer + niceload --soft -f 0.5 'seq 20000000 | wc;echo This should finish last' & (sleep 1; seq 20000000 | wc;echo This should finish first) & wait 20000000 20000000 168888897 This should finish first 20000000 20000000 168888897 This should finish last +echo '### niceload with no arguments should give no output' ### niceload with no arguments should give no output + niceload +echo '### Test -t and -s' ### Test -t and -s + niceload -v -t 1 -s 2 sleep 4.5 Sleeping 1s Running 2s Sleeping 1s Running 2s +echo 'bug #38908: niceload: Ctrl-C/TERM should resume jobs if using -p - Order may change, but not output' bug #38908: niceload: Ctrl-C/TERM should resume jobs if using -p - Order may change, but not output + # This should take 10 seconds to run + delay from niceload + # niceload killed after 1 sec => The delay from niceload should be no more than 1 second + stdout /usr/bin/time -f %e perl -e 'for(1..100) { select(undef, undef, undef, 0.1); } print "done\n"' | int & niceload -vt 1 -s 10 -p $! & export A=$!; sleep 2; kill -s TERM $A; wait; echo Finished done 10 Finished Sleeping 1s Running 10s +echo 'bug #38908: niceload: Ctrl-C should resume jobs if using -p' bug #38908: niceload: Ctrl-C should resume jobs if using -p + # This should take 10 seconds to run + delay from niceload + # niceload killed after 1 sec => The delay from niceload should be no more than 1 second + stdout /usr/bin/time -f %e perl -e 'for(1..100) { select(undef, undef, undef, 0.1); } print "done\n"' | int & niceload -vt 1 -s 10 -p $! & export A=$!; sleep 2; kill -s INT $A; wait done 10 Sleeping 1s diff --git a/testsuite/wanted-results/parallel-freebsd b/testsuite/wanted-results/parallel-freebsd index 8667def8..0ec998a5 100644 --- a/testsuite/wanted-results/parallel-freebsd +++ b/testsuite/wanted-results/parallel-freebsd @@ -1,6 +1,9 @@ ### These tests requires VirtualBox running with the following images tange@freebsd7 +echo 'bug #40136: FreeBSD: No more processes' bug #40136: FreeBSD: No more processes +# Long line due to FreeBSD's /bin/sh stupidity + sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' && sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' && sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' && sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' && sem --wait --id my_id First started Second started Third started @@ -9,27 +12,45 @@ Fourth started The second finished The third finished The fourth finished +echo 'Test --compress --pipe' Test --compress --pipe + jot 1000 | parallel --compress --pipe cat | wc 1000 1000 3893 +echo 'bug #41613: --compress --line-buffer no newline'; bug #41613: --compress --line-buffer no newline + perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo It worked +echo 'bug #40135: FreeBSD: sem --fg does not finish under /bin/sh' bug #40135: FreeBSD: sem --fg does not finish under /bin/sh + sem --fg 'sleep 1; echo The job finished' The job finished +echo 'bug #40133: FreeBSD: --round-robin gives no output' bug #40133: FreeBSD: --round-robin gives no output + jot 1000000 | parallel --round-robin --pipe -kj3 wc | sort 299592 299592 2097144 299594 299594 2097158 400814 400814 2694594 + jot 1000000 | parallel --round-robin --pipe -kj4 wc | sort 149797 149797 1048579 235145 235145 1646016 299593 299593 2097151 315465 315465 2097150 +echo 'bug #40134: FreeBSD: --shebang not working' bug #40134: FreeBSD: --shebang not working + (echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang; + chmod 755 ./shebang; ./shebang It worked +echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' bug #40134: FreeBSD: --shebang(-wrap) not working + (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap; + chmod 755 ./shebang-wrap; ./shebang-wrap wrap works wrap works +echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' bug #40134: FreeBSD: --shebang(-wrap) with options not working + (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/bin/perl -w :::'; echo 'print @ARGV,"\n";') > shebang-wrap; + chmod 755 ./shebang-wrap; ./shebang-wrap wrap works with options /usr/bin/perl -w ./shebang-wrap wrap wrap /usr/bin/perl -w ./shebang-wrap works diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 52442028..f4f63571 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1,6 +1,10 @@ +echo '### Test exit val - true'; echo true | parallel; echo $? ### Test exit val - true 0 +echo '**' ** +echo '### Test exit val - false'; echo false | parallel; echo $? ### Test exit val - false 1 +echo '**' ** diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 64d3d3a5..863ad290 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -1,17 +1,27 @@ +echo '### Test if we can deal with output > 4 GB' ### Test if we can deal with output > 4 GB +## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum + echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum 46a318993dfc8e2afd71ff2bc6f605f1 - +echo '**' ** +echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" ### Test Force outside the file handle limit, 2009-02-17 Gave fork error + (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' parallel: Warning: Only enough file handles to run 252 jobs in parallel. Raising ulimit -n or /etc/security/limits.conf may help. Start end +echo '**' ** +echo '### Test of --retries on unreachable host' ### Test of --retries on unreachable host + seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo ssh: connect to host 4.3.2.1 port 22: Connection timed out parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. echo 1 1 echo 2 2 +echo '**' ** diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index 602ff44c..2d1c59ab 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -1,3 +1,4 @@ +echo '### Test race condition on 8 CPU (my laptop)'; seq 1 5000000 > /tmp/parallel_test; seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" ### Test race condition on 8 CPU (my laptop) 1 2 @@ -9,8 +10,12 @@ 8 9 10 +echo '**' ** +echo "### Test --tmpdir running full. bug #40733 was caused by this" ### Test --tmpdir running full. bug #40733 was caused by this + stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy parallel: Error: Output is incomplete. Cannot append to buffer file in $TMPDIR. Is the disk full? parallel: Error: Change $TMPDIR with --tmpdir or use --compress. +echo '**' ** diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index ff941fdf..515199c4 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -1,12 +1,19 @@ +echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)' ### bug #42089: --results with arg > 256 chars (should be 1 char shorter) + parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456; ls parallel_test_dir/1/ 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 + rm -rf parallel_test_dir +echo '**' ** +echo '### Test --halt-on-error 0'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 0; echo $? ### Test --halt-on-error 0 1 2 /bin/bash: non_exist: command not found +echo '**' ** +echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1; echo $? ### Test --halt-on-error 1 1 127 @@ -17,7 +24,9 @@ parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: sleep 2;false parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: sleep 4; non_exist +echo '**' ** +echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? ### Test --halt-on-error 2 1 1 @@ -25,7 +34,9 @@ parallel: This job failed: sleep 2;false parallel: This job failed: sleep 2;false +echo '**' ** +echo '### Test last dying print --halt-on-error 1'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? ### Test last dying print --halt-on-error 1 exit code 9 0 @@ -57,13 +68,16 @@ parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 +echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? ### Test last dying print --halt-on-error 2 exit code 1 0 1 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 +echo '**' ** +echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'; seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo ### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834 1 2 diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 3e01a52c..8284c456 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -1,26 +1,44 @@ +echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' ### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host + (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & wait With --controlmaster - finish first No --controlmaster - finish last +echo '### Stop if all hosts are filtered and there are no hosts left to run on' ### Stop if all hosts are filtered and there are no hosts left to run on + stdout parallel --filter-hosts -S no-such.host echo ::: 1 parallel: Warning: Removed no-such.host parallel: Error: There are no hosts left to run on. +echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' ### bug #41805: Idea: propagate --env for parallel --number-of-cores + echo '** test_zsh' ** test_zsh + FOO=test_zsh parallel --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin + echo '** test_zsh_filter' ** test_zsh_filter + FOO=test_zsh_filter parallel --filter-hosts --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh_filter PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin + echo '** test_csh' ** test_csh + FOO=test_csh parallel --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin + echo '** test_csh_filter' ** test_csh_filter + FOO=test_csh_filter parallel --filter-hosts --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh_filter PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin + echo '** bug #41805 done' ** bug #41805 done +echo '### Deal with long command lines on remote servers' ### Deal with long command lines on remote servers + perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc 4 10 100010 +echo '### Test bug #34241: --pipe should not spawn unneeded processes' ### Test bug #34241: --pipe should not spawn unneeded processes + seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end 1 2 Block_end @@ -28,45 +46,98 @@ Block_end 4 5 Block_end +echo '### --env _' ### --env _ + fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test OK FUBAR test + fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test DEBEMAIL: Undefined variable. +echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' ### --env _ with explicit mentioning of normally ignored var $DEBEMAIL + fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test OK FUBAR ole@tange.dk test + fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test OK FUBAR ole@tange.dk test +echo 'bug #40137: SHELL not bash: Warning when exporting funcs' bug #40137: SHELL not bash: Warning when exporting funcs + . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning no_warning + . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning warning parallel: Warning: Shell functions may not be supported in /bin/sh +echo '### bug #40002: --files and --nonall seem not to work together:' ### bug #40002: --files and --nonall seem not to work together: + parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l 1 +echo '### bug #40001: --joblog and --nonall seem not to work together:' ### bug #40001: --joblog and --nonall seem not to work together: + parallel --joblog - --nonall -S lo,localhost true | wc -l 3 +echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME' ### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME + cd && parallel --workdir . -S lo pwd ::: "" /home/tange +echo '### test filename :' ### test filename : + echo content-of-: > :; echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; cat :.:; rm : :.: remote-:.: content-of-: +echo '### Test --wd ... --cleanup which should remove the filled tmp dir' ### Test --wd ... --cleanup which should remove the filled tmp dir + find ~/.parallel/tmp |grep uNiQuE_sTrInG.6 | parallel rm; stdout parallel -j9 -k --retries 3 --wd ... --cleanup -S lo -v echo ">"{}.6 ::: uNiQuE_sTrInG; find ~/.parallel/tmp |grep uNiQuE_sTrInG.6 echo >uNiQuE_sTrInG.6 +echo '### Test --wd --' ### Test --wd -- + stdout parallel --wd -- -S lo echo OK ">"{}.7 ::: uNiQuE_sTrInG; cat ~/--/uNiQuE_sTrInG.7; stdout rm ~/--/uNiQuE_sTrInG.7 OK +echo '### Test --wd " "' ### Test --wd " " + stdout parallel --wd " " -S lo echo OK ">"{}.8 ::: uNiQuE_sTrInG; cat ~/" "/uNiQuE_sTrInG.8; stdout rm ~/" "/uNiQuE_sTrInG.8 OK +echo "### Test --wd \"'\"" ### Test --wd "'" + stdout parallel --wd "'" -S lo echo OK ">"{}.9 ::: uNiQuE_sTrInG; cat ~/"'"/uNiQuE_sTrInG.9; stdout rm ~/"'"/uNiQuE_sTrInG.9 OK +echo '### Test --trc --/--foo1' ### Test --trc --/--foo1 + mkdir -p ./--; echo 'Content --/--foo1' > ./--/--foo1; stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./--/--foo1; cat ./--/--foo1.1; stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: --/--foo1; cat ./--/--foo1.2 Content --/--foo1 remote1 Content --/--foo1 remote2 +echo '### Test --trc :dir/:foo2' ### Test --trc :dir/:foo2 + mkdir -p ./:dir; echo 'Content :dir/:foo2' > ./:dir/:foo2; stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./:dir/:foo2; cat ./:dir/:foo2.1; stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: :dir/:foo2; cat ./:dir/:foo2.2 Content :dir/:foo2 remote1 Content :dir/:foo2 remote2 +echo '### Test --trc " "/" "foo3' ### Test --trc " "/" "foo3 + mkdir -p ./" "; echo 'Content _/_foo3' > ./" "/" "foo3; stdout parallel --trc {}.1 -S lo '(cat {}; echo remote1) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1; stdout parallel --trc {}.2 -S lo '(cat ./{}; echo remote2) > {}.2' ::: " "/" "foo3; cat ./" "/" "foo3.2 Content _/_foo3 remote1 Content _/_foo3 remote2 +#echo '### Test --trc --/./--foo4' +# mkdir -p ./--; echo 'Content --/./--foo4' > ./--/./--foo4; # stdout parallel --trc {}.1 -S lo '(cat ./--foo4; echo remote{}) > --foo4.1' ::: --/./--foo4; cat ./--/./--foo4.1 +# +#echo '### Test --trc :/:foo2' +# mkdir -p ./:; echo 'Content :/:foo2' > ./:/:foo2; # stdout parallel --trc {}.1 -S lo '(cat {}; echo remote) > {}.1' ::: ./:/:foo2; cat ./:/:foo2.1 +# +#echo '### Test --trc " "/" "foo3' +# mkdir -p ./" "; echo 'Content _/_foo2' > ./" "/" "foo3; # stdout parallel --trc {}.1 -S lo '(cat {}; echo remote) > {}.1' ::: ./" "/" "foo3; cat ./" "/" "foo3.1 +# +# +# +# +# +# +#echo '### Test --trc --/./--foo4 :/./:foo5 " "/./" "foo6 ./foo10/./foo10' +# mkdir ./--; echo 'Content --/--foo4' > ./--/--foo4; # mkdir ./:; echo 'Content :/:foo5' > ./:/:foo5; # mkdir ./" "; echo 'Content _/_foo2' > ./" "/" "foo3; ### echo '### Test --trc --basefile --/./--foo7 :/./:foo8 " "/./" "foo9 ./foo11/./foo11' +### echo missing +### echo '### Test --trc "-- "' +### echo missing +### echo '### Test --trc " --"' +### echo missing +### diff --git a/testsuite/wanted-results/parallel-local-ssh2 b/testsuite/wanted-results/parallel-local-ssh2 index 6f8945fc..a1abfc9a 100644 --- a/testsuite/wanted-results/parallel-local-ssh2 +++ b/testsuite/wanted-results/parallel-local-ssh2 @@ -1,14 +1,22 @@ +echo '### --filter-hosts --slf <()' ### --filter-hosts --slf <() + parallel --nonall --filter-hosts --slf <(echo localhost) echo OK OK +echo '### --wd no-such-dir - csh' ### --wd no-such-dir - csh + stdout parallel --wd /no-such-dir -S csh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? mkdir: cannot create directory ‘/no-such-dir’: Permission denied /no-such-dir: No such file or directory. Exit code 1 +echo '### --wd no-such-dir - tcsh' ### --wd no-such-dir - tcsh + stdout parallel --wd /no-such-dir -S tcsh@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? mkdir: cannot create directory ‘/no-such-dir’: Permission denied /no-such-dir: No such file or directory. Exit code 1 +echo '### --wd no-such-dir - bash' ### --wd no-such-dir - bash + stdout parallel --wd /no-such-dir -S parallel@localhost echo ::: "ERROR IF PRINTED"; echo Exit code $? mkdir: cannot create directory ‘/no-such-dir’: Permission denied bash: line 0: cd: /no-such-dir: No such file or directory Exit code 1 diff --git a/testsuite/wanted-results/parallel-local1 b/testsuite/wanted-results/parallel-local1 index 84bfc5fa..881f6fda 100644 --- a/testsuite/wanted-results/parallel-local1 +++ b/testsuite/wanted-results/parallel-local1 @@ -1,141 +1,222 @@ +echo "### Test --basenamereplace" ### Test --basenamereplace + parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b.c b.c b.c b b b + parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b.c b.c b.c b b b +echo "### Test --bnr" ### Test --bnr + parallel -j1 -k -X --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b.c b.c b.c b b b + parallel -k --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b.c b.c b.c b b b +echo "### Test --extensionreplace" ### Test --extensionreplace + parallel -j1 -k -X --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b /a/b a/b b /a/b a/b b + parallel -k --extensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b /a/b a/b b /a/b a/b b +echo "### Test --er" ### Test --er + parallel -j1 -k -X --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b /a/b a/b b /a/b a/b b + parallel -k --er FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b /a/b a/b b /a/b a/b b +echo "### Test --basenameextensionreplace" ### Test --basenameextensionreplace + parallel -j1 -k -X --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b b b b b b + parallel -k --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b b b b b b +echo "### Test --bner" ### Test --bner + parallel -j1 -k -X --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b b b b b b + parallel -k --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b b b b b b b +echo "### Test {/}" ### Test {/} + parallel -j1 -k -X echo {/} ::: /a/b.c a/b.c b.c /a/b a/b b b.c b.c b.c b b b +echo "### Test {/.}" ### Test {/.} + parallel -j1 -k -X echo {/.} ::: /a/b.c a/b.c b.c /a/b a/b b b b b b b b +echo "### Test {#/.}" ### Test {#/.} + parallel -j1 -k -X echo {2/.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 number2 +echo "### Test {#/}" ### Test {#/} + parallel -j1 -k -X echo {2/} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 number2.c +echo "### Test {#.}" ### Test {#.} + parallel -j1 -k -X echo {2.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 a/number2 +echo "### bug #34241: --pipe should not spawn unneeded processes" ### bug #34241: --pipe should not spawn unneeded processes + echo | parallel -r -j2 -N1 --pipe md5sum -c && echo OK OK +echo '### Test of quoting of > bug' ### Test of quoting of > bug + echo '>/dev/null' | parallel echo >/dev/null +echo '### Test of quoting of > bug if line continuation' ### Test of quoting of > bug if line continuation + (echo '> '; echo '> '; echo '>') | parallel --max-lines 3 echo > > > +echo '### Test of --trim illegal' ### Test of --trim illegal + stdout parallel --trim fj ::: echo parallel: Error: --trim must be one of: r l rl lr. +echo '### Test of eof string on :::' ### Test of eof string on ::: + parallel -k -E ole echo ::: foo ole bar foo +echo '### Test of ignore-empty string on :::' ### Test of ignore-empty string on ::: + parallel -k -r echo ::: foo '' ole bar foo ole bar +echo '### Test of trailing space continuation' ### Test of trailing space continuation + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo foo ole bar quux + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 echo foo ole bar quux + parallel -kr -L2 echo ::: foo '' 'ole ' bar quux foo ole bar quux +echo '### Test of trailing space continuation with -E eof' ### Test of trailing space continuation with -E eof + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 -E bar echo foo ole + (echo foo; echo '';echo 'ole ';echo bar;echo quux) | parallel -kr -L2 -E bar echo foo ole + parallel -kr -L2 -E bar echo ::: foo '' 'ole ' bar quux foo ole +echo '### Test of --colsep' ### Test of --colsep + echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} a b c + (echo 'a%c%b'; echo a%c%b%d) | parallel -k --colsep % echo {1} {3} {2} {4} a b c a b c d + (echo a%c%b; echo d%f%e) | parallel -k --colsep % echo {1} {3} {2} a b c d e f + parallel -k --colsep % echo {1} {3} {2} ::: a%c%b d%f%e a b c d e f + parallel -k --colsep % echo {1} {3} {2} ::: a%c%b a b c + parallel -k --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d a b c a b c d +echo '### Test of tab as colsep' ### Test of tab as colsep + printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1} abc def ghi jkl + parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1} abc def ghi jkl +echo '### Test of multiple -a plus colsep' ### Test of multiple -a plus colsep + parallel --xapply -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1} abc def mno jkl +echo '### Test of multiple -a no colsep' ### Test of multiple -a no colsep + parallel --xapply -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1} abc def ghi jkl mno pqr +echo '### Test of quoting after colsplit' ### Test of quoting after colsplit + parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null' >/tmp/null >/dev/null +echo '### Test of --colsep as regexp' ### Test of --colsep as regexp + (echo 'a%c%%b'; echo a%c%b%d) | parallel -k --colsep %+ echo {1} {3} {2} {4} a b c a b c d + parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d a b c a b c d + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} a b c a b c d + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' a_b_c_ a_b_c_d +echo '### Test of -C' ### Test of -C + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' a_b_c_ a_b_c_d +echo '### Test of --trim n' ### Test of --trim n + (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' a_b_\ c\ _ a_\ b\ _c_d + parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d +echo '### Test of bug: If input is empty string' ### Test of bug: If input is empty string + (echo ; echo abcbdbebf;echo abc) | parallel -k --colsep b -v echo {1}{2} echo echo ac ac echo ac ac +echo '### Test bug #34241: --pipe should not spawn unneeded processes' ### Test bug #34241: --pipe should not spawn unneeded processes + seq 3 | parallel -j30 --pipe --block-size 10 cat\;echo o 1 2 3 o +echo '### Test :::: mixed with :::' ### Test :::: mixed with ::: +echo '### Test :::: < ::: :::' ### Test :::: < ::: ::: + parallel -k echo {1} {2} {3} :::: <(seq 6 7) ::: 4 5 ::: 1 2 3 6 4 1 6 4 2 6 4 3 @@ -148,7 +229,9 @@ o 7 5 1 7 5 2 7 5 3 +echo '### Test :::: < < :::: <' ### Test :::: < < :::: < + parallel -k echo {1} {2} {3} :::: <(seq 6 7) <(seq 4 5) :::: <(seq 1 3) 6 4 1 6 4 2 6 4 3 @@ -161,7 +244,9 @@ o 7 5 1 7 5 2 7 5 3 +echo '### Test -a :::: < :::: <' ### Test -a :::: < :::: < + parallel -k -a <(seq 6 7) echo {1} {2} {3} :::: <(seq 4 5) :::: <(seq 1 3) 6 4 1 6 4 2 6 4 3 @@ -174,7 +259,9 @@ o 7 5 1 7 5 2 7 5 3 +echo '### Test -a -a :::' ### Test -a -a ::: + parallel -k -a <(seq 6 7) -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 6 4 1 6 4 2 6 4 3 @@ -187,7 +274,9 @@ o 7 5 1 7 5 2 7 5 3 +echo '### Test -a - -a :::' ### Test -a - -a ::: + seq 6 7 | parallel -k -a - -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3 6 4 1 6 4 2 6 4 3 @@ -200,7 +289,9 @@ o 7 5 1 7 5 2 7 5 3 +echo '### Test :::: < - :::' ### Test :::: < - ::: + seq 4 5 | parallel -k echo {1} {2} {3} :::: <(seq 6 7) - ::: 1 2 3 6 4 1 6 4 2 6 4 3 @@ -213,7 +304,9 @@ o 7 5 1 7 5 2 7 5 3 +echo '### Test -E' ### Test -E + seq 1 100 | parallel -k -E 5 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) 1 2 3 1 2 4 1 3 3 @@ -238,14 +331,22 @@ o 4 3 4 4 4 3 4 4 4 +echo '### Test -E one empty' ### Test -E one empty + seq 1 100 | parallel -k -E 3 echo :::: - ::: 2 3 4 5 6 7 8 9 10 :::: <(seq 3 11) 1 2 2 2 +echo '### Test -E 2 empty' ### Test -E 2 empty + seq 1 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) 1 2 +echo '### Test -E all empty' ### Test -E all empty + seq 3 100 | parallel -k -E 3 echo :::: - ::: 3 4 5 6 7 8 9 10 :::: <(seq 3 11) +echo '### Test {#}' ### Test {#} + seq 1 10 | parallel -k echo {#} 1 2 3 @@ -256,17 +357,28 @@ o 8 9 10 +echo '### Test --seqreplace and line too long' ### Test --seqreplace and line too long + seq 1 1000 | stdout parallel -j1 -s 210 -k --seqreplace I echo IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII \|wc | uniq -c 9 1 1 101 90 1 1 201 1 parallel: Error: Command line too long (309 >= 210) at number 0: 100... +echo '### bug #37042: -J foo is taken from the whole command line - not just the part before the command' ### bug #37042: -J foo is taken from the whole command line - not just the part before the command + echo '--tagstring foo' > ~/.parallel/bug_37042_profile; + parallel -J bug_37042_profile echo ::: tag_with_foo; foo tag_with_foo + parallel --tagstring a -J bug_37042_profile echo ::: tag_with_a; a tag_with_a + parallel --tagstring a echo -J bug_37042_profile ::: print_-J_bug_37042_profile; a -J bug_37042_profile print_-J_bug_37042_profile +echo '### Bug introduce by fixing bug #37042' ### Bug introduce by fixing bug #37042 + parallel --xapply -a <(printf 'abc') --colsep '\t' echo {1} abc +echo "### Test --header with -N" ### Test --header with -N + (echo h1; echo h2; echo 1a;echo 1b; echo 2a;echo 2b; echo 3a)| parallel -j1 --pipe -N2 -k --header '.*\n.*\n' echo Start\;cat \; echo Stop Start h1 h2 @@ -284,7 +396,9 @@ h1 h2 3a Stop +echo "### Test --header with --block 1k" ### Test --header with --block 1k + (echo h1; echo h2; perl -e '$a="x"x110;for(1..22){print $_,$a,"\n"'})| parallel -j1 --pipe -k --block 1k --header '.*\n.*\n' echo Start\;cat \; echo Stop Start h1 h2 @@ -315,5 +429,7 @@ h2 21xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 22xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Stop +echo "### Test --header with multiple :::" ### Test --header with multiple ::: + parallel --header : echo {a} {b} {1} {2} ::: b b1 ::: a a2 a2 b1 b1 a2 diff --git a/testsuite/wanted-results/parallel-local114 b/testsuite/wanted-results/parallel-local114 index 7ac7987f..4dab2cdb 100644 --- a/testsuite/wanted-results/parallel-local114 +++ b/testsuite/wanted-results/parallel-local114 @@ -1,14 +1,23 @@ +echo "### --line-buffer" ### --line-buffer + seq 10 | parallel -j20 --line-buffer 'seq {} 10 | pv -qL 10' > /tmp/parallel_l$$; seq 10 | parallel -j20 'seq {} 10 | pv -qL 10' > /tmp/parallel_$$; cat /tmp/parallel_l$$ | wc; diff /tmp/parallel_$$ /tmp/parallel_l$$ >/dev/null ; echo These must diff: $? 55 55 120 These must diff: 1 +echo "### --pipe --line-buffer" ### --pipe --line-buffer + seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_pl$$; seq 200| parallel -N10 -L1 --pipe -j20 --tagstring {#} pv -qL 10 > /tmp/parallel_p$$; cat /tmp/parallel_pl$$ | wc; diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $? 200 400 1202 These must diff: 1 +echo "### --pipe --line-buffer --compress" ### --pipe --line-buffer --compress + seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc 200 400 1202 +echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations" ### bug #41482: --pipe --compress blocks at different -j/seq combinations + seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat; echo echo 1-4 + 1-4 1 echo 1-4 + 1-4 + seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo; echo 4 times wc to stderr to stdout echo 1 1 echo 2 @@ -18,37 +27,52 @@ echo 3 echo 4 4 4 times wc to stderr to stdout + (seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null 1 1 2 1 1 2 1 1 2 1 1 2 + echo 1 2 3 4 1 2 3 4 + seq 4 | parallel -k -j3 --compress echo; echo 1 2 3 4 1 2 3 4 1 2 3 4 + seq 4 | parallel -k -j1 --compress echo; echo 1 2 1 2 3 4 1 2 + seq 2 | parallel -k -j1 --compress echo; echo 1 2 3 1 2 1 2 3 + seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat; echo "### bug #41609: --compress fails" 1 2 3 ### bug #41609: --compress fails + seq 12 | parallel --compress --compress-program bzip2 -k seq {} 1000000 | md5sum 24812dd0f24a26d08a780f988b9d5ad2 - + seq 12 | parallel --compress -k seq {} 1000000 | md5sum 24812dd0f24a26d08a780f988b9d5ad2 - +echo "### --compress race condition (use nice): Fewer than 400 would run" ### --compress race condition (use nice): Fewer than 400 would run + seq 400| nice parallel -j200 --compress echo | wc 400 400 1492 +echo "### -v --pipe: Dont spawn too many - 1 is enough" ### -v --pipe: Dont spawn too many - 1 is enough + seq 1 | parallel -j10 -v --pipe cat cat 1 +echo "### Test -N0 and --tagstring (fails)" ### Test -N0 and --tagstring (fails) + echo tagstring arg | parallel --tag -N0 echo foo foo +echo "### Test -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' ### Test -I 1 1 2 1 @@ -105,6 +129,7 @@ cat 10 8 10 9 10 10 +echo "### Test -X -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::' ### Test -X -I a1 b1 a2 b1 b2 @@ -116,6 +141,7 @@ a7 b1 b2 b3 b4 b5 b6 b7 a8 b1 b2 b3 b4 b5 b6 b7 b8 a9 b1 b2 b3 b4 b5 b6 b7 b8 b9 a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 +echo "### Test -m -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::' ### Test -m -I a1 b1 a2 b1 2 @@ -127,18 +153,23 @@ a7 b1 2 3 4 5 6 7 a8 b1 2 3 4 5 6 7 8 a9 b1 2 3 4 5 6 7 8 9 a10 b1 2 3 4 5 6 7 8 9 10 +echo "### Test max line length -m -I"; seq 1 60000 | parallel -I :: -m -j1 echo a::b::c | tee >(sort |md5sum) >/tmp/114-a$$; export CHAR=$(cat /tmp/114-a$$ | wc -c); export LINES=$(cat /tmp/114-a$$ | wc -l); echo "Chars per line ($CHAR/$LINES): "$(echo "$CHAR/$LINES" | bc); rm /tmp/114-a$$ ### Test max line length -m -I 31d9274be5fdc2de59487cb05ba57776 - Chars per line (697800/6): 116300 +echo "### Test max line length -X -I"; seq 1 60000 | parallel -I :: -X -j1 echo a::b::c | tee >(sort |md5sum) >/tmp/114-b$$; export CHAR=$(cat /tmp/114-b$$ | wc -c); export LINES=$(cat /tmp/114-b$$ | wc -l); echo "Chars per line ($CHAR/$LINES): "$(echo "$CHAR/$LINES" | bc); rm /tmp/114-b$$ ### Test max line length -X -I 22074f9acada52462defb18ba912d744 - Chars per line (817788/7): 116826 +echo "### bug #36659: --sshlogin strips leading slash from ssh command"; parallel --sshlogin '/usr/bin/ssh localhost' echo ::: OK ### bug #36659: --sshlogin strips leading slash from ssh command OK +echo "### bug #36660: --workdir mkdir does not use --sshlogin custom ssh"; rm -rf /tmp/foo36660; cd /tmp; echo OK > parallel_test.txt; ssh () { echo Failed; }; export -f ssh; parallel --workdir /tmp/foo36660/bar --transfer --sshlogin '/usr/bin/ssh localhost' cat ::: parallel_test.txt; echo "bug #36657: --load does not work with custom ssh"; cd /tmp; echo OK > parallel_test.txt; ssh () { echo Failed; }; export -f ssh; parallel --load=1000% -S "/usr/bin/ssh localhost" echo ::: OK ### bug #36660: --workdir mkdir does not use --sshlogin custom ssh OK bug #36657: --load does not work with custom ssh OK +echo "bug #34958: --pipe with record size measured in lines"; seq 10 | parallel -k --pipe -L 4 cat\;echo bug 34958-1 bug #34958: --pipe with record size measured in lines 1 2 @@ -152,6 +183,7 @@ bug 34958-1 9 10 bug 34958-1 +echo "bug #37325: Inefficiency of --pipe -L"; seq 2000 | parallel -k --pipe --block 1k -L 4 wc\;echo FOO | uniq bug #37325: Inefficiency of --pipe -L 276 276 996 FOO @@ -171,6 +203,7 @@ FOO FOO 180 180 900 FOO +echo "bug #34958: --pipe with record size measured in lines"; seq 10 | parallel -k --pipe -l 4 cat\;echo bug 34958-2 bug #34958: --pipe with record size measured in lines 1 2 @@ -184,6 +217,7 @@ bug 34958-2 9 10 bug 34958-2 +echo "### Test --results"; mkdir -p /tmp/parallel_results_test; parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII; ls /tmp/parallel_results_test/testA/*/*/*/*/*; rm -rf /tmp/parallel_results_test/testA* ### Test --results I III I IIII @@ -197,6 +231,7 @@ II IIII /tmp/parallel_results_test/testA/1/II/2/III/stdout /tmp/parallel_results_test/testA/1/II/2/IIII/stderr /tmp/parallel_results_test/testA/1/II/2/IIII/stdout +echo "### Test --res"; mkdir -p /tmp/parallel_results_test; parallel -k --res /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII; ls /tmp/parallel_results_test/testD/*/*/*/*/*; rm -rf /tmp/parallel_results_test/testD* ### Test --res I III I IIII @@ -210,6 +245,7 @@ II IIII /tmp/parallel_results_test/testD/1/II/2/III/stdout /tmp/parallel_results_test/testD/1/II/2/IIII/stderr /tmp/parallel_results_test/testD/1/II/2/IIII/stdout +echo "### Test --result"; mkdir -p /tmp/parallel_results_test; parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII; ls /tmp/parallel_results_test/testE/*/*/*/*/*; rm -rf /tmp/parallel_results_test/testE* ### Test --result I III I IIII @@ -223,6 +259,7 @@ II IIII /tmp/parallel_results_test/testE/1/II/2/III/stdout /tmp/parallel_results_test/testE/1/II/2/IIII/stderr /tmp/parallel_results_test/testE/1/II/2/IIII/stdout +echo "### Test --results --header :"; mkdir -p /tmp/parallel_results_test; parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII; ls /tmp/parallel_results_test/testB/*/*/*/*/*; rm -rf /tmp/parallel_results_test/testB* ### Test --results --header : I III I IIII @@ -236,6 +273,7 @@ II IIII /tmp/parallel_results_test/testB/a/II/b/III/stdout /tmp/parallel_results_test/testB/a/II/b/IIII/stderr /tmp/parallel_results_test/testB/a/II/b/IIII/stdout +echo "### Test --results --header : named - a/b swapped"; mkdir -p /tmp/parallel_results_test; parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: b III IIII ::: a I II; ls /tmp/parallel_results_test/testC/*/*/*/*/*; rm -rf /tmp/parallel_results_test/testC* ### Test --results --header : named - a/b swapped I III II III @@ -249,11 +287,13 @@ II IIII /tmp/parallel_results_test/testC/a/II/b/III/stdout /tmp/parallel_results_test/testC/a/II/b/IIII/stderr /tmp/parallel_results_test/testC/a/II/b/IIII/stdout +echo "### Test --results --header : piped"; mkdir -p /tmp/parallel_results_test; (echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel --header : --result /tmp/parallel_results_test/testF true; find /tmp/parallel_results_test/testF/*/*/* | sort; rm -rf /tmp/parallel_results_test/testF* ### Test --results --header : piped /tmp/parallel_results_test/testF/Col/backslash\\tab/2 /tmp/parallel_results_test/testF/Col/backslash\\tab/2/slash\_null\0eof /tmp/parallel_results_test/testF/Col/backslash\\tab/2/slash\_null\0eof/stderr /tmp/parallel_results_test/testF/Col/backslash\\tab/2/slash\_null\0eof/stdout +echo "### Test --results --header : piped - non-existing column header"; mkdir -p /tmp/parallel_results_test; (printf "Col1\t\n"; printf "v1\tv2\tv3\n"; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel --header : --result /tmp/parallel_results_test/testG true; find /tmp/parallel_results_test/testG/ | sort; rm -rf /tmp/parallel_results_test/testG* ### Test --results --header : piped - non-existing column header /tmp/parallel_results_test/testG/ /tmp/parallel_results_test/testG/Col1 diff --git a/testsuite/wanted-results/parallel-local13 b/testsuite/wanted-results/parallel-local13 index 3d882f34..191c60d5 100644 --- a/testsuite/wanted-results/parallel-local13 +++ b/testsuite/wanted-results/parallel-local13 @@ -1,5 +1,7 @@ ### Tests from xargs +echo '### -0 -n3 echo < files0.xi' ### -0 -n3 echo < files0.xi +stdout xargs -0 -n3 echo < files0.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h @@ -8,6 +10,7 @@ /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -0 -n3 echo < files0.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h @@ -16,18 +19,26 @@ /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -d o -n1 echo < helloworld.xi' ### -d o -n1 echo < helloworld.xi +stdout xargs -d o -n1 echo < helloworld.xi hell w rld +stdout parallel -k -d o -n1 echo < helloworld.xi hell w rld +echo '### -E_ -0 echo < eof_-0.xi' ### -E_ -0 echo < eof_-0.xi +stdout xargs -E_ -0 echo < eof_-0.xi one two _ three four +stdout parallel -X -k -E_ -0 echo < eof_-0.xi one two +echo '### -i -0 echo from \{\} to x{}y < items-0.xi' ### -i -0 echo from \{\} to x{}y < items-0.xi +stdout xargs -i -0 echo from \{\} to x{}y < items-0.xi from one to xoney from to x @@ -48,6 +59,7 @@ n n y +stdout parallel -k -i -0 echo from \{\} to x{}y < items-0.xi from one to xoney from to x @@ -68,12 +80,17 @@ n n y +echo '### -i -s26 -0 echo from \{\} to x{}y < items-0.xi' ### -i -s26 -0 echo from \{\} to x{}y < items-0.xi +stdout xargs -i -s26 -0 echo from \{\} to x{}y < items-0.xi xargs: argument list too long from one to xoney +stdout parallel -k -i -s26 -0 echo from \{\} to x{}y < items-0.xi parallel: Error: Command line too long (42 >= 26) at number 0: ... +echo '### -l -0 echo < ldata-0.xi' ### -l -0 echo < ldata-0.xi +stdout xargs -l -0 echo < ldata-0.xi 1 22 333 4444 55555 666666 7777777 @@ -83,6 +100,7 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 666666 7777777 88888888 999999999 +stdout parallel -l -k -0 echo < ldata-0.xi 1 22 333 4444 55555 666666 7777777 @@ -92,7 +110,9 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 666666 7777777 88888888 999999999 +echo '### -l -0 echo < ldatab-0.xi' ### -l -0 echo < ldatab-0.xi +stdout xargs -l -0 echo < ldatab-0.xi 1 22 333 4444 55555 666666 @@ -103,6 +123,7 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 666666 7777777 88888888 999999999 +stdout parallel -l -k -0 echo < ldatab-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 @@ -110,35 +131,46 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 666666 7777777 88888888 999999999 +echo '### -L2 -0 echo < ldata-0.xi' ### -L2 -0 echo < ldata-0.xi +stdout xargs -L2 -0 echo < ldata-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L2 -0 echo < ldata-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -L2 -0 echo < ldatab-0.xi' ### -L2 -0 echo < ldatab-0.xi +stdout xargs -L2 -0 echo < ldatab-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L2 -0 echo < ldatab-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -L3 -0 echo < ldata-0.xi' ### -L3 -0 echo < ldata-0.xi +stdout xargs -L3 -0 echo < ldata-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L3 -0 echo < ldata-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -n1 -0 echo < stairs-0.xi' ### -n1 -0 echo < stairs-0.xi +stdout xargs -n1 -0 echo < stairs-0.xi 1 22 333 @@ -157,6 +189,7 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 7777777 88888888 999999999 +stdout parallel -k -n1 -0 echo < stairs-0.xi 1 22 333 @@ -175,7 +208,9 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 7777777 88888888 999999999 +echo '### -n2 -0 echo < stairs-0.xi' ### -n2 -0 echo < stairs-0.xi +stdout xargs -n2 -0 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -185,6 +220,7 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -n2 -0 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -194,7 +230,9 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 4444 55555 666666 7777777 88888888 999999999 +echo '### -n2 -s26 -0 echo < stairs-0.xi' ### -n2 -s26 -0 echo < stairs-0.xi +stdout xargs -n2 -s26 -0 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -204,6 +242,7 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -n2 -s26 -0 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -213,7 +252,9 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 4444 55555 666666 7777777 88888888 999999999 +echo '### -n2 -s26 -x -0 echo < stairs-0.xi' ### -n2 -s26 -x -0 echo < stairs-0.xi +stdout xargs -n2 -s26 -x -0 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -223,6 +264,7 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -n2 -s26 -x -0 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -232,52 +274,71 @@ parallel: Error: Command line too long (42 >= 26) at number 0: 4444 55555 666666 7777777 88888888 999999999 +echo '### -n3 -0 echo < stairs2-0.xi' ### -n3 -0 echo < stairs2-0.xi +stdout xargs -n3 -0 echo < stairs2-0.xi 999999999 88888888 7777777 666666 55555 4444 333 22 1 999999999 88888888 7777777 666666 55555 4444 333 22 1 +stdout parallel -k -n3 -0 echo < stairs2-0.xi 999999999 88888888 7777777 666666 55555 4444 333 22 1 999999999 88888888 7777777 666666 55555 4444 333 22 1 +stdout xargs -n3 -s36 -0 echo < stairs2-0.xi 999999999 88888888 7777777 666666 55555 4444 333 22 1 999999999 88888888 7777777 666666 55555 4444 333 22 1 +stdout parallel -k -n3 -s36 -0 echo < stairs2-0.xi 999999999 88888888 7777777 666666 55555 4444 333 22 1 999999999 88888888 7777777 666666 55555 4444 333 22 1 +echo '### -0 echo < noeof-0.xi' ### -0 echo < noeof-0.xi +stdout xargs -0 echo < noeof-0.xi first second +stdout parallel -k -0 echo < noeof-0.xi first second +echo '### -P3 -n1 -IARG sh -c ARG < Pdata.xi' ### -P3 -n1 -IARG sh -c ARG < Pdata.xi +stdout xargs -P3 -n1 -IARG sh -c ARG < Pdata.xi three two one +stdout parallel -k -P3 -n1 -IARG sh -c ARG < Pdata.xi one two three +echo '### -r echo this plus that < blank.xi' ### -r echo this plus that < blank.xi +stdout xargs -r echo this plus that < blank.xi +stdout parallel -k -r echo this plus that < blank.xi +echo '### -0 -s118 echo < stairs-0.xi' ### -0 -s118 echo < stairs-0.xi +stdout xargs -0 -s118 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -X -0 -s118 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -0 -s19 echo < stairs-0.xi' ### -0 -s19 echo < stairs-0.xi +stdout xargs -0 -s19 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 @@ -288,6 +349,7 @@ three 7777777 88888888 999999999 +stdout parallel -k -X -0 -s19 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 @@ -298,7 +360,9 @@ three 7777777 88888888 999999999 +echo '### -0 -s19 echo < stairs2-0.xi' ### -0 -s19 echo < stairs2-0.xi +stdout xargs -0 -s19 echo < stairs2-0.xi 999999999 88888888 7777777 @@ -309,6 +373,7 @@ three 7777777 666666 55555 4444 333 22 1 +stdout parallel -k -X -0 -s19 echo < stairs2-0.xi 999999999 88888888 7777777 @@ -322,7 +387,9 @@ three 333 22 1 +echo '### -0 -s20 echo < stairs-0.xi' ### -0 -s20 echo < stairs-0.xi +stdout xargs -0 -s20 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 @@ -332,6 +399,7 @@ three 666666 7777777 88888888 999999999 +stdout parallel -k -X -0 -s20 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 @@ -341,99 +409,141 @@ three 666666 7777777 88888888 999999999 +echo '### -0 -s30 echo < stairs-0.xi' ### -0 -s30 echo < stairs-0.xi +stdout xargs -0 -s30 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -X -0 -s30 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -0 echo this plus that < space.xi' ### -0 echo this plus that < space.xi +stdout xargs -0 echo this plus that < space.xi this plus that +stdout parallel -k -0 echo this plus that < space.xi this plus that +echo '### -r echo this plus that < space.xi' ### -r echo this plus that < space.xi +stdout xargs -r echo this plus that < space.xi +stdout parallel -k -r echo this plus that < space.xi this plus that +echo '### -t -0 echo this plus that < space.xi' ### -t -0 echo this plus that < space.xi +stdout xargs -t -0 echo this plus that < space.xi echo this plus that this plus that +stdout parallel -k -t -0 echo this plus that < space.xi echo this plus that ' '\ \ \ \ \ ' ' this plus that +echo '### true < 32767-ys.xi' ### true < 32767-ys.xi +stdout xargs true < 32767-ys.xi +stdout parallel -k true < 32767-ys.xi +echo '### true < 16383-ys.xi' ### true < 16383-ys.xi +stdout xargs true < 16383-ys.xi +stdout parallel -k true < 16383-ys.xi +echo '### -E EOF echo < EOFb.xi' ### -E EOF echo < EOFb.xi +stdout xargs -E EOF echo < EOFb.xi one two bEOF three four +stdout parallel -k -E EOF echo < EOFb.xi one two bEOF three four +echo '### -E EOF echo < EOFe.xi' ### -E EOF echo < EOFe.xi +stdout xargs -E EOF echo < EOFe.xi one two EOFe three four +stdout parallel -k -E EOF echo < EOFe.xi one two EOFe three four +echo '### -E EOF echo < EOF.xi' ### -E EOF echo < EOF.xi +stdout xargs -E EOF echo < EOF.xi one two +stdout parallel -k -E EOF echo < EOF.xi one two EOF three four +echo '### -E_ echo < eof_.xi' ### -E_ echo < eof_.xi +stdout xargs -E_ echo < eof_.xi one two +stdout parallel -k -E_ echo < eof_.xi one two +echo '### -E_ -IARG echo from ARG to xARGy < eof_.xi' ### -E_ -IARG echo from ARG to xARGy < eof_.xi +stdout xargs -E_ -IARG echo from ARG to xARGy < eof_.xi from one to xoney from two to xtwoy +stdout parallel -k -E_ -IARG echo from ARG to xARGy < eof_.xi from one to xoney from two to xtwoy +echo '### -s470 echo hi there < files.xi' ### -s470 echo hi there < files.xi +stdout xargs -s470 echo hi there < files.xi hi there /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure hi there /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -s470 -X echo hi there < files.xi hi there /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure hi there /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh hi there /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh hi there /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi hi there /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info hi there /src/gnu/autoconf-1.11/texinfo.tex +echo '### -IARG echo from ARG to xARGy -E_ < eof_.xi' ### -IARG echo from ARG to xARGy -E_ < eof_.xi +stdout xargs -IARG echo from ARG to xARGy -E_ < eof_.xi from one to xoney -E_ from two to xtwoy -E_ from _ to x_y -E_ from three to xthreey -E_ from four to xfoury -E_ +stdout parallel -k -IARG echo from ARG to xARGy -E_ < eof_.xi from one to xoney -E_ from two to xtwoy -E_ from _ to x_y -E_ from three to xthreey -E_ from four to xfoury -E_ +echo '### -IARG echo from ARG to xARGy < items.xi' ### -IARG echo from ARG to xARGy < items.xi +stdout xargs -IARG echo from ARG to xARGy < items.xi from dumb to xdumby from s s to xs s y from f f to xf f y from r r to xr r y from t t to xt t y from v v to xv v y +stdout parallel -k -IARG echo from ARG to xARGy < items.xi from to x y from dumb to xdumby from s s to x s s y @@ -442,7 +552,9 @@ from r r to x r r y from t t to x t t y from v v to x v v y from to x y +echo '### -IARG -s15 echo ARG < stairs.xi' ### -IARG -s15 echo ARG < stairs.xi +stdout xargs -IARG -s15 echo ARG < stairs.xi 1 22 333 @@ -461,6 +573,7 @@ from to x y 7777777 88888888 999999999 +stdout parallel -k -IARG -X -s15 echo ARG < stairs.xi 1 22 333 4444 55555 @@ -475,15 +588,20 @@ from to x y 7777777 88888888 999999999 +echo '### -L2 echo < ldatab.xi' ### -L2 echo < ldatab.xi +stdout xargs -L2 echo < ldatab.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L2 echo < ldatab.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -L2 -n2 echo < ldata.xi' ### -L2 -n2 echo < ldata.xi +stdout xargs -L2 -n2 echo < ldata.xi 1 22 333 4444 55555 666666 @@ -493,19 +611,25 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L2 -n2 echo < ldata.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -L3 echo < ldata.xi' ### -L3 echo < ldata.xi +stdout xargs -L3 echo < ldata.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L3 echo < ldata.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -n1 echo < stairs.xi' ### -n1 echo < stairs.xi +stdout xargs -n1 echo < stairs.xi 1 22 333 @@ -524,6 +648,7 @@ from to x y 7777777 88888888 999999999 +stdout parallel -k -n1 echo < stairs.xi 1 22 333 @@ -542,7 +667,9 @@ from to x y 7777777 88888888 999999999 +echo '### -n2 echo < stairs.xi' ### -n2 echo < stairs.xi +stdout xargs -n2 echo < stairs.xi 1 22 333 4444 55555 666666 @@ -552,6 +679,7 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -n2 echo < stairs.xi 1 22 333 4444 55555 666666 @@ -561,7 +689,9 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +echo '### -n2 -s26 echo < stairs.xi' ### -n2 -s26 echo < stairs.xi +stdout xargs -n2 -s26 echo < stairs.xi 1 22 333 4444 55555 666666 @@ -571,6 +701,7 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -n2 -s26 echo < stairs.xi 1 22 333 4444 55555 666666 @@ -580,7 +711,9 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +echo '### -n2 -s26 -x echo < stairs.xi' ### -n2 -s26 -x echo < stairs.xi +stdout xargs -n2 -s26 -x echo < stairs.xi 1 22 333 4444 55555 666666 @@ -590,6 +723,7 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -n2 -s26 -x echo < stairs.xi 1 22 333 4444 55555 666666 @@ -599,7 +733,9 @@ from to x y 4444 55555 666666 7777777 88888888 999999999 +echo '### -n3 echo < files.xi' ### -n3 echo < files.xi +stdout xargs -n3 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h @@ -608,6 +744,7 @@ from to x y /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -n3 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h @@ -616,40 +753,54 @@ from to x y /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -n3 -s36 echo < stairs2.xi' ### -n3 -s36 echo < stairs2.xi +stdout xargs -n3 -s36 echo < stairs2.xi 999999999 88888888 7777777 666666 55555 4444 333 22 1 999999999 88888888 7777777 666666 55555 4444 333 22 1 +stdout parallel -k -n3 -s36 echo < stairs2.xi 999999999 88888888 7777777 666666 55555 4444 333 22 1 999999999 88888888 7777777 666666 55555 4444 333 22 1 +echo '### echo < noeof.xi' ### echo < noeof.xi +stdout xargs echo < noeof.xi first second +stdout parallel -k echo < noeof.xi first second +echo '### echo < quotes.xi' ### echo < quotes.xi +stdout xargs echo < quotes.xi this is quoted stuff and an embedded newline with single quotes as well. +stdout parallel -k echo < quotes.xi this is "quoted stuff" and \ an embedded newline with 'single quotes' as well. +echo '### -s118 echo < stairs.xi' ### -s118 echo < stairs.xi +stdout xargs -s118 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -X -s118 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -s19 echo < stairs2.xi' ### -s19 echo < stairs2.xi +stdout xargs -s19 echo < stairs2.xi 999999999 88888888 7777777 @@ -660,6 +811,7 @@ with 'single quotes' as well. 7777777 666666 55555 4444 333 22 1 +stdout parallel -k -X -s19 echo < stairs2.xi 999999999 88888888 7777777 @@ -673,7 +825,9 @@ with 'single quotes' as well. 333 22 1 +echo '### -s19 echo < stairs.xi' ### -s19 echo < stairs.xi +stdout xargs -s19 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 @@ -684,6 +838,7 @@ with 'single quotes' as well. 7777777 88888888 999999999 +stdout parallel -k -X -s19 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 @@ -694,7 +849,9 @@ with 'single quotes' as well. 7777777 88888888 999999999 +echo '### -s20 echo < stairs.xi' ### -s20 echo < stairs.xi +stdout xargs -s20 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 @@ -704,6 +861,7 @@ with 'single quotes' as well. 666666 7777777 88888888 999999999 +stdout parallel -k -X -s20 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 @@ -713,21 +871,27 @@ with 'single quotes' as well. 666666 7777777 88888888 999999999 +echo '### -s30 echo < stairs.xi' ### -s30 echo < stairs.xi +stdout xargs -s30 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -X -s30 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### -s470 echo < files.xi' ### -s470 echo < files.xi +stdout xargs -s470 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -X -s470 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs @@ -737,7 +901,9 @@ with 'single quotes' as well. /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -s47 echo < files.xi' ### -s47 echo < files.xi +stdout xargs -s47 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in @@ -760,6 +926,7 @@ with 'single quotes' as well. /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -X -s47 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in @@ -782,7 +949,9 @@ with 'single quotes' as well. /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -s48 echo < files.xi' ### -s48 echo < files.xi +stdout xargs -s48 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in @@ -805,6 +974,7 @@ with 'single quotes' as well. /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -X -s48 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in @@ -827,10 +997,15 @@ with 'single quotes' as well. /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -s6 echo < files.xi' ### -s6 echo < files.xi +stdout xargs -s6 echo < files.xi xargs: argument line too long +stdout parallel -k -X -s6 echo < files.xi parallel: Error: Command line too long (27 >= 6) at number -7: /src/gnu/autoconf-1.11... +echo '### -iARG -s86 echo ARG is xARGx < files.xi' ### -iARG -s86 echo ARG is xARGx < files.xi +stdout xargs -iARG -s86 echo ARG is xARGx < files.xi /src/gnu/autoconf-1.11 is x/src/gnu/autoconf-1.11x /src/gnu/autoconf-1.11/README is x/src/gnu/autoconf-1.11/READMEx /src/gnu/autoconf-1.11/Makefile.in is x/src/gnu/autoconf-1.11/Makefile.inx @@ -853,6 +1028,7 @@ parallel: Error: Command line too long (27 >= 6) at number -7: /src/gnu/autoconf /src/gnu/autoconf-1.11/make-stds.texi is x/src/gnu/autoconf-1.11/make-stds.texix /src/gnu/autoconf-1.11/standards.info is x/src/gnu/autoconf-1.11/standards.infox /src/gnu/autoconf-1.11/texinfo.tex is x/src/gnu/autoconf-1.11/texinfo.texx +stdout parallel -k -iARG -s86 echo ARG is xARGx < files.xi /src/gnu/autoconf-1.11 is x/src/gnu/autoconf-1.11x /src/gnu/autoconf-1.11/README is x/src/gnu/autoconf-1.11/READMEx /src/gnu/autoconf-1.11/Makefile.in is x/src/gnu/autoconf-1.11/Makefile.inx @@ -875,64 +1051,99 @@ parallel: Error: Command line too long (27 >= 6) at number -7: /src/gnu/autoconf /src/gnu/autoconf-1.11/make-stds.texi is x/src/gnu/autoconf-1.11/make-stds.texix /src/gnu/autoconf-1.11/standards.info is x/src/gnu/autoconf-1.11/standards.infox /src/gnu/autoconf-1.11/texinfo.tex is x/src/gnu/autoconf-1.11/texinfo.texx +echo '### echo this plus that < space.xi' ### echo this plus that < space.xi +stdout xargs echo this plus that < space.xi this plus that +stdout parallel -k echo this plus that < space.xi this plus that this plus that +echo '### -IARG echo from ARG to xARGy < space.xi' ### -IARG echo from ARG to xARGy < space.xi +stdout xargs -IARG echo from ARG to xARGy < space.xi +stdout parallel -k -IARG echo from ARG to xARGy < space.xi from to xy from to x y +echo '### printf "\[%s\]\n" < verticaltabs.xi' ### printf "\[%s\]\n" < verticaltabs.xi +stdout xargs printf "\[%s\]\n" < verticaltabs.xi \[q w e r\] \[6 7 8 9\] -[q w e r 6 7 8 9]n### printf "\[%s\]\n" < formfeeds.xi +stdout parallel -k printf "\[%s\]\n" < verticaltabs.xi +[q w e r 6 7 8 9]necho '### printf "\[%s\]\n" < formfeeds.xi' +### printf "\[%s\]\n" < formfeeds.xi +stdout xargs printf "\[%s\]\n" < formfeeds.xi \[1 2 3 4\] \[a b c d\] -[1 2 3 4]n[a b c d]n### -L2 echo < ldata.xi +stdout parallel -k printf "\[%s\]\n" < formfeeds.xi +[1 2 3 4]n[a b c d]necho '### -L2 echo < ldata.xi' +### -L2 echo < ldata.xi +stdout xargs -L2 echo < ldata.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -k -L2 echo < ldata.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +echo '### echo < unmatched2.xi' ### echo < unmatched2.xi +stdout xargs echo < unmatched2.xi xargs: unmatched double quote; by default quotes are special to xargs unless you use the -0 option one +stdout parallel -k echo < unmatched2.xi one "two +echo '### echo < unmatched.xi' ### echo < unmatched.xi +stdout xargs echo < unmatched.xi xargs: unmatched double quote; by default quotes are special to xargs unless you use the -0 option one +stdout parallel -k echo < unmatched.xi one "two three +echo '### -n2 -x echo < unmatched.xi' ### -n2 -x echo < unmatched.xi +stdout xargs -n2 -x echo < unmatched.xi xargs: unmatched double quote; by default quotes are special to xargs unless you use the -0 option +stdout parallel -k -n2 -x echo < unmatched.xi one "two three +echo '### -eEOF echo < eofstr.xi' ### -eEOF echo < eofstr.xi +stdout xargs -eEOF echo < eofstr.xi firstline secondline +stdout parallel -k -eEOF echo < eofstr.xi firstline secondline +echo '### -e echo < eof_.xi' ### -e echo < eof_.xi +stdout xargs -e echo < eof_.xi one two _ three four +stdout parallel -e -k echo < eof_.xi one two _ three four +echo '### -E_ echo < eof1.xi' ### -E_ echo < eof1.xi +stdout xargs -E_ echo < eof1.xi firstline secondline +stdout parallel -k -E_ echo < eof1.xi firstline secondline _ thirdline fourthline +echo '### -iARG echo ARG is xARGx < files.xi' ### -iARG echo ARG is xARGx < files.xi +stdout xargs -iARG echo ARG is xARGx < files.xi /src/gnu/autoconf-1.11 is x/src/gnu/autoconf-1.11x /src/gnu/autoconf-1.11/README is x/src/gnu/autoconf-1.11/READMEx /src/gnu/autoconf-1.11/Makefile.in is x/src/gnu/autoconf-1.11/Makefile.inx @@ -955,6 +1166,7 @@ fourthline /src/gnu/autoconf-1.11/make-stds.texi is x/src/gnu/autoconf-1.11/make-stds.texix /src/gnu/autoconf-1.11/standards.info is x/src/gnu/autoconf-1.11/standards.infox /src/gnu/autoconf-1.11/texinfo.tex is x/src/gnu/autoconf-1.11/texinfo.texx +stdout parallel -k -iARG echo ARG is xARGx < files.xi /src/gnu/autoconf-1.11 is x/src/gnu/autoconf-1.11x /src/gnu/autoconf-1.11/README is x/src/gnu/autoconf-1.11/READMEx /src/gnu/autoconf-1.11/Makefile.in is x/src/gnu/autoconf-1.11/Makefile.inx @@ -977,13 +1189,16 @@ fourthline /src/gnu/autoconf-1.11/make-stds.texi is x/src/gnu/autoconf-1.11/make-stds.texix /src/gnu/autoconf-1.11/standards.info is x/src/gnu/autoconf-1.11/standards.infox /src/gnu/autoconf-1.11/texinfo.tex is x/src/gnu/autoconf-1.11/texinfo.texx +echo '### -i echo from \{\} to x{}y < items.xi' ### -i echo from \{\} to x{}y < items.xi +stdout xargs -i echo from \{\} to x{}y < items.xi from dumb to xdumby from s s to xs s y from f f to xf f y from r r to xr r y from t t to xt t y from v v to xv v y +stdout parallel -k -i echo from \{\} to x{}y < items.xi from: can't read /var/mail/{} from: can't read /var/mail/{} from: can't read /var/mail/{} @@ -992,28 +1207,37 @@ from: can't read /var/mail/{} from: can't read /var/mail/{} from: can't read /var/mail/{} from: can't read /var/mail/{} +echo '### -i -s26 echo from \{\} to x{}y < items.xi' ### -i -s26 echo from \{\} to x{}y < items.xi +stdout xargs -i -s26 echo from \{\} to x{}y < items.xi xargs: argument list too long from dumb to xdumby +stdout parallel -k -i -s26 echo from \{\} to x{}y < items.xi parallel: Error: Command line too long (36 >= 26) at number 0: ... +echo '### -i__ echo FIRST __ IS OK < quotes.xi' ### -i__ echo FIRST __ IS OK < quotes.xi +stdout xargs -i__ echo FIRST __ IS OK < quotes.xi FIRST this is IS OK FIRST quoted stuff IS OK FIRST and an embedded newline IS OK FIRST with single quotes as well. IS OK +stdout parallel -k -i__ echo FIRST __ IS OK < quotes.xi FIRST this is IS OK FIRST "quoted stuff" IS OK FIRST and \ IS OK FIRST an embedded newline IS OK FIRST with 'single quotes' as well. IS OK +echo '### -l echo < ldatab.xi' ### -l echo < ldatab.xi +stdout xargs -l echo < ldatab.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +stdout parallel -l -k echo < ldatab.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 @@ -1021,7 +1245,9 @@ FIRST with 'single quotes' as well. IS OK 666666 7777777 88888888 999999999 +echo '### -l echo < ldata.xi' ### -l echo < ldata.xi +stdout xargs -l echo < ldata.xi 1 22 333 4444 55555 666666 7777777 @@ -1031,6 +1257,7 @@ FIRST with 'single quotes' as well. IS OK 666666 7777777 88888888 999999999 +stdout parallel -l -k echo < ldata.xi 1 22 333 4444 55555 666666 7777777 @@ -1040,20 +1267,25 @@ FIRST with 'single quotes' as well. IS OK 666666 7777777 88888888 999999999 +echo '### -l1 -n4 echo < files.xi' ### -l1 -n4 echo < files.xi +stdout xargs -l1 -n4 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -l1 -n4 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -l2 echo < files.xi' ### -l2 echo < files.xi +stdout xargs -l2 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING @@ -1065,6 +1297,7 @@ FIRST with 'single quotes' as well. IS OK /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +stdout parallel -k -l2 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING @@ -1076,7 +1309,9 @@ FIRST with 'single quotes' as well. IS OK /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +echo '### -s30 -t echo < stairs.xi - xargs' ### -s30 -t echo < stairs.xi - xargs +stdout xargs -s30 -t echo < stairs.xi echo 1 22 333 4444 55555 1 22 333 4444 55555 echo 666666 7777777 88888888 @@ -1087,8 +1322,11 @@ echo 55555 666666 7777777 55555 666666 7777777 echo 88888888 999999999 88888888 999999999 +echo '### -s30 -t echo < stairs.xi - parallel' ### -s30 -t echo < stairs.xi - parallel +echo 'Because of -t these lines can be flipped around therefore sort' Because of -t these lines can be flipped around therefore sort +stdout parallel -k -X -s30 -t echo < stairs.xi | sort 1 22 333 4444 55555 55555 666666 7777777 666666 7777777 88888888 @@ -1101,14 +1339,19 @@ echo 666666 7777777 88888888 echo 88888888 echo 999999999 echo 999999999 1 22 333 4444 +echo '### -t echo this plus that < space.xi' ### -t echo this plus that < space.xi +stdout xargs -t echo this plus that < space.xi echo this plus that this plus that +stdout parallel -k -t echo this plus that < space.xi echo this plus that echo this plus that \ \ \ \ \ this plus that this plus that +echo '### -n1 printf "@%s@\n" < empty.xi' ### -n1 printf "@%s@\n" < empty.xi +stdout xargs -n1 printf "@%s@\n" < empty.xi @foo@ @@ @bar@ @@ -1116,11 +1359,15 @@ this plus that @@ @@ @fnord@ -@foo '' bar@n@baz ''@n@'' fnord@n### -n2 -t echo < foobar.xi +stdout parallel -k -n1 printf "@%s@\n" < empty.xi +@foo '' bar@n@baz ''@n@'' fnord@necho '### -n2 -t echo < foobar.xi' +### -n2 -t echo < foobar.xi +stdout xargs -n2 -t echo < foobar.xi echo foo bar foo bar echo baz ugh baz ugh +stdout parallel -k -n2 -t echo < foobar.xi echo foo\ bar baz\ echo ugh foo bar baz diff --git a/testsuite/wanted-results/parallel-local15 b/testsuite/wanted-results/parallel-local15 index e18f7d36..3db4b4fb 100644 --- a/testsuite/wanted-results/parallel-local15 +++ b/testsuite/wanted-results/parallel-local15 @@ -62,6 +62,7 @@ touch more_than_5000-9997 touch more_than_5000-9998 touch more_than_5000-9999 ### rm +ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort 1 1-col.txt 1 1-col.txt.diff 1 2-col.txt @@ -170,6 +171,7 @@ touch more_than_5000-9999 4 ab 4 b 6 a +ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort 1 1-col.txt 1 1-col.txt.diff 1 2-col.txt @@ -278,6 +280,7 @@ touch more_than_5000-9999 4 ab 4 b 6 a +find . -type f | parallel --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort ./ ab /c' d/ ef"g.diff ./ ab /c' d/ ef"g.diff ./ ab/c"d/ef g.diff @@ -296,6 +299,7 @@ touch more_than_5000-9999 ./a/bar.diff ./b/bar.diff ./b/bar.diff +find . -type f | parallel -v --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort ./ ab /c' d/ ef"g.diff ./ ab /c' d/ ef"g.diff ./ ab/c"d/ef g.diff @@ -435,6 +439,7 @@ perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ./more_than_5000-9996 perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ./more_than_5000-9997 perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ./more_than_5000-9998 perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ./more_than_5000-9999 +find . -type f | parallel -q --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' | sort ./ ab /c' d/ ef"g.diff ./ ab /c' d/ ef"g.diff ./ ab/c"d/ef g.diff @@ -453,6 +458,7 @@ perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ./more_than_5000-9999 ./a/bar.diff ./b/bar.diff ./b/bar.diff +find . -type f | parallel -qv --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' | sort ./ ab /c' d/ ef"g.diff ./ ab /c' d/ ef"g.diff ./ ab/c"d/ef g.diff diff --git a/testsuite/wanted-results/parallel-local150 b/testsuite/wanted-results/parallel-local150 index 349e76f9..9bc9b4c1 100644 --- a/testsuite/wanted-results/parallel-local150 +++ b/testsuite/wanted-results/parallel-local150 @@ -1,5 +1,8 @@ +echo '### bug #41565: Print happens in blocks - not after each job complete' ### bug #41565: Print happens in blocks - not after each job complete +echo 'The timing here is important: 2 3 4 5 6' The timing here is important: 2 3 4 5 6 + ping -c 7 lo | parallel -j3 'echo {#}' | timestamp -dd | perl -pe '$_=int($_+0.2)."\n"' 2 1 1 @@ -12,7 +15,9 @@ The timing here is important: 2 3 4 5 6 0 0 0 +echo '300 ms jobs:' 300 ms jobs: + ping -i .3 -c 10 lo | parallel -j3 --delay 0.3 echo | timestamp -d -d | perl -pe 's/(...).*/int($1*10+0.2)/e' | tail -n +2 3 3 3 @@ -27,31 +32,39 @@ The timing here is important: 2 3 4 5 6 3 3 3 +echo '### Test --tagstring' ### Test --tagstring + nice parallel -j1 -X -v --tagstring a{}b echo ::: 3 4 echo 3 4 a3b a4b 3 4 + nice parallel -j1 -k -v --tagstring a{}b echo ::: 3 4 echo 3 a3b 3 echo 4 a4b 4 + nice parallel -j1 -k -v --tagstring a{}b echo job{#} ::: 3 4 echo job1 a3b job1 echo job2 a4b job2 + nice parallel -j1 -k -v --tagstring ajob{#}b echo job{#} ::: 3 4 echo job1 ajob1b job1 echo job2 ajob2b job2 +echo '### Bug in --load'; nice parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3 ### Bug in --load 1 2 3 +echo '### Test --timeout'; nice parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 7.7 8.8 9.9 ### Test --timeout 1.1 1.1 7.7 8.8 9.9 +echo '### Test retired'; stdout parallel -B foo; stdout parallel -g; stdout parallel -H 1; stdout parallel -T; stdout parallel -U foo; stdout parallel -W foo; stdout parallel -Y; ### Test retired parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. @@ -109,32 +122,44 @@ parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. +echo '### Test --joblog followed by --resume --joblog' ### Test --joblog followed by --resume --joblog + rm -f /tmp/joblog; timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4; cat /tmp/joblog | wc -lw; rm -f /tmp/joblog; 5 49 +echo '### Test --resume --joblog followed by --resume --joblog'; rm -f /tmp/joblog2; timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4; cat /tmp/joblog2 | wc -lw; rm -f /tmp/joblog2; ### Test --resume --joblog followed by --resume --joblog 5 49 +echo '### Test --header'; printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.}; echo '### 64-bit wierdness - this did not complete on a 64-bit machine'; seq 1 2 | parallel -j1 'seq 1 1 | parallel true' ### Test --header 3/4.5 1.2 3/4 4.5 3 4 ### 64-bit wierdness - this did not complete on a 64-bit machine +echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'"; stdout bash -c 'parallel -k -a <(seq 1 3) echo' ### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo' 1 2 3 +echo "### bug #35268: shell_quote doesn't treats [] brackets correctly"; touch /tmp/foo1; stdout parallel echo ::: '/tmp/foo[123]' ### bug #35268: shell_quote doesn't treats [] brackets correctly /tmp/foo[123] +echo '### Test make .deb package'; cd ~/privat/parallel/packager/debian; stdout make | grep 'To install the GNU Parallel Debian package, run:' ### Test make .deb package To install the GNU Parallel Debian package, run: +echo '### Test of segfaulting issue' ### Test of segfaulting issue + echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; echo 'before adding wait() before exit'; seq 1 300 | stdout parallel ./trysegfault This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 before adding wait() before exit +echo '### Test basic --arg-sep'; parallel -k echo ::: a b ### Test basic --arg-sep a b +echo '### Run commands using --arg-sep'; parallel -kv ::: 'echo a' 'echo b' ### Run commands using --arg-sep echo a a echo b b +echo '### Change --arg-sep'; parallel --arg-sep ::: -kv ::: 'echo a' 'echo b'; parallel --arg-sep .--- -kv .--- 'echo a' 'echo b'; parallel --argsep ::: -kv ::: 'echo a' 'echo b'; parallel --argsep .--- -kv .--- 'echo a' 'echo b' ### Change --arg-sep echo a a @@ -152,25 +177,31 @@ echo a a echo b b +echo '### Test stdin goes to first command only' ### Test stdin goes to first command only + echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b' cat via cat echo b b + echo via cat |parallel -kv ::: 'cat' 'echo b' cat via cat echo b b +echo '### Bug made 4 5 go before 1 2 3'; parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5" ### Bug made 4 5 go before 1 2 3 1 2 3 4 5 +echo '### Bug made 3 go before 1 2'; parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3" ### Bug made 3 go before 1 2 1 2 3 +echo '### Bug did not quote'; echo '>' | parallel -v echo; parallel -v echo ::: '>'; (echo '>'; echo 2) | parallel -j1 -vX echo; parallel -X -j1 echo ::: '>' 2 ### Bug did not quote echo \> > @@ -179,6 +210,7 @@ echo \> echo \> 2 > 2 > 2 +echo '### Must not quote'; echo 'echo | wc -l' | parallel -v; parallel -v ::: 'echo | wc -l'; echo 'echo a b c | wc -w' | parallel -v; parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w' ### Must not quote echo | wc -l 1 @@ -190,7 +222,9 @@ echo a b c | wc -w 3 echo a b | wc -w 2 +echo '### Test bug #35820: sem breaks if $HOME is not writable' ### Test bug #35820: sem breaks if $HOME is not writable + echo 'Workaround: use another writable dir'; rm -rf /tmp/.parallel; HOME=/tmp sem echo OK; HOME=/tmp sem --wait; HOME=/usr/this/should/fail stdout sem echo should fail Workaround: use another writable dir OK parallel: Error: Cannot write to /usr/this/should/fail/.parallel: No such file or directory diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index b361b0a8..b49a69b4 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -1,10 +1,18 @@ +echo '### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name' ### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name + echo "echo echo from ./local_test_profile" > local_test_profile; parallel --profile local_test_profile echo ::: 1; rm local_test_profile echo from ./local_test_profile echo 1 +echo "### Test --delay" ### Test --delay +seq 9 | /usr/bin/time -f %e parallel -j3 --delay 0.57 true {} 2>&1 | perl -ne '$_ > 3.3 and print "More than 3.3 secs: OK\n"' More than 3.3 secs: OK +echo '### test --sshdelay' ### test --sshdelay + stdout /usr/bin/time -f %e parallel -j0 --sshdelay 0.5 -S localhost true ::: 1 2 3 | perl -ne 'print($_ > 1.30 ? "OK\n" : "Not OK\n")' OK +echo '### bug #38299: --resume-failed -k' ### bug #38299: --resume-failed -k + rm /tmp/joblog-38299; parallel -k --resume-failed --joblog /tmp/joblog-38299 echo job{#} val {}\;exit {} ::: 0 1 2 3 0 1; echo try 2. Gives failing - not 0; parallel -k --resume-failed --joblog /tmp/joblog-38299 echo job{#} val {}\;exit {} ::: 0 1 2 3 0 1; echo with exit 0; parallel -k --resume-failed --joblog /tmp/joblog-38299 echo job{#} val {}\;exit 0 ::: 0 1 2 3 0 1; echo try 2 again. Gives empty; parallel -k --resume-failed --joblog /tmp/joblog-38299 echo job{#} val {}\;exit {} ::: 0 1 2 3 0 1 job1 val 0 job2 val 1 job3 val 2 @@ -22,7 +30,9 @@ job3 val 2 job4 val 3 job6 val 1 try 2 again. Gives empty +echo '### --resume -k' ### --resume -k + rm -f /tmp/joblog-resume; parallel -k --resume --joblog /tmp/joblog-resume echo job{}id\;exit {} ::: 0 1 2 3 0 5; echo try 2 = nothing; parallel -k --resume --joblog /tmp/joblog-resume echo job{}id\;exit {} ::: 0 1 2 3 0 5; echo two extra; parallel -k --resume --joblog /tmp/joblog-resume echo job{}id\;exit 0 ::: 0 1 2 3 0 5 6 7 job0id job1id job2id @@ -33,24 +43,38 @@ try 2 = nothing two extra job6id job7id +echo "bug #37694: Empty string argument skipped when using --quote" bug #37694: Empty string argument skipped when using --quote + parallel -q --nonall perl -le 'print scalar @ARGV' 'a' 'b' '' 3 +echo '### Test -k 5'; sleep 5 ### Test -k 5 +echo '### Test -k 3'; sleep 3 ### Test -k 3 +echo '### Test -k 4'; sleep 4 ### Test -k 4 +echo '### Test -k 2'; sleep 2 ### Test -k 2 +echo '### Test -k 1'; sleep 1 ### Test -k 1 +echo "### Computing length of command line" ### Computing length of command line + seq 1 2 | parallel -k -N2 echo {1} {2} 1 2 + parallel --xapply -k -a <(seq 11 12) -a <(seq 1 3) echo 11 1 12 2 11 3 + parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d + parallel -k -C %+ echo {4} ::: 'a% c %%b' +echo "### test08"; cd input-files/test08; ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort; ### test08 b d +seq 1 10 | parallel -j 1 echo | sort 1 10 2 @@ -61,6 +85,7 @@ d 7 8 9 +seq 1 10 | parallel -j 2 echo | sort 1 10 2 @@ -71,6 +96,7 @@ d 7 8 9 +seq 1 10 | parallel -j 3 echo | sort 1 10 2 @@ -81,9 +107,13 @@ d 7 8 9 +echo "bug #37956: --colsep does not default to '\t' as specified in the man page." bug #37956: --colsep does not default to '\t' as specified in the man page. + printf "A\tB\n1\tone" | parallel --header : echo {B} {A} one 1 +echo '### Test --tollef' ### Test --tollef + stdout parallel -k --tollef echo -- 1 2 3 ::: a b c | sort parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. parallel: Error: -B has been retired. Use --bf. parallel: Error: -H has been retired. Use --halt. @@ -92,7 +122,9 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -g has been retired. Use --group. +echo '### Test --tollef --gnu' ### Test --tollef --gnu + stdout parallel -k --tollef --gnu echo ::: 1 2 3 -- a b c parallel: Error: -g has been retired. Use --group. parallel: Error: -B has been retired. Use --bf. parallel: Error: -T has been retired. Use --tty. @@ -101,7 +133,9 @@ parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load for -l. +echo '### Test --gnu' ### Test --gnu + parallel -k --gnu echo ::: 1 2 3 -- a b c 1 2 3 @@ -109,135 +143,185 @@ parallel: Error: --tollef has been retired. Use -u -q --arg-sep -- and --load fo a b c +echo '### Test {//}' ### Test {//} + parallel -k echo {//} {} ::: a a/b a/b/c . a a a/b a/b a/b/c + parallel -k echo {//} {} ::: /a /a/b /a/b/c / /a /a /a/b /a/b /a/b/c + parallel -k echo {//} {} ::: ./a ./a/b ./a/b/c . ./a ./a ./a/b ./a/b ./a/b/c + parallel -k echo {//} {} ::: a.jpg a/b.jpg a/b/c.jpg . a.jpg a a/b.jpg a/b a/b/c.jpg + parallel -k echo {//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg / /a.jpg /a /a/b.jpg /a/b /a/b/c.jpg + parallel -k echo {//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg . ./a.jpg ./a ./a/b.jpg ./a/b ./a/b/c.jpg +echo '### Test {1//}' ### Test {1//} + parallel -k echo {1//} {} ::: a a/b a/b/c . a a a/b a/b a/b/c + parallel -k echo {1//} {} ::: /a /a/b /a/b/c / /a /a /a/b /a/b /a/b/c + parallel -k echo {1//} {} ::: ./a ./a/b ./a/b/c . ./a ./a ./a/b ./a/b ./a/b/c + parallel -k echo {1//} {} ::: a.jpg a/b.jpg a/b/c.jpg . a.jpg a a/b.jpg a/b a/b/c.jpg + parallel -k echo {1//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg / /a.jpg /a /a/b.jpg /a/b /a/b/c.jpg + parallel -k echo {1//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg . ./a.jpg ./a ./a/b.jpg ./a/b ./a/b/c.jpg +echo '### Test --dnr' ### Test --dnr + parallel --dnr II -k echo II {} ::: a a/b a/b/c . a a a/b a/b a/b/c +echo '### Test --dirnamereplace' ### Test --dirnamereplace + parallel --dirnamereplace II -k echo II {} ::: a a/b a/b/c . a a a/b a/b a/b/c +echo '### Test https://savannah.gnu.org/bugs/index.php?31716' ### Test https://savannah.gnu.org/bugs/index.php?31716 + seq 1 5 | stdout parallel -k -l echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK + seq 1 5 | stdout parallel -k -l 1 echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -l -0' ### -k -l -0 + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -0 -l' ### -k -0 -l + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -0 -l 1' ### -k -0 -l 1 + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -0 -l 0' ### -k -0 -l 0 + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -0 -L -0 - -0 is argument for -L' ### -k -0 -L -0 - -0 is argument for -L + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK OK OK OK OK OK +echo '### -k -0 -L 0 - -L always takes arg' ### -k -0 -L 0 - -L always takes arg + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK OK OK OK OK OK +echo '### -k -0 -L 0 - -L always takes arg' ### -k -0 -L 0 - -L always takes arg + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK OK OK OK OK OK +echo '### -k -e -0' ### -k -e -0 + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -0 -e eof' ### -k -0 -e eof + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -i -0' ### -k -i -0 + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### -k -0 -i repl' ### -k -0 -i repl + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK 1 OK 2 OK 3 OK 4 OK 5 OK +echo '### Negative replacement strings' ### Negative replacement strings + parallel -X -j1 -N 6 echo {-1}orrec{1} ::: t B X D E c correct + parallel -N 6 echo {-1}orrect ::: A B X D E c correct + parallel --colsep ' ' echo '{2} + {4} = {2} + {-1}=' '$(( {2} + {-1} ))' ::: "1 2 3 4" 2 + 4 = 2 + 4= 6 + parallel --colsep ' ' echo '{-3}orrect' ::: "1 c 3 4" correct +echo 'bug #38439: "open files" with --files --pipe blocks after a while' bug #38439: "open files" with --files --pipe blocks after a while + ulimit -n 20; yes |head -n 10M | parallel --pipe -k echo {#} of 20 1 of 20 2 of 20 3 of 20 @@ -258,10 +342,14 @@ bug #38439: "open files" with --files --pipe blocks after a while 18 of 20 19 of 20 20 of 20 +echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' bug #34241: --pipe should not spawn unneeded processes - part 2 + seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null; ls *.par | wc -l; rm *.par; seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files --dry-run wc >/dev/null; echo No .par should exist; stdout ls *.par 2 No .par should exist ls: cannot access *.par: No such file or directory +echo "bug: --gnu was ignored if env var started with space: PARALLEL=' --gnu'" bug: --gnu was ignored if env var started with space: PARALLEL=' --gnu' + export PARALLEL=" -v" && parallel echo ::: 'space in envvar OK' echo space\ in\ envvar\ OK space in envvar OK diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index ee9417d4..8ffce207 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -55,7 +55,9 @@ rm -- 2-col/abc-2-col-2-col.txt rm -- a/abc-a-a rm -- b/abc-b-b rm -- \\\\\\\ \(Zh\\nggu\\\)/abc-\\\\\\\ \(Zh\\nggu\\\)-\\\\\\\ \(Zh\\nggu\\\) +echo '### Test compress' ### Test compress + seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10' 1 1 2 1 2 2 @@ -71,7 +73,9 @@ rm -- \ 5 3 5 4 5 5 +echo '### Test compress - stderr' ### Test compress - stderr + seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10 >&2' 2>&1 >/dev/null 1 1 2 1 2 2 @@ -87,48 +91,77 @@ rm -- \ 5 3 5 4 5 5 +echo '### Test weird regexp chars' ### Test weird regexp chars + seq 1 6 | parallel -j1 -I :: -X echo a::b::^c::[.}c a1b1^c1[.}c a2b2^c2[.}c a3b3^c3[.}c a4b4^c4[.}c a5b5^c5[.}c a6b6^c6[.}c +echo '### Test -m' ### Test -m +(echo foo;echo bar;echo joe.gif) | parallel -j1 -km echo 1{}2{.}3 A{.}B{.}C 1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC +(echo foo;echo bar;echo joe.gif) | parallel -j1 -kX echo 1{}2{.}3 A{.}B{.}C 1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC +seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -km echo a{}b{.}c{.} a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6 +seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -kX echo a{}b{.}c{.} a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 +echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -km echo a{}b{.}c{.} | tee >(wc) >(md5sum) >/dev/null; wait ### Test -m with 60000 args cded9cd15e00550b08e57afc0172caa8 - 12 180000 1286718 +echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc) >(md5sum) >/dev/null; wait ### Test -X with 60000 args 12de4813eda45d364a51bef697eee299 - 13 120000 1586682 +echo '### Test -X with 60000 args and 5 expansions' ### Test -X with 60000 args and 5 expansions +seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l 19 +seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l 15 +seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.} | wc -l 13 +seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c | wc -l 10 +seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b | wc -l 7 +echo '### Test {.} does not repeat more than {}' ### Test {.} does not repeat more than {} +seq 1 15 | perl -pe 's/$/.gif\n/' | parallel -j1 -s 80 -kX echo a{}b{.}c{.} a1.gifb1c1 abc a2.gifb2c2 abc a3.gifb3c3 abc a4.gifb4c4 abc a5.gifb5c5 abc a6.gifb6c6 abc a7.gifb7c7 abc a8.gifb8c8 abc a9.gifb9c9 abc a10.gifb10c10 abc a11.gifb11c11 abc a12.gifb12c12 abc a13.gifb13c13 abc a14.gifb14c14 abc a15.gifb15c15 abc +seq 1 15 | perl -pe 's/$/.gif\n/' | parallel -j1 -s 80 -km echo a{}b{.}c{.} a1.gif 2.gif 3.gif 4.gif 5.gif b1 2 3 4 5 c1 2 3 4 5 a6.gif 7.gif 8.gif 9.gif 10.gif b6 7 8 9 10 c6 7 8 9 10 a11.gif 12.gif 13.gif 14.gif b11 12 13 14 c11 12 13 14 a15.gif b15 c15 +seq 1 15 | perl -pe 's/$/.gif/' | parallel -j1 -s 80 -kX echo a{}b{.}c{.} a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 a7.gifb7c7 a8.gifb8c8 a9.gifb9c9 a10.gifb10c10 a11.gifb11c11 a12.gifb12c12 a13.gifb13c13 a14.gifb14c14 a15.gifb15c15 +seq 1 15 | perl -pe 's/$/.gif/' | parallel -j1 -s 80 -km echo a{}b{.}c{.} a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif 7.gifb1 2 3 4 5 6 7c1 2 3 4 5 6 7 a8.gif 9.gif 10.gif 11.gif 12.gif 13.gifb8 9 10 11 12 13c8 9 10 11 12 13 a14.gif 15.gifb14 15c14 15 +echo '### Test -I with shell meta chars' ### Test -I with shell meta chars +seq 1 60000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l 9 +seq 1 60000 | parallel -j1 -I '<>' -X echo 'a<>b<>c<>' | wc -l 9 +seq 1 60000 | parallel -j1 -I '<' -X echo 'a' -X echo 'a>b>c>' | wc -l 9 +echo '### Test {.}' ### Test {.} +echo a | parallel -qX echo "'"{.}"' " 'a' +echo a | parallel -qX echo "'{.}'" 'a' +(echo "sleep 3; echo begin"; seq 1 30 | parallel -kq echo "sleep 1; echo {.}"; echo "echo end") | parallel -k -j0 begin 1 2 @@ -161,7 +194,9 @@ begin 29 30 end +echo '### Test -I with -X and -m' ### Test -I with -X and -m +seq 1 10 | parallel -k 'seq 1 {.} | 'parallel' -k -I :: echo {.} ::' 1 1 2 1 2 2 @@ -217,6 +252,7 @@ end 10 8 10 9 10 10 +seq 1 10 | parallel -k 'seq 1 {.} | 'parallel' -j1 -X -k -I :: echo a{.} b::' a1 b1 a2 b1 b2 a3 b1 b2 b3 @@ -227,6 +263,7 @@ a7 b1 b2 b3 b4 b5 b6 b7 a8 b1 b2 b3 b4 b5 b6 b7 b8 a9 b1 b2 b3 b4 b5 b6 b7 b8 b9 a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 +seq 1 10 | parallel -k 'seq 1 {.} | 'parallel' -j1 -m -k -I :: echo a{.} b::' a1 b1 a2 b1 2 a3 b1 2 3 @@ -237,15 +274,23 @@ a7 b1 2 3 4 5 6 7 a8 b1 2 3 4 5 6 7 8 a9 b1 2 3 4 5 6 7 8 9 a10 b1 2 3 4 5 6 7 8 9 10 +echo '### Test -i' ### Test -i +(echo a; echo END; echo b) | parallel -k -i -eEND echo repl{.}ce replace +echo '### Test --replace' ### Test --replace +(echo a; echo END; echo b) | parallel -k --replace -eEND echo repl{.}ce replace +echo '### Test -t' ### Test -t +(echo b; echo c; echo f) | parallel -k -t echo {.}ar 2>&1 >/dev/null echo bar echo car echo far +echo '### Test --verbose' ### Test --verbose +(echo b; echo c; echo f) | parallel -k --verbose echo {.}ar 2>&1 >/dev/null echo bar echo car echo far diff --git a/testsuite/wanted-results/parallel-local2 b/testsuite/wanted-results/parallel-local2 index 07ee806d..46d42128 100644 --- a/testsuite/wanted-results/parallel-local2 +++ b/testsuite/wanted-results/parallel-local2 @@ -1,9 +1,17 @@ +echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first job" bug #38441: CPU usage goes to 100% if load is higher than --load at first job +/usr/bin/time -f %e parallel --load 100% true ::: a 2>&1 | perl -ne '$_ > 1 and print "More than 1 secs wall clock: OK\n"' More than 1 secs wall clock: OK +/usr/bin/time -f %U parallel --load 100% true ::: a 2>&1 | perl -ne '$_ < 1 and print "Less than 1 secs user time: OK\n"' Less than 1 secs user time: OK +echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834' ### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834 + seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo 1 2 3 +echo '### Test too slow spawning' ### Test too slow spawning +# Let the commands below run during high load +seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 & sleep 1; seq 1 1000 | stdout nice nice parallel -s 100 -uj0 true | perl -pe '/parallel: Warning: Starting \d+ processes took/ and do {close STDIN; `killall -9 burnP6`; print "OK\n"; exit }' OK diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 70602d0a..cf9b30f5 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -1,38 +1,53 @@ +echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; CHAR=$(cat /tmp/a$$ | wc -c); LINES=$(cat /tmp/a$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/awc$$; rm /tmp/a$$ /tmp/awc$$ ### Test of xargs -m command lines > 130k 31d9274be5fdc2de59487cb05ba57776 - Chars per line: 116300 6 119994 697800 +echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$ ### Test of xargs -X command lines > 130k 22074f9acada52462defb18ba912d744 - Chars per line: 116826 7 60000 817788 +echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum ### Test of xargs -m command lines > 130k b35d8e49be8d94899b719c40d3f1f4bb - +echo '### This causes problems if we kill child processes'; seq 2 40 | parallel -j 0 seq 1 10 | sort | md5sum ### This causes problems if we kill child processes 437c0d47a99b9a7c5bcb1d132f94c2e6 - +echo '### This causes problems if we kill child processes (II)'; seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort | md5sum ### This causes problems if we kill child processes (II) d7fb96d6a56d4347bc24930a395c431a - +echo '### Test -m'; (echo foo;echo bar) | parallel -j1 -m echo 1{}2{}3 A{}B{}C ### Test -m 1foo bar2foo bar3 Afoo barBfoo barC +echo '### Test -X'; (echo foo;echo bar) | parallel -j1 -X echo 1{}2{}3 A{}B{}C ### Test -X 1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC +echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop'; echo a | parallel -qX echo "'"{}"' " ### Bug before 2009-08-26 causing regexp compile error or infinite loop 'a' +echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop (II)'; echo a | parallel -qX echo "'{}'" ### Bug before 2009-08-26 causing regexp compile error or infinite loop (II) 'a' +echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL'; seq 1 2 | SHELL=tcsh MANPATH=. stdout parallel -k --nice 8 setenv a b\;echo \$SHELL ### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL tcsh 1 tcsh 2 +echo '### bug #42041: Implement $PARALLEL_JOBSLOT' ### bug #42041: Implement $PARALLEL_JOBSLOT + parallel -k --slotreplace // -j2 sleep 1\;echo // ::: {1..4} 1 2 1 2 + parallel -k -j2 sleep 1\;echo {%} ::: {1..4} 1 2 1 2 +echo '### bug #42363: --pipepart and --fifo/--cat does not work' ### bug #42363: --pipepart and --fifo/--cat does not work + seq 100 > /tmp/bug42363; parallel --pipepart --block 31 -a /tmp/bug42363 -k --fifo wc | perl -pe s:/tmp/...........pip:/tmp/XXXX: ; parallel --pipepart --block 31 -a /tmp/bug42363 -k --cat wc | perl -pe s:/tmp/...........pip:/tmp/XXXX: ; 13 14 32 /tmp/XXXX 11 11 33 /tmp/XXXX 11 11 33 /tmp/XXXX diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index bda4eeb5..122664da 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -1,6 +1,8 @@ +echo '### bug #42329: --line-buffer gives wrong output'; parallel --line-buffer --tag seq ::: 10000000 | wc -c; parallel --line-buffer seq ::: 10000000 | wc -c ### bug #42329: --line-buffer gives wrong output 168888897 78888897 +echo '### Test \0 as recend'; printf "a\0b\0c\0" | parallel --recend '\0' -k -N1 --pipe cat -v \; echo; printf "\0a\0b\0c" | parallel --recstart '\0' -k -N1 --pipe cat -v \; echo ### Test \0 as recend a^@ b^@ @@ -8,6 +10,7 @@ c^@ ^@a ^@b ^@c +echo '### Test filenames containing UTF-8'; cd tmp; find . -name '*.jpg' | nice nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; find |grep -v CVS | sort; echo '### bug #39554: Feature request: line buffered output'; parallel -j0 --linebuffer 'echo -n start {};sleep 0.{#};echo middle -n {};sleep 1.{#}5;echo next to last {};sleep 1.{#};echo -n last {}' ::: A B C ### Test filenames containing UTF-8 . ./1-col.txt @@ -29,7 +32,9 @@ start Cmiddle -n C next to last A next to last B next to last C -last Alast Blast C +last Alast Blast Cecho + +echo '### bug #39554: Feature request: line buffered output --tag'; parallel --tag -j0 --linebuffer 'echo -n start {};sleep 0.{#};echo middle -n {};sleep 1.{#}5;echo next to last {};sleep 1.{#};echo -n last {}' ::: A B C ### bug #39554: Feature request: line buffered output --tag A start Amiddle -n A B start Bmiddle -n B @@ -37,13 +42,17 @@ C start Cmiddle -n C A next to last A B next to last B C next to last C -A last AB last BC last C +A last AB last BC last Cecho + +echo '### test round-robin'; nice seq 1000 | parallel --block 1k --pipe --round-robin wc | sort ### test round-robin 223 223 893 250 250 1000 250 250 1000 277 277 1000 +echo '### --version must have higher priority than retired options' ### --version must have higher priority than retired options + parallel --version -g -Y -U -W -T | tail License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. @@ -54,17 +63,26 @@ When using programs that use GNU Parallel to process data for publication please O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. +echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken + nice perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo | perl -ne 'print length $_,"\n"' 131064 37830 +echo '### --delay should grow by 2 sec per arg' ### --delay should grow by 2 sec per arg +stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 | perl -ne '$_ >= 2 and $_ <= 5 and print "OK\n"' OK +stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 3 | perl -ne '$_ >= 4 and $_ <= 7 and print "OK\n"' OK +echo '### Exit value should not be affected if an earlier job times out' ### Exit value should not be affected if an earlier job times out + parallel -j2 --timeout 1 --joblog - -k ::: "sleep 10" "exit 255" | field 7 Exitval -1 255 +echo '### --header regexp' ### --header regexp + (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header '(%.*\n)*' echo JOB{#}\;cat JOB1 %head1 %head2 @@ -79,7 +97,9 @@ JOB3 %head1 %head2 5 +echo '### --header num' ### --header num + (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat JOB1 %head1 %head2 @@ -94,17 +114,23 @@ JOB3 %head1 %head2 5 +echo '### --header regexp --round-robin' ### --header regexp --round-robin + (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort 4 4 18 5 5 20 JOB JOB +echo '### --header num --round-robin' ### --header num --round-robin + (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort 4 4 18 5 5 20 JOB1 JOB2 +echo '### shebang-wrap' ### shebang-wrap + nice nice parallel -k {} {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] ./input-files/shebang/shebangwrap.oct A B diff --git a/testsuite/wanted-results/parallel-local5 b/testsuite/wanted-results/parallel-local5 index f1852c98..86b3ec37 100644 --- a/testsuite/wanted-results/parallel-local5 +++ b/testsuite/wanted-results/parallel-local5 @@ -1,25 +1,32 @@ ### Test --pipe +echo '### Test 200M records with too small block'; ( echo start; seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; echo end; ) | stdout parallel -k --block 200m -j2 --pipe --recend 'end\n' wc -c | egrep -v '^0$' ### Test 200M records with too small block parallel: Warning: A record was longer than 200000000. Increasing to --blocksize 260000001 303111434 303111434 303111434 +echo '### Test -N with multiple jobslots and multiple args' ### Test -N with multiple jobslots and multiple args +seq 1 1 | parallel -j2 -k -N 3 --pipe 'cat;echo a' 1 a +seq 1 2 | parallel -j2 -k -N 3 --pipe 'cat;echo bb' 1 2 bb +seq 1 3 | parallel -j2 -k -N 3 --pipe 'cat;echo ccc' 1 2 3 ccc +seq 1 4 | parallel -j2 -k -N 3 --pipe 'cat;echo dddd' 1 2 3 dddd 4 dddd +seq 1 5 | parallel -j2 -k -N 3 --pipe 'cat;echo eeeee' 1 2 3 @@ -27,6 +34,7 @@ eeeee 4 5 eeeee +seq 1 6 | parallel -j2 -k -N 3 --pipe 'cat;echo ffffff' 1 2 3 @@ -35,6 +43,7 @@ ffffff 5 6 ffffff +seq 1 7 | parallel -j2 -k -N 3 --pipe 'cat;echo ggggggg' 1 2 3 @@ -45,6 +54,7 @@ ggggggg ggggggg 7 ggggggg +seq 1 8 | parallel -j2 -k -N 3 --pipe 'cat;echo hhhhhhhh' 1 2 3 @@ -56,6 +66,7 @@ hhhhhhhh 7 8 hhhhhhhh +seq 1 9 | parallel -j2 -k -N 3 --pipe 'cat;echo iiiiiiiii' 1 2 3 @@ -68,6 +79,7 @@ iiiiiiiii 8 9 iiiiiiiii +seq 1 10 | parallel -j2 -k -N 3 --pipe 'cat;echo jjjjjjjjjj' 1 2 3 @@ -82,7 +94,9 @@ jjjjjjjjjj jjjjjjjjjj 10 jjjjjjjjjj +echo '### Test -l -N -L and -n with multiple jobslots and multiple args' ### Test -l -N -L and -n with multiple jobslots and multiple args +seq 1 12 | parallel -kj20 -l 2 --block 8 --pipe "cat; echo a" 1 2 3 @@ -99,6 +113,7 @@ a 11 12 a +seq 1 5 | parallel -kj2 -N 2 --pipe "cat; echo b" 1 2 b @@ -107,6 +122,7 @@ b b 5 b +seq 1 5 | parallel -kj2 -n 2 --pipe "cat; echo d" 1 2 d @@ -115,7 +131,9 @@ d d 5 d +echo '### Test -L --pipe' ### Test -L --pipe +seq 1 5 | parallel -kj2 -L 2 --pipe "cat; echo c" 1 2 3 @@ -123,48 +141,59 @@ d c 5 c +echo '### Test output is the same for different block size' ### Test output is the same for different block size +echo -n 01a02a0a0a12a34a45a6a | parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' 1>01a02a0a 2>0a12a34a 3>45a6a +echo -n 01a02a0a0a12a34a45a6a | stdout parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' parallel: Warning: A record was longer than 1. Increasing to --blocksize 3 parallel: Warning: A record was longer than 3. Increasing to --blocksize 5 1>01a02a0a parallel: Warning: A record was longer than 5. Increasing to --blocksize 8 2>0a12a34a 3>45a6a +echo '### Test 10M records with too big block'; ( echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; echo end; ) | stdout parallel -k --block 10M -j2 --pipe --recstart 'start\n' wc -c | egrep -v '^0$' ### Test 10M records with too big block 6888906 6888906 6888906 +echo '### Test --rrs -N1 --recend single'; echo 12a34a45a6 | parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' ### Test --rrs -N1 --recend single 1>12 2>34 3>45 4>6 +echo '### Test --rrs -N1 --regexp --recend alternate'; echo 12a34b45a6 | parallel -k --pipe --regexp --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' ### Test --rrs -N1 --regexp --recend alternate 1>12 2>34 3>45 4>6 +echo '### Test --rrs -N1 --recend single'; echo 12a34b45a6 | parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' ### Test --rrs -N1 --recend single 1>12a34 2>45a6 +echo '### Test --rrs --recend single'; echo 12a34a45a6 | parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' ### Test --rrs --recend single 1>123445 2>6 +echo '### Test --rrs --regexp --recend alternate'; echo 12a34b45a6 | parallel -k --pipe --regexp --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' ### Test --rrs --regexp --recend alternate 1>123445 2>6 +echo '### Test --rrs --recend single'; echo 12a34b45a6 | parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' ### Test --rrs --recend single 1>12a34 2>45a6 +echo '### Test -N even'; seq 1 10 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 0.\$PARALLEL_SEQ" ### Test -N even 1 2 @@ -181,6 +210,7 @@ ole 9 10 ole +echo '### Test -N odd'; seq 1 11 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 0.\$PARALLEL_SEQ" ### Test -N odd 1 2 @@ -199,6 +229,7 @@ ole ole 11 ole +echo '### Test -N even+2'; seq 1 12 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 1.\$PARALLEL_SEQ" ### Test -N even+2 1 2 @@ -218,10 +249,14 @@ ole 11 12 ole +echo '### Test --recstart + --recend'; cat /tmp/blocktest | parallel --block 1M -k --recstart 44 --recend "44" -j10 --pipe sort -n |md5sum ### Test --recstart + --recend 3c20e43c58152da30261c5827a1f9084 - +echo '### Race condition bug - 1 - would block'; seq 1 80 | nice parallel -j0 'seq 1 10| parallel --block 1 --recend "" --pipe cat;true' >/dev/null ### Race condition bug - 1 - would block +echo '### Race condition bug - 2 - would block'; seq 1 100 | nice parallel -j100 --block 1 --recend "" --pipe cat >/dev/null ### Race condition bug - 2 - would block +echo '### Test --block size=1'; seq 1 10| parallel --block 1 --files --recend "" --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} ### Test --block size=1 @@ -244,12 +279,16 @@ ole 7 8 9 +echo '### Test --block size=1M -j10 --files - more jobs than data'; sort -n < /tmp/blocktest | md5sum; cat /tmp/blocktest | parallel --files --recend "\n" -j10 --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --block size=1M -j10 --files - more jobs than data 8a7095c1c23bfadc311fe6b16d950582 - 8a7095c1c23bfadc311fe6b16d950582 - +echo '### Test --block size=1M -j1 - more data than cpu'; cat /tmp/blocktest | parallel --files --recend "\n" -j1 --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --block size=1M -j1 - more data than cpu 8a7095c1c23bfadc311fe6b16d950582 - +echo '### Test --block size=1M -j1 - more data than cpu'; cat /tmp/blocktest | parallel --files --recend "\n" -j2 --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --block size=1M -j1 - more data than cpu 8a7095c1c23bfadc311fe6b16d950582 - +echo '### Test --pipe default settings'; cat /tmp/blocktest | parallel --pipe sort | sort -n | md5sum ### Test --pipe default settings 8a7095c1c23bfadc311fe6b16d950582 - diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 16769458..c3ed118b 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -1,27 +1,60 @@ +echo 'bug #41613: --compress --line-buffer no newline'; bug #41613: --compress --line-buffer no newline + perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo It worked +echo 'bug #41613: --compress --line-buffer no --tagstring'; bug #41613: --compress --line-buffer no --tagstring + diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters +echo 'bug #41613: --compress --line-buffer with --tagstring'; bug #41613: --compress --line-buffer with --tagstring + diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters +echo 'bug #41613: --compress --line-buffer - no newline'; bug #41613: --compress --line-buffer - no newline + echo 'pipe compress tagstring' pipe compress tagstring -1 OK + perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat +1 O echo "K" +K + echo 'pipe compress notagstring' pipe compress notagstring -OK + perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat +O echo "K" +K + echo 'pipe nocompress tagstring' pipe nocompress tagstring -1 OK + perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat +1 O echo "K" +K + echo 'pipe nocompress notagstring' pipe nocompress notagstring -OK + perl -e 'print "O"'| parallel --pipe --line-buffer cat +O echo "K" +K + echo 'nopipe compress tagstring' nopipe compress tagstring -1 OK + parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n +1 O echo "K" +K + echo 'nopipe compress notagstring' nopipe compress notagstring -OK + parallel --compress --line-buffer echo {} O ::: -n +O echo "K" +K + echo 'nopipe nocompress tagstring' nopipe nocompress tagstring -1 OK + parallel --tagstring {#} --line-buffer echo {} O ::: -n +1 O echo "K" +K + echo 'nopipe nocompress notagstring' nopipe nocompress notagstring -OK + parallel --line-buffer echo {} O ::: -n +O echo "K" +K +echo 'bug #41412: --timeout + --delay causes deadlock'; bug #41412: --timeout + --delay causes deadlock + seq 10 | parallel -j10 --timeout 1 --delay .3 echo; 1 2 3 @@ -32,17 +65,21 @@ bug #41412: --timeout + --delay causes deadlock 8 9 10 + parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; 1 2 3 + parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 4 5 6 1 2 +echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; seq 1 5 | stdout parallel -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; ### Test --spreadstdin - more procs than args 1 2 3 4 5 +echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout parallel -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.* ### Test --spreadstdin - more args than procs 1 2 @@ -54,53 +91,101 @@ bug #41412: --timeout + --delay causes deadlock 8 9 10 +nice nice seq 1 1000 | nice nice parallel -j1 --spreadstdin cat "|cat "|wc -c 3893 +nice nice seq 1 10000 | nice nice parallel -j10 --spreadstdin cat "|cat "|wc -c 48894 +nice nice seq 1 100000 | nice nice parallel -j1 --spreadstdin cat "|cat "|wc -c 588895 +nice nice seq 1 1000000 | nice nice parallel -j10 --spreadstdin cat "|cat "|wc -c 6888896 +seq 1 10 | parallel --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz +echo '### Test --spreadstdin - similar to the failing below'; nice seq 1 100000 | nice nice parallel --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c) ### Test --spreadstdin - similar to the failing below +echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; nice seq 1 1000000 | nice nice parallel --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum ### Test --spreadstdin - this failed during devel 8a7095c1c23bfadc311fe6b16d950582 - 8a7095c1c23bfadc311fe6b16d950582 - +echo '### Test --spreadstdin -k'; nice seq 1 1000000 | nice nice parallel -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum ### Test --spreadstdin -k 8a7095c1c23bfadc311fe6b16d950582 - +echo '### Test --spreadstdin --files'; nice seq 1 1000000 | shuf | parallel --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --spreadstdin --files 8a7095c1c23bfadc311fe6b16d950582 - +echo '### Test --number-of-cpus'; stdout parallel --number-of-cpus ### Test --number-of-cpus 1 +echo '### Test --number-of-cores'; stdout parallel --number-of-cores ### Test --number-of-cores 8 +echo '### Test --use-cpus-instead-of-cores'; (seq 1 4 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 4 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 4 physical CPUs'; wait ### Test --use-cpus-instead-of-cores Cores should complete first on machines with less than 4 physical CPUs cores done CPUs done +echo '### Test --tag ::: a ::: b'; stdout parallel -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b ### Test --tag ::: a ::: b a b stdout-a b a b stderr-a b +echo '### Test --tag ::: a b'; stdout parallel -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag ::: a b a stdout-a a stderr-a b stdout-b b stderr-b +echo '### Test --tag -X ::: a b'; stdout parallel -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag -X ::: a b a b stdout-a stdout-b a b stderr-a stderr-b +echo '### Test bash redirection <()'; ### Test bash redirection <() + parallel 'cat <(echo {}); echo b' ::: a a b +echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' ### Test bug https://savannah.gnu.org/bugs/index.php?33352 +# produce input slowly to parallel so that it will reap a process +# while blocking in read() +# Having found the solution it is suddenly very easy to reproduce the +# problem - even on other hardware: +# +# perl -e '@x=1 .. 17000; for(1..30) { print "@x\n"}' | pv -qL 200000 +# |parallel -j2 --pipe --keeporder --block 150000 cat | md5sum +# +# This gives different md5sums for each run. +# +# The problem is that read(STDIN) is being interrupted by a dead +# child. The chance of this happening is very small if there are few +# children dying or read(STDIN) never has to wait for data. +# +# The test above forces data to arrive slowly (using pv) which causes +# read(STDIN) to take a long time - thus being interrupted by a dead +# child. +echo "# md5sum - directly" # md5sum - directly + perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum 350eda13a37912d755c9d733d149bdaf - +echo "# parallel | md5sum" # parallel | md5sum + nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR cat | md5sum 350eda13a37912d755c9d733d149bdaf - +echo "# --recend ''" # --recend '' + nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' cat | md5sum 350eda13a37912d755c9d733d149bdaf - +echo "# --recend '' --files" # --recend '' --files + nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - +echo "# --recend '' --files --tmpdir" # --recend '' --files --tmpdir + nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --tmpdir /dev/shm cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - +echo "# --recend '' --files --halt-on-error" # --recend '' --files --halt-on-error + nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - +echo '### Test of -j filename - non-existent file'; nice stdout parallel -j no_such_file echo ::: 1 ### Test of -j filename - non-existent file Usage: parallel [options] [command [arguments]] < list_of_arguments @@ -131,14 +216,18 @@ When using programs that use GNU Parallel to process data for publication please O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. +echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 ### Test of -j filename sleep 6 sleep 8 sleep 10 sleep 5 sleep 4 +echo '### Test ::::' ### Test :::: +echo '### Change --arg-file-sep' ### Change --arg-file-sep +$XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 @@ -150,6 +239,7 @@ sleep 4 9 13 10 14 1 15 +$XAP --arg-file-sep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 @@ -161,6 +251,7 @@ sleep 4 9 13 10 14 1 15 +$XAP --argfilesep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 @@ -172,6 +263,7 @@ sleep 4 9 13 10 14 1 15 +$XAP --argfilesep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 @@ -183,20 +275,30 @@ sleep 4 9 13 10 14 1 15 +echo '### Test xapply --max-replace-args' ### Test xapply --max-replace-args +seq 0 7 | $XAP -k --max-replace-args=3 echo {3} {2} {1} 2 1 0 5 4 3 7 6 +echo '### Test -N' ### Test -N +seq 1 5 | $XAP -kN3 echo {1} {2} {3} 1 2 3 4 5 +echo '### Test -N with 0' ### Test -N with 0 +seq 0 7 | $XAP -kN3 echo {1} {2} {3} 0 1 2 3 4 5 6 7 +echo '### Test :::: on nonexistent' ### Test :::: on nonexistent +stdout $XAP -k echo {1} {2} {3} :::: nonexistent parallel: Error: Cannot open input file `nonexistent': No such file or directory. +echo '### Test :::: two files' ### Test :::: two files +$XAP -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 @@ -208,81 +310,116 @@ parallel: Error: Cannot open input file `nonexistent': No such file or directory 9 13 10 14 1 15 +echo '### Test -d, ::::' ### Test -d, :::: +$XAP -kd, 'echo a{1} {2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) a1 5b a2 6b a3 7b a 8 b +echo '### Test -d, :::: one file too much' ### Test -d, :::: one file too much +$XAP -kd, echo 'a{1}' '{2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) <(echo 9,0) a1 5b a2 6b a3 7b a 8 b +echo '### Bug: did not quote' ### Bug: did not quote +$XAP echo {1} {2} :::: <(echo '>') <(echo b) > b +echo '### Quote test triplet 1' ### Quote test triplet 1 +$XAP -kv :::: <(echo 'echo a'; echo 'echo b') echo a a echo b b +$XAP -kv -a <(echo 'echo a'; echo 'echo b') echo a a echo b b +(echo 'echo a'; echo 'echo b') | $XAP -kv echo a a echo b b +echo '### Quote test triplet 2' ### Quote test triplet 2 +$XAP -kv echo :::: <(echo 'echo a'; echo 'echo b') echo echo\ a echo a echo echo\ b echo b +$XAP -kv -a <(echo 'echo a'; echo 'echo b') echo echo echo\ a echo a echo echo\ b echo b +(echo 'echo a'; echo 'echo b') | $XAP -kv echo echo echo\ a echo a echo echo\ b echo b +echo '### Quoting if there is a command and 2 arg files' ### Quoting if there is a command and 2 arg files +$XAP -kv echo :::: <(echo 'echo a') <(echo 'echo b') echo echo\ a echo\ b echo a echo b +echo '### Quoting if there is a command and 2 arg files of uneven length' ### Quoting if there is a command and 2 arg files of uneven length +$XAP -kv echo :::: <(echo 'echo a';echo a1) <(echo 'echo b') echo echo\ a echo\ b echo a echo b echo a1 echo\ b a1 echo b +echo '### Quoting if there is no command and 2 arg files' ### Quoting if there is no command and 2 arg files +$XAP -kv :::: <(echo 'echo a') <(echo 'echo b') echo a echo b a echo b +echo '### Quoting if there is no command and 2 arg files of uneven length' ### Quoting if there is no command and 2 arg files of uneven length +$XAP -kv :::: <(echo 'echo a';echo echo a1) <(echo 'echo b') echo a echo b a echo b echo a1 echo b a1 echo b +echo '### Test multiple -a' ### Test multiple -a +$XAP -kv -a <(echo a) -a <(echo b) echo {2} {1} echo b a b a +$XAP -kv echo {2} {1} :::: <(echo a) <(echo b) echo b a b a +echo '### Multiple -a: An unused file' ### Multiple -a: An unused file +$XAP -kv -a <(echo a) -a <(echo b) -a <(echo c) echo {2} {1} echo b a b a +$XAP -kv echo {2} {1} :::: <(echo a) <(echo b) <(echo c) echo b a b a +echo '### Multiple -a: nonexistent' ### Multiple -a: nonexistent +stdout $XAP -kv echo {2} {1} :::: nonexist nonexist2 parallel: Error: Cannot open input file `nonexist': No such file or directory. +stdout $XAP -kv -a nonexist -a nonexist2 echo {2} {1} parallel: Error: Cannot open input file `nonexist': No such file or directory. +echo '### Test {#.}' ### Test {#.} +$XAP -kv -a <(echo a-noext) -a <(echo b-withext.extension) -a <(echo c-ext.gif) echo {3.} {2.} {1.} echo c-ext b-withext a-noext c-ext b-withext a-noext +echo "### Tests that failed for OO-rewrite" ### Tests that failed for OO-rewrite +parallel -u --semaphore seq 1 10 '|' pv -qL 20; sem --wait; echo done 1 2 3 @@ -294,25 +431,38 @@ c-ext b-withext a-noext 9 10 done +echo a | parallel echo {1} a +echo "echo a" | parallel a +parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 a1b1^c1[.}c +echo "### BUG: The length for -X is not close to max (131072)" ### BUG: The length for -X is not close to max (131072) +seq 1 4000 | parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc 1 12821 131060 +echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit +echo | parallel --show-limits Maximal size of command: 131071 Maximal used size of command: 131071 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please press CTRL-D or CTRL-C +echo '### Test -N' ### Test -N +seq 1 5 | parallel -kN3 echo {1} {2} {3} 1 2 3 4 5 +echo '### Test --arg-file-sep with files of different lengths' ### Test --arg-file-sep with files of different lengths +parallel --xapply --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) 1 3 1 4 +echo '### Test respect -s' ### Test respect -s +parallel -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 echo 1 22 333 1 22 333 echo 4444 @@ -327,31 +477,53 @@ echo 88888888 88888888 echo 999999999 999999999 +echo '### Test eof string after :::' ### Test eof string after ::: +parallel -k -E ole echo ::: foo ole bar foo +echo '### Test -C and --trim rl' ### Test -C and --trim rl +parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d +echo '### Test empty input' ### Test empty input + 9 simultaneous 98 99 100 +echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' ### --filter-hosts - OK, non-such-user, connection refused, wrong host + parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname aspire +echo '### test --workdir . in $HOME' ### test --workdir . in $HOME + cd && mkdir -p parallel-test && cd parallel-test && echo OK > testfile && parallel --workdir . --transfer -S parallel@parallel-server3 cat {} ::: testfile OK +echo '### test --timeout --retries' ### test --timeout --retries + parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 parallel@parallel-server3 parallel@lo parallel@parallel-server2 parallel@parallel-server3 parallel@lo parallel@parallel-server2 +echo '### test --filter-hosts with server w/o ssh, non-existing server' ### test --filter-hosts with server w/o ssh, non-existing server + parallel -S 192.168.1.197,8.8.8.8,parallel@parallel-server3,parallel@lo,parallel@parallel-server2 --filter-hosts --nonall -k --tag echo parallel@lo parallel@parallel-server2 parallel@parallel-server3 +echo '### Missing: test --filter-hosts proxied through the one host' ### Missing: test --filter-hosts proxied through the one host diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 index c5886a1a..9b034bd1 100644 --- a/testsuite/wanted-results/test15 +++ b/testsuite/wanted-results/test15 @@ -29,120 +29,170 @@ sleep 0.1; echo opt--interactive 3 ?...y opt--interactive 1 opt--interactive 3 +echo '### Test killing children with --timeout and exit value (failed if timed out)' ### Test killing children with --timeout and exit value (failed if timed out) + pstree | grep sleep | grep -v anacron | grep -v screensave | wc; parallel --timeout 3 'true {} ; for i in `seq 100 120`; do bash -c "(sleep $i)" & sleep $i & done; wait; echo No good' ::: 1000000000 1000000001 ; echo $?; sleep 7; pstree | grep sleep | grep -v anacron | grep -v screensave | wc 0 0 0 2 0 0 0 +echo '### Test -L -l and --max-lines' ### Test -L -l and --max-lines +(echo a_b;echo c) | parallel -km -L2 echo a_b c +(echo a_b;echo c) | parallel -k -L2 echo a_b c +(echo a_b;echo c) | xargs -L2 echo a_b c +echo '### xargs -L1 echo' ### xargs -L1 echo +(echo a_b;echo c) | parallel -km -L1 echo a_b c +(echo a_b;echo c) | parallel -k -L1 echo a_b c +(echo a_b;echo c) | xargs -L1 echo a_b c +echo 'Lines ending in space should continue on next line' Lines ending in space should continue on next line +echo '### xargs -L1 echo' ### xargs -L1 echo +(echo a_b' ';echo c;echo d) | parallel -km -L1 echo a_b c d +(echo a_b' ';echo c;echo d) | parallel -k -L1 echo a_b c d +(echo a_b' ';echo c;echo d) | xargs -L1 echo a_b c d +echo '### xargs -L2 echo' ### xargs -L2 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo a_b c d e +echo '### xargs -l echo' ### xargs -l echo +(echo a_b' ';echo c;echo d;echo e) | parallel -l -km echo # This behaves wrong a_b c d e +(echo a_b' ';echo c;echo d;echo e) | parallel -l -k echo # This behaves wrong a_b c d e +(echo a_b' ';echo c;echo d;echo e) | xargs -l echo a_b c d e +echo '### xargs -l2 echo' ### xargs -l2 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | parallel -k -l2 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo a_b c d e +echo '### xargs -l1 echo' ### xargs -l1 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | parallel -k -l1 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | xargs -l1 echo a_b c d e +echo '### xargs --max-lines=2 echo' ### xargs --max-lines=2 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo a_b c d e +(echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo a_b c d e +echo '### xargs --max-lines echo' ### xargs --max-lines echo +(echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -km echo # This behaves wrong a_b c d e +(echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -k echo # This behaves wrong a_b c d e +(echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo a_b c d e +echo '### test too long args' ### test too long args +perl -e 'print "z"x1000000' | parallel echo 2>&1 parallel: Error: Command line too long (1000005 >= 131071) at number 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +perl -e 'print "z"x1000000' | xargs echo 2>&1 xargs: argument line too long +(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo 1 2 3 4 5 6 7 8 9 10 parallel: Error: Command line too long (1000007 >= 10) at number 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout xargs -s 10 echo 1 2 3 4 5 6 7 8 xargs: argument line too long 9 10 +(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -kX -s 10 echo 1 2 3 4 5 6 7 8 9 10 parallel: Error: Command line too long (1000007 >= 10) at number 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +echo '### Test -x' ### Test -x +(seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo 1 2 3 4 5 6 7 8 9 10 parallel: Error: Command line too long (10 >= 10) at number 0: 12345... +(seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -kX -s 10 -x echo 1 2 3 4 5 6 7 8 9 10 parallel: Error: Command line too long (10 >= 10) at number 0: 12345... +(seq 1 10; echo 12345; seq 12 15) | stdout xargs -s 10 -x echo 1 2 3 4 5 6 xargs: argument line too long 7 8 +(seq 1 10; echo 1234; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo 1 2 3 4 5 6 @@ -153,6 +203,7 @@ xargs: argument line too long 13 14 15 +(seq 1 10; echo 1234; seq 12 15) | stdout parallel -j1 -kX -s 10 -x echo 1 2 3 4 5 6 @@ -163,6 +214,7 @@ xargs: argument line too long 13 14 15 +(seq 1 10; echo 1234; seq 12 15) | stdout xargs -s 10 -x echo 1 2 3 4 5 6 @@ -173,7 +225,9 @@ xargs: argument line too long 13 14 15 +echo '### Test -a and --arg-file: Read input from file instead of stdin' ### Test -a and --arg-file: Read input from file instead of stdin +seq 1 10 >/tmp/$$-1; parallel -k -a /tmp/$$-1 echo 1 2 3 @@ -184,6 +238,7 @@ xargs: argument line too long 8 9 10 +seq 1 10 >/tmp/$$-2; parallel -k --arg-file /tmp/$$-2 echo 1 2 3 diff --git a/testsuite/wanted-results/test30 b/testsuite/wanted-results/test30 index d8500251..61824a10 100644 --- a/testsuite/wanted-results/test30 +++ b/testsuite/wanted-results/test30 @@ -1,29 +1,46 @@ +echo '### Test of --eta' ### Test of --eta + seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l 16 +echo '### Test of --eta with no jobs' ### Test of --eta with no jobs + stdout parallel --eta "sleep 1; echo {}" < /dev/null Computers / CPU cores / Max jobs to run 1:local / 8 / 1 0 +echo '### Test of --progress' ### Test of --progress + seq 1 10 | stdout parallel --progress "sleep 1; echo {}" | wc -l 16 +echo '### Test of --progress with no jobs' ### Test of --progress with no jobs + stdout parallel --progress "sleep 1; echo {}" < /dev/null Computers / CPU cores / Max jobs to run 1:local / 8 / 1 0 +echo '### bug #34422: parallel -X --eta crashes with div by zero' ### bug #34422: parallel -X --eta crashes with div by zero +# We do not care how long it took + seq 2 | stdout parallel -X --eta echo | grep -E -v 'ETA:.*AVG' Computers / CPU cores / Max jobs to run 1:local / 8 / 1 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete local:1/0/100%/0.0s 1 2 +echo '### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed' ### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed + parallel -j0 --timeout 6 --onall -S localhost,parallel@parallel-server1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $? slept 1 slept 1 jobs failed: 2 +echo '### --pipe without command' ### --pipe without command + seq -w 10 | stdout parallel --pipe parallel: Error: --pipe must have a command to pipe into (e.g. 'cat'). +echo '### bug #36260: {n} expansion in --colsep files fails for empty fields if all following fields are also empty' ### bug #36260: {n} expansion in --colsep files fails for empty fields if all following fields are also empty + echo A,B,, | parallel --colsep , echo {1}{3}{2} AB diff --git a/testsuite/wanted-results/test45 b/testsuite/wanted-results/test45 index f2dc1e8c..526b33ab 100644 --- a/testsuite/wanted-results/test45 +++ b/testsuite/wanted-results/test45 @@ -1,19 +1,37 @@ +echo "### BUG: The length for -X is not close to max (131072)"; seq 1 60000 | nice parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc ### BUG: The length for -X is not close to max (131072) 1 12821 131060 + seq 1 60000 | nice parallel -X echo a{}b{}c |head -n 1 |wc 1 10948 131060 + seq 1 60000 | nice parallel -X echo |head -n 1 |wc 1 23695 131064 + seq 1 60000 | nice parallel -X echo a{}b{}c {} |head -n 1 |wc 1 15810 131064 + seq 1 60000 | nice parallel -X echo {}aa{} |head -n 1 |wc 1 11790 131058 + seq 1 60000 | nice parallel -X echo {} aa {} |head -n 1 |wc 1 25545 131055 +echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin' ### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin + seq 1 150 | stdout nice parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null" +echo '### Test --load locally - should take >10s' ### Test --load locally - should take >10s + echo '# This will run 10 processes in parallel for 10s'; seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" & # This will run 10 processes in parallel for 10s + stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' OK +echo '### Test --load remote' ### Test --load remote + ssh parallel@parallel-server1 'seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null"' & + stdout /usr/bin/time -f %e parallel -S parallel@parallel-server1 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' OK +echo '### Test --load read from a file - more than 3s' ### Test --load read from a file - more than 3s + echo '# This will run 10 processes in parallel for 10s'; seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" & ( echo 8 > /tmp/parallel_load_file; sleep 10; echo 1000 > /tmp/parallel_load_file ) & sleep 1;stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 9 and print "OK\n"' # This will run 10 processes in parallel for 10s OK +echo '### Test --load read from a file - less than 10s'; echo '# This will run 10 processes in parallel for 10s'; seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" & ### Test --load read from a file - less than 10s # This will run 10 processes in parallel for 10s + ( echo 8 > /tmp/parallel_load_file2; sleep 10; echo 1000 > /tmp/parallel_load_file2 ) & sleep 1;stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 20 and print "OK\n"' OK diff --git a/testsuite/wanted-results/test60 b/testsuite/wanted-results/test60 index b87c3122..8ad59c89 100644 --- a/testsuite/wanted-results/test60 +++ b/testsuite/wanted-results/test60 @@ -1,3 +1,4 @@ +echo '### Test --onall'; parallel --onall -S parallel@parallel-server1,parallel@parallel-server2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 ### Test --onall 1 2 @@ -11,6 +12,7 @@ 2 1 2 +echo '### Test | --onall'; seq 3 | parallel --onall -S parallel@parallel-server1,parallel@parallel-server2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - ### Test | --onall 1 2 @@ -30,6 +32,7 @@ 1 2 3 +echo '### Test --onall -u'; parallel --onall -S parallel@parallel-server1,parallel@parallel-server2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort ### Test --onall -u 1 1 @@ -49,24 +52,31 @@ 3 3 3 +echo '### Test --nonall'; parallel --nonall -k -S parallel@parallel-server1,parallel@parallel-server2 'hostname' | sort ### Test --nonall centos3.tange.dk redhat9.tange.dk +echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall -S parallel@parallel-server1,parallel@parallel-server2 -u 'hostname|grep -q redhat9 && sleep 3; hostname;sleep 12;hostname;' ### Test --nonall -u - should be interleaved x y x y centos3.tange.dk redhat9.tange.dk centos3.tange.dk redhat9.tange.dk +echo '### Test read sshloginfile from STDIN'; echo parallel@parallel-server1 | parallel -S - --nonall hostname; echo parallel@parallel-server1 | parallel --sshloginfile - --nonall hostname ### Test read sshloginfile from STDIN redhat9.tange.dk redhat9.tange.dk +echo '### Test --nonall --basefile'; touch /tmp/nonall--basefile; parallel --nonall --basefile /tmp/nonall--basefile -S parallel@parallel-server1,parallel@parallel-server2 ls /tmp/nonall--basefile ### Test --nonall --basefile /tmp/nonall--basefile /tmp/nonall--basefile +echo '### Test --onall --basefile'; touch /tmp/onall--basefile; parallel --onall --basefile /tmp/onall--basefile -S parallel@parallel-server1,parallel@parallel-server2 ls ::: /tmp/onall--basefile ### Test --onall --basefile /tmp/onall--basefile /tmp/onall--basefile +echo '### Test --workdir .'; ssh parallel@parallel-server1 mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S parallel@parallel-server1 ::: pwd ### Test --workdir . /home/parallel/mydir +echo '### Test --wd .'; ssh parallel@parallel-server2 mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S parallel@parallel-server2 ::: pwd ### Test --wd . /home/parallel/mydir diff --git a/testsuite/wanted-results/test61 b/testsuite/wanted-results/test61 index 8e81d490..0ff31da1 100644 --- a/testsuite/wanted-results/test61 +++ b/testsuite/wanted-results/test61 @@ -1,22 +1,33 @@ +echo '### Test --return of weirdly named file' ### Test --return of weirdly named file +stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b' ssh -tt -oLogLevel=quiet parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status; +echo '### Test if remote login shell is csh' ### Test if remote login shell is csh +stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c ssh -tt -oLogLevel=quiet csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$PARALLEL_PID\ \$PARALLEL_SEQ\ a\|\ wc\ -w; 3 ssh -tt -oLogLevel=quiet csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$PARALLEL_PID\ \$PARALLEL_SEQ\ b\|\ wc\ -w; 3 ssh -tt -oLogLevel=quiet csh@localhost 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$PARALLEL_PID\ \$PARALLEL_SEQ\ c\|\ wc\ -w; 3 +echo '### Test {} multiple times in different commands' ### Test {} multiple times in different commands +seq 10 | parallel -v -Xj1 echo {} \; echo {} echo 1 2 3 4 5 6 7 8 9 10 ; echo 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 +echo '### Test of -X {1}-{2} with multiple input sources' ### Test of -X {1}-{2} with multiple input sources +parallel -j1 -kX echo {1}-{2} ::: a ::: b a-b +parallel -j2 -kX echo {1}-{2} ::: a b ::: c d a-c b-c +parallel -j2 -kX echo {1}-{2} ::: a b c ::: d e f a-d b-f +parallel -j0 -kX echo {1}-{2} ::: a b c ::: d e f a-d a-e a-f @@ -26,12 +37,17 @@ b-f c-d c-e c-f +echo '### Test of -X {}-{.} with multiple input sources' ### Test of -X {}-{.} with multiple input sources +parallel -j1 -kX echo {}-{.} ::: a ::: b a-a b-b +parallel -j2 -kX echo {}-{.} ::: a b ::: c d a-a c-c a-a d-d b-b c-c b-b d-d +parallel -j2 -kX echo {}-{.} ::: a b c ::: d e f a-a d-d a-a e-e a-a f-f b-b d-d b-b e-e b-b f-f c-c d-d c-c e-e c-c f-f +parallel -j0 -kX echo {}-{.} ::: a b c ::: d e f a-a d-d a-a e-e a-a f-f @@ -41,10 +57,15 @@ b-b f-f c-c d-d c-c e-e c-c f-f +echo '### Test of -r with --pipe - the first should give an empty line. The second should not.' ### Test of -r with --pipe - the first should give an empty line. The second should not. +echo | parallel -j2 -N1 --pipe cat | wc -l 1 +echo | parallel -r -j2 -N1 --pipe cat | wc -l 0 +echo '### Test --tty' ### Test --tty +seq 0.1 0.1 0.5 | parallel -j1 --tty tty\;sleep /dev/tty /dev/tty /dev/tty diff --git a/testsuite/wanted-results/test65 b/testsuite/wanted-results/test65 index a2497f88..ad0304fb 100644 --- a/testsuite/wanted-results/test65 +++ b/testsuite/wanted-results/test65 @@ -1,4 +1,6 @@ +echo "### Test memory consumption stays (almost) the same for 30 and 300 jobs" ### Test memory consumption stays (almost) the same for 30 and 300 jobs + out30=$( stdout memusg parallel -j2 true :::: <(perl -e '$a="x"x100000;for(1..30){print $a,"\n"}') ); out300=$( stdout memusg parallel -j2 true :::: <(perl -e '$a="x"x100000;for(1..300){print $a,"\n"}') ); mem30=$(echo $out30 | tr -cd 0-9); mem300=$(echo $out300 | tr -cd 0-9); echo "Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 150% "; echo $(($mem300*100 < $mem30 * 150)) Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 150% 1 ### Test --shellquote