From c81a15677d7aefab432ad4c66dafb1db2aedcdfe Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 16 Jun 2010 05:03:52 +0200 Subject: [PATCH] Some systems does not receive the SIGCHLD (maybe when running under /bin/dash?). Unittest failed if /bin/sh was dash. --- autom4te.cache/output.0 | 20 ++-- autom4te.cache/output.1 | 20 ++-- autom4te.cache/traces.1 | 2 +- configure | 20 ++-- configure.ac | 2 +- doc/FUTURE_IDEAS | 105 ++++++++++++----- doc/release_new_version | 2 +- src/parallel | 13 ++- unittest/Makefile | 5 + unittest/actual-results/test02 | 7 +- unittest/actual-results/test16 | 4 +- unittest/actual-results/test18 | 35 +++--- unittest/actual-results/test19 | 201 -------------------------------- unittest/actual-results/test21 | 4 +- unittest/tests-to-run/test02.sh | 3 +- unittest/tests-to-run/test16.sh | 6 +- unittest/tests-to-run/test17.sh | 10 +- unittest/tests-to-run/test18.sh | 2 +- unittest/tests-to-run/test19.sh | 2 +- unittest/tests-to-run/test21.sh | 2 +- unittest/tests-to-run/test23.sh | 4 +- unittest/tests-to-run/test24.sh | 4 + unittest/wanted-results/test02 | 7 +- unittest/wanted-results/test16 | 4 +- unittest/wanted-results/test18 | 18 +-- unittest/wanted-results/test19 | 10 +- unittest/wanted-results/test21 | 4 +- 27 files changed, 190 insertions(+), 326 deletions(-) create mode 100644 unittest/tests-to-run/test24.sh diff --git a/autom4te.cache/output.0 b/autom4te.cache/output.0 index 2812890a..e41b97bc 100644 --- a/autom4te.cache/output.0 +++ b/autom4te.cache/output.0 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100601. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100616. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100601' -PACKAGE_STRING='parallel 20100601' +PACKAGE_VERSION='20100616' +PACKAGE_STRING='parallel 20100616' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100601 to adapt to many kinds of systems. +\`configure' configures parallel 20100616 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100601:";; + short | recursive ) echo "Configuration of parallel 20100616:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100601 +parallel configure 20100616 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100601, which was +It was created by parallel $as_me 20100616, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100601' + VERSION='20100616' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100601, which was +This file was extended by parallel $as_me 20100616, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100601 +parallel config.status 20100616 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/output.1 b/autom4te.cache/output.1 index 2812890a..e41b97bc 100644 --- a/autom4te.cache/output.1 +++ b/autom4te.cache/output.1 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100601. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100616. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100601' -PACKAGE_STRING='parallel 20100601' +PACKAGE_VERSION='20100616' +PACKAGE_STRING='parallel 20100616' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100601 to adapt to many kinds of systems. +\`configure' configures parallel 20100616 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100601:";; + short | recursive ) echo "Configuration of parallel 20100616:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100601 +parallel configure 20100616 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100601, which was +It was created by parallel $as_me 20100616, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100601' + VERSION='20100616' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100601, which was +This file was extended by parallel $as_me 20100616, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100601 +parallel config.status 20100616 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/traces.1 b/autom4te.cache/traces.1 index 5c12546e..8c884ffe 100644 --- a/autom4te.cache/traces.1 +++ b/autom4te.cache/traces.1 @@ -1,4 +1,4 @@ -m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100601], [bug-parallel@gnu.org]) +m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100616], [bug-parallel@gnu.org]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS']) diff --git a/configure b/configure index 188dc472..0c51185b 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.65 for parallel 20100601. +# Generated by GNU Autoconf 2.65 for parallel 20100616. # # Report bugs to . # @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100601' -PACKAGE_STRING='parallel 20100601' +PACKAGE_VERSION='20100616' +PACKAGE_STRING='parallel 20100616' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100601 to adapt to many kinds of systems. +\`configure' configures parallel 20100616 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100601:";; + short | recursive ) echo "Configuration of parallel 20100616:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100601 +parallel configure 20100616 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100601, which was +It was created by parallel $as_me 20100616, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100601' + VERSION='20100616' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100601, which was +This file was extended by parallel $as_me 20100616, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100601 +parallel config.status 20100616 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 2fef886c..4b5b6376 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20100601], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20100616], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index da118c15..becb7e4c 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,6 +1,6 @@ ---progress -Fixed bug if transfered file contains : - +Some systems does not receive the SIGCHLD (maybe when running under +dash?). +Unittest failed if /bin/sh was dash. * 100% options complete with xargs. All options for xargs can now be used in GNU Parallel - even the more exotic. @@ -21,6 +21,8 @@ Fixed bug if transfered file contains : * New video showing the new options. + + =head1 YouTube video GNU Parallel is a tool with lots of uses in shell. Every time you use @@ -32,17 +34,28 @@ it easy to distribute jobs to these. terminal2: ssh parallel@vh2.pi.dk ssh parallel@vh2.pi.dk + and - -PS1="\e[7mGNU Parallel:\[\033[01;34m\]\w\[\033[00m\]\e[27m$ " +PS1="\[\e[7m\]GNU Parallel:\[\033[01;34m\]\w\[\033[00m\e[27m\]$ " gunzip logs/*gz -rm logs/*bz2 +rm -f logs/*bz2* rm -rf zip/*[^p] +rm -rf dirs/* +rm -rf parallel-*bz2 + +xvidcap +ffmpeg -i 20100615_002.mp4 -ab 320k -ar 44100 speak.mp3 +# Merge video using youtube +#ffmpeg -i speak.mp3 -i xvidcap.mpeg -target mpeg -hq -minrate 8000000 \ +#-title "GNU Parallel" -author "Ole Tange" -copyright "(CC-By-SA) 2010" -comment "Intro video of GNU Parallel 20100615" videoaudio.mpg + +# GNU PARALLEL - BASIC USAGE +# A GNU tool for parallelizing shell commands # GET GNU PARALLEL -wget ftp://ftp.gnu.org/gnu/parallel/parallel-20100601.tar.bz2 -tar xvjf parallel-20100601.tar.bz2 -cd parallel-20100601 +wget ftp://ftp.gnu.org/gnu/parallel/parallel-20100615.tar.bz2 +tar xjf parallel-20100615.tar.bz2 +cd parallel-20100615 ./configure && make ## su make install @@ -52,20 +65,39 @@ cd # YOUR FIRST PARALLEL JOBS cd logs du -/usr/bin/time gzip * +/usr/bin/time gzip -1 * +## 24 sek - 22 sek /usr/bin/time gunzip * -ls | time parallel gzip +## 24 sek - 18 +ls | time parallel gzip -1 +## 17 sek - 10 ls | time parallel gunzip +## 25 sek - 19 # RECOMPRESS gz TO bz2 -ls | time parallel gzip -ls *.gz | time parallel 'zcat {} | bzip2 > {.}.bz2' +ls | time parallel gzip -1 +ls *.gz | time parallel -j+0 --progress 'zcat {} | bzip2 -1 > {.}.bz2' +## Explain command line ## vis top local -# RECOMPRESS gz TO bz2 USING local(:) AND REMOTE server1 -ls *.gz | time parallel -S server1,: --trc {.}.bz2 \ -'zcat {} | bzip2 > {.}.bz2' +## Man that is boring +## 2m41s - 2m + +# RECOMPRESS gz TO bz2 USING local(:) AND REMOTE server1-3 +ls *.gz |time parallel -j+0 --progress -Sserver1,server2,server3,: \ +--transfer --return {.}.bz2 --cleanup 'zcat {} | bzip2 -1 > {.}.bz2' +## Explain command line +## Explain server config ## vis top local -## vis top remote +## vis top remote1-3 +## 49 sek + +# RECOMPRESS gz TO bz2 USING A SCRIPT ON local AND REMOTE server1-3 +# (imagine the script is way more complex) +cp ../recompress /tmp +cat /tmp/recompress +ls *.gz |time parallel -j+0 --progress -Sserver1,server2,server3,: \ +--trc {.}.bz2 --basefile /tmp/recompress '/tmp/recompress {} {.}.bz2' + # MAKING SMALL SCRIPTS cd ../zip @@ -76,38 +108,55 @@ ls -l # GROUP OUTPUT traceroute debian.org traceroute debian.org & traceroute freenetproject.org ### -(echo debian.org; echo freenetproject.org) \ -| parallel traceroute +(echo debian.org; echo freenetproject.org) | parallel traceroute ### # KEEP ORDER -(echo debian.org; echo freenetproject.org) \ -| parallel -k traceroute +(echo debian.org; echo freenetproject.org) | parallel -k traceroute ### # RUN MANY JOBS. USE OUTPUT # Find the number of hosts responding to ping +ping -c 1 178.63.11.1 +ping -c 1 178.63.11.1 | grep '64 bytes' seq 1 255 | parallel -j255 ping -c 1 178.63.11.{} 2>&1 \ | grep '64 bytes' | wc -l seq 1 255 | parallel -j0 ping -c 1 178.63.11.{} 2>&1 \ | grep '64 bytes' | wc -l # MULTIPLE ARGUMENTS -# make dir: (1-20000).dir +# make dir: test-(1-20000).dir cd ../dirs rm -rf *; sync -seq 1 20000 | time parallel mkdir {}.dir +seq 1 10 | parallel echo mkdir test-{}.dir +seq 1 5000 | time parallel mkdir test-{}.dir +## 15 sek + rm -rf *; sync -seq 1 20000 | time parallel -X mkdir {}.dir +seq 1 10 | parallel -X echo mkdir test-{}.dir +seq 1 5000 | time parallel -X mkdir test-{}.dir # CALLING GNU PARALLEL FROM GNU PARALLEL -# make dir: top-(1-100)/sub-(1-300) +# make dir: top-(1-100)/sub-(1-100) rm -rf *; sync -seq 1 100 | time parallel -I /// \ -'mkdir top-///;cd top-///; seq 1 300 | parallel -X mkdir sub-{}' +seq 1 100 | time parallel -I @@ \ +'mkdir top-@@; seq 1 100 | parallel -X mkdir top-@@/sub-{}' find | wc -l -# Thanks for watching +cd +# Thank you for watching +# +# If you like GNU Parallel: +# * Post this video on your blog/Twitter/Facebook/Linkedin +# * Join the mailing list http://lists.gnu.org/mailman/listinfo/parallel +# * Request or write a review for your favourite magazine +# * Request or build a package for your favourite distribution +# * Invite me for your next conference (Contact http://ole.tange.dk) +# +# If GNU Parallel saves you money: +# * Donate to FSF https://my.fsf.org/donate/ +# # Find GNU Parallel at http://www.gnu.org/software/parallel/ + # GIVE ME THE FIRST RESULT (echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -H2 traceroute {}";false" diff --git a/doc/release_new_version b/doc/release_new_version index d8b3c4eb..99665baf 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -37,7 +37,7 @@ make dist-bzip2 YYYYMMDD=`yyyymmdd` cp doc/parallel.directive parallel-$YYYYMMDD.tar.bz2.directive -perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-*.tar.*directive +perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-*.tar.*directive gpg --clearsign parallel-$YYYYMMDD.tar.bz2.directive YYYYMMDD=`yyyymmdd` diff --git a/src/parallel b/src/parallel index 5d51c9c0..309a014d 100755 --- a/src/parallel +++ b/src/parallel @@ -30,12 +30,15 @@ possible to use output from GNU B as input for other programs. For each line of input GNU B will execute I with the line as arguments. If no I is given, the line of input is executed. Several lines will be run in parallel. GNU B can -often be used as a substitute for B or B. +often be used as a substitute for B or B. Before looking at the options you may want to check out the examples after the list of options. That will give you an idea of what GNU B is capable of. +You can also watch the intro video for a quick introduction: +http://www.youtube.com/watch?v=LlXDtd_pRaY + =head1 OPTIONS =over 9 @@ -411,7 +414,8 @@ times: B<--return> is often used with B<--transfer> and B<--cleanup>. -B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. +B<--return> is ignored when used with B<--sshlogin :> or when not used +with B<--sshlogin>. =item B<--max-chars>=I @@ -1599,7 +1603,7 @@ if($::opt_halt_on_error) { sub parse_options { # Defaults: - $Global::version = 20100601; + $Global::version = 20100616; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -2520,6 +2524,7 @@ sub drain_job_queue { while($Global::total_running > 0) { debug("jobs running: $Global::total_running Memory usage:".my_memory_usage()."\n"); sleep 1; + Reaper(); # Some systems fail to catch the SIGCHLD if($::opt_progress) { my %progress = progress(); if($last_header ne $progress{'header'}) { @@ -2628,7 +2633,7 @@ sub columns { if(not $Global::columns) { $Global::columns = $ENV{'COLUMNS'}; if(not $Global::columns) { - my $resize = qx{ resize }; + my $resize = qx{ resize 2>/dev/null }; $resize =~ /COLUMNS=(\d+);/ and do { $Global::columns = $1; }; } $Global::columns ||= 80; diff --git a/unittest/Makefile b/unittest/Makefile index 40cdd1b4..c9030d33 100644 --- a/unittest/Makefile +++ b/unittest/Makefile @@ -2,6 +2,11 @@ unittest: ../src/parallel tests-to-run/* wanted-results/* echo | mop || (echo mop is required for unittest; /bin/false) seq 1 2 | mop || (echo seq is required for unittest; /bin/false) stdout echo || (echo stdout is required for unittest; /bin/false) + convert | mop || (echo convert is required for unittest; /bin/false) + ren 2>&1 | mop || (echo ren is required for unittest; /bin/false) + echo | buffer | mop || (echo buffer is required for unittest; /bin/false) + echo 1+2 | bc | mop || (echo bc is required for unittest; /bin/false) + stdout gawk | mop || (echo gawk is required for parallel; /bin/false) sh Start.sh clean: diff --git a/unittest/actual-results/test02 b/unittest/actual-results/test02 index 3a8e2719..213ceba1 100644 --- a/unittest/actual-results/test02 +++ b/unittest/actual-results/test02 @@ -1,13 +1,14 @@ +### Test filenames containing UTF-8 . +./1-col.txt +./2-col.txt ./a -./a/foo2 ./a/bar ./a/foo +./a/foo2 ./b ./b/bar ./b/foo -./2-col.txt ./中国 (Zhōngguó) ./中国 (Zhōngguó)/China's (中国) road.jpg ./中国 (Zhōngguó)/thumb_China's (中国) road.jpg -./1-col.txt diff --git a/unittest/actual-results/test16 b/unittest/actual-results/test16 index 85513278..c00983b5 100644 --- a/unittest/actual-results/test16 +++ b/unittest/actual-results/test16 @@ -60,8 +60,8 @@ rm -- 中国\ \(Zhōngguó\)/abc-中国\ \(Zhōngguó\)-中国\ \(Zhōngguó\) ### Test -m 1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC 1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC -a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif b1 2 3 4 5 6 c1 2 3 4 5 6 -a1.gifb1c1 a 2.gifb 2c 2 a 3.gifb 3c 3 a 4.gifb 4c 4 a 5.gifb 5c 5 a 6.gifb 6c 6 abc +a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6 +a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 ### Test -m with 60000 args 98c94dcab1efedab3f820935d230bc77 - 12 180011 1286837 diff --git a/unittest/actual-results/test18 b/unittest/actual-results/test18 index eae7e07f..156d5728 100644 --- a/unittest/actual-results/test18 +++ b/unittest/actual-results/test18 @@ -18,14 +18,12 @@ Warning: --transfer ignored as there are no remote --sshlogin ### Check warning if --return but not --sshlogin Warning: --return ignored as there are no remote --sshlogin -### Check warning if --cleanup but not --sshlogin -Warning: --cleanup ignored as there are no remote --sshlogin - -### Test --sshlogin -S --sshloginfile -1 -2 -3 -### Test --sshloginfile with extra content +### Check warning if --cleanup but not --sshlog10 +### Check forced number of CPUs being respected +alpha +### Check more than 9 simultaneous sshlogins +### Check more than 9(relative) simultaneous sshlogins +### Check -S syntax 1 2 3 @@ -36,17 +34,18 @@ Warning: --cleanup ignored as there are no remote --sshlogin 8 9 10 -### Check forced number of CPUs being respected +11 +PUs being respected +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk alpha -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk alpha alpha alpha diff --git a/unittest/actual-results/test19 b/unittest/actual-results/test19 index e7014af4..f74a0fc8 100644 --- a/unittest/actual-results/test19 +++ b/unittest/actual-results/test19 @@ -1,202 +1 @@ ### Test --transfer --return --cleanup - files with newline -### --transfer - file with newline -newline -newline -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -ls: cannot access tmp/parallel.file*: No such file or directory -### --transfer --cleanup - file with newline -newline -newline -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -ls: cannot access tmp/parallel.file*: No such file or directory -### --return - file with newline -tmp/parallel.file. -newline1.out -tmp/parallel.file. -newline2.out -### --return --cleanup - file with newline -tmp/parallel.file. -newline1.out -tmp/parallel.file. -newline2.out -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -OK -ls: cannot access tmp/parallel.file*: No such file or directory -OK -### --transfer --return --cleanup - file with newline -tmp/parallel.file. -newline1.out -tmp/parallel.file. -newline2.out -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -OK -ls: cannot access tmp/parallel.file*: No such file or directory -OK -### --trc - file with newline -tmp/parallel.file. -newline1.out -tmp/parallel.file. -newline2.out -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -OK -ls: cannot access tmp/parallel.file*: No such file or directory -OK -### --trc - multiple file with newline -tmp/parallel.file. -newline1.out -tmp/parallel.file. -newline1.out2 -tmp/parallel.file. -newline2.out -tmp/parallel.file. -newline2.out2 -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -OK -ls: cannot access tmp/parallel.file*: No such file or directory -OK -### Test use special ssh -tmp/parallel.file. -newline1.out -tmp/parallel.file. -newline1.out2 -tmp/parallel.file. -newline2.out -tmp/parallel.file. -newline2.out2 -good if no file -ls: cannot access tmp/parallel.file*: No such file or directory -OK -ls: cannot access tmp/parallel.file*: No such file or directory -OK -Input for ssh -parallel-server1 rsync --server -lDErRze.iLsf . ./ -parallel-server1 cat tmp/parallel.file.' -'newline2 > tmp/parallel.file.' -'newline2.out;cat tmp/parallel.file.' -'newline2 > tmp/parallel.file.' -'newline2.out2 -parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' -'newline2.out -parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' -'newline2.out2 -parallel-server1 rm -f tmp/parallel.file.' -'newline2 --l parallel parallel-server2 rsync --server -lDErRze.iLsf . ./ -parallel@parallel-server2 cat tmp/parallel.file.' -'newline1 > tmp/parallel.file.' -'newline1.out;cat tmp/parallel.file.' -'newline1 > tmp/parallel.file.' -'newline1.out2 --l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' -'newline1.out --l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' -'newline1.out2 -parallel@parallel-server2 rm -f tmp/parallel.file.' -'newline1 -### Test use special ssh with > 9 simultaneous -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -18 -19 -20 -21 -22 -23 -24 -25 -26 -27 -28 -29 -30 -31 -32 -33 -34 -35 -36 -37 -38 -39 -40 -41 -42 -43 -44 -45 -46 -47 -48 -49 -50 -51 -52 -53 -54 -55 -56 -57 -58 -59 -60 -61 -62 -63 -64 -65 -66 -67 -68 -69 -70 -71 -72 -73 -74 -75 -76 -77 -78 -79 -80 -81 -82 -83 -84 -85 -86 -87 -88 -89 -90 -91 -92 -93 -94 -95 -96 -97 -98 -99 -100 diff --git a/unittest/actual-results/test21 b/unittest/actual-results/test21 index 690ea1ac..fb071aab 100644 --- a/unittest/actual-results/test21 +++ b/unittest/actual-results/test21 @@ -2,10 +2,10 @@ 1 ssh -l parallel parallel-server2 echo\ 1; 1 -ssh parallel-server1 echo\ 2; +ssh parallel-server3 echo\ 2; 2 ### Test ~/.parallelrc ssh -l parallel parallel-server2 echo\ 1; 1 -ssh parallel-server1 echo\ 2; +ssh parallel-server3 echo\ 2; 2 diff --git a/unittest/tests-to-run/test02.sh b/unittest/tests-to-run/test02.sh index 36437a02..8845e4d0 100755 --- a/unittest/tests-to-run/test02.sh +++ b/unittest/tests-to-run/test02.sh @@ -6,10 +6,11 @@ rm -rf tmp 2>/dev/null cp -a input-files/testdir2 tmp cd tmp +echo '### Test filenames containing UTF-8' find . -name '*.jpg' | $PAR -j +0 convert -geometry 120 {} {}_thumb.jpg find . -name '*_thumb.jpg' | ren 's:/([^/]+)_thumb.jpg$:/thumb_$1:' -find |grep -v CVS +find |grep -v CVS | sort cd .. rm -rf tmp diff --git a/unittest/tests-to-run/test16.sh b/unittest/tests-to-run/test16.sh index 99a191c5..31405326 100755 --- a/unittest/tests-to-run/test16.sh +++ b/unittest/tests-to-run/test16.sh @@ -17,7 +17,7 @@ echo '### Test {.} with files that have no . but dir does' mkdir -p /tmp/test-of-{.}-parallel/subdir touch /tmp/test-of-{.}-parallel/subdir/file touch /tmp/test-of-{.}-parallel/subdir/file{.}.funkyextension}} -find /tmp/test-of-{.}-parallel -type f | $PAR echo {.} +find /tmp/test-of-{.}-parallel -type f | $PAR echo {.} | sort rm -rf /tmp/test-of-{.}-parallel/subdir @@ -34,8 +34,8 @@ ls | $PAR -kv rm -- {.}/abc-{.}-{} 2>&1 echo '### Test -m' (echo foo;echo bar;echo joe.gif) | $PAR -km echo 1{}2{.}3 A{.}B{.}C (echo foo;echo bar;echo joe.gif) | $PAR -kX echo 1{}2{.}3 A{.}B{.}C -seq 1 6 | $PAR -kX echo -e '{}.gif\\n' | $PAR -km echo a{}b{.}c{.} -seq 1 6 | $PAR -kX echo -e '{}.gif\\n' | $PAR -kX echo a{}b{.}c{.} +seq 1 6 | $PAR -k printf '{}.gif\\n' | $PAR -km echo a{}b{.}c{.} +seq 1 6 | $PAR -k printf '{}.gif\\n' | $PAR -kX echo a{}b{.}c{.} echo '### Test -m with 60000 args' seq 1 60000 | perl -pe 's/$/.gif\n/' | $PAR -km echo a{}b{.}c{.} | mop -d 4 "|md5sum" "| wc" echo '### Test -X with 60000 args' diff --git a/unittest/tests-to-run/test17.sh b/unittest/tests-to-run/test17.sh index f763a714..95e16884 100755 --- a/unittest/tests-to-run/test17.sh +++ b/unittest/tests-to-run/test17.sh @@ -2,7 +2,7 @@ PAR=parallel -SERVER1=parallel-server1 +SERVER1=parallel-server3 SERVER2=parallel-server2 # Make sure sort order is the same @@ -13,12 +13,12 @@ echo '### Test --transfer --return --cleanup' rm -rf /tmp/parallel.file* stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' -(seq 1 3;echo '>fire';seq 5 10; echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17 +(seq 1 3;echo '>fire';seq 5 10; /bin/echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17 echo '# Create some weirdly files in /tmp' mkdir -p /tmp/parallel.file -cat /tmp/test17 | $PAR -k echo file{} '>'/tmp/parallel.file{}.file -cat /tmp/test17 | $PAR -k echo /tmp/parallel.file{}.file >/tmp/test17abs -cat /tmp/test17 | $PAR -k echo tmp/parallel.file{}.file >/tmp/test17rel +cat /tmp/test17 | $PAR -k /bin/echo file{} '>'/tmp/parallel.file{}.file +cat /tmp/test17 | $PAR -k /bin/echo /tmp/parallel.file{}.file >/tmp/test17abs +cat /tmp/test17 | $PAR -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel echo '### --transfer - abspath' stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' diff --git a/unittest/tests-to-run/test18.sh b/unittest/tests-to-run/test18.sh index f233857a..8c59d901 100644 --- a/unittest/tests-to-run/test18.sh +++ b/unittest/tests-to-run/test18.sh @@ -2,7 +2,7 @@ PAR=parallel -SERVER1=parallel-server1 +SERVER1=parallel-server3 SERVER2=parallel-server2 echo '### Check warning if --transfer but file not found' diff --git a/unittest/tests-to-run/test19.sh b/unittest/tests-to-run/test19.sh index a845e859..8d15696f 100644 --- a/unittest/tests-to-run/test19.sh +++ b/unittest/tests-to-run/test19.sh @@ -4,7 +4,7 @@ PAR=parallel -SERVER1=parallel-server1 +SERVER1=parallel-server3 SERVER2=parallel-server2 echo '### Test --transfer --return --cleanup - files with newline' diff --git a/unittest/tests-to-run/test21.sh b/unittest/tests-to-run/test21.sh index 42a13dca..5bed50a5 100644 --- a/unittest/tests-to-run/test21.sh +++ b/unittest/tests-to-run/test21.sh @@ -1,7 +1,7 @@ #!/bin/bash PAR=parallel -SERVER1=parallel-server1 +SERVER1=parallel-server3 SERVER2=parallel-server2 echo '### Test $PARALLEL' diff --git a/unittest/tests-to-run/test23.sh b/unittest/tests-to-run/test23.sh index ac5be479..ecd4b94a 100644 --- a/unittest/tests-to-run/test23.sh +++ b/unittest/tests-to-run/test23.sh @@ -1,7 +1,7 @@ #!/bin/bash PAR=parallel -SERVER1=parallel-server1 +SERVER1=parallel-server3 SERVER2=parallel-server2 cd /tmp @@ -25,6 +25,6 @@ rm -f parallel_*.test parallel_*.out seq 1 13 | parallel echo {} '>' parallel_{}.test ls parallel_*.test | parallel -j+0 --trc {.}.out -B my_script \ --S parallel-server1,parallel@parallel-server2,: "./my_script {} > {.}.out" +-S $SERVER1,parallel@$SERVER2,: "./my_script {} > {.}.out" cat parallel_*.test parallel_*.out diff --git a/unittest/tests-to-run/test24.sh b/unittest/tests-to-run/test24.sh new file mode 100644 index 00000000..5d3717da --- /dev/null +++ b/unittest/tests-to-run/test24.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo '### 64-bit wierdness - this did not complete on a 64-bit machine' +seq 1 2 | parallel -j1 'seq 1 1 | parallel true' diff --git a/unittest/wanted-results/test02 b/unittest/wanted-results/test02 index 3a8e2719..213ceba1 100644 --- a/unittest/wanted-results/test02 +++ b/unittest/wanted-results/test02 @@ -1,13 +1,14 @@ +### Test filenames containing UTF-8 . +./1-col.txt +./2-col.txt ./a -./a/foo2 ./a/bar ./a/foo +./a/foo2 ./b ./b/bar ./b/foo -./2-col.txt ./中国 (Zhōngguó) ./中国 (Zhōngguó)/China's (中国) road.jpg ./中国 (Zhōngguó)/thumb_China's (中国) road.jpg -./1-col.txt diff --git a/unittest/wanted-results/test16 b/unittest/wanted-results/test16 index 85513278..c00983b5 100644 --- a/unittest/wanted-results/test16 +++ b/unittest/wanted-results/test16 @@ -60,8 +60,8 @@ rm -- 中国\ \(Zhōngguó\)/abc-中国\ \(Zhōngguó\)-中国\ \(Zhōngguó\) ### Test -m 1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC 1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC -a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif b1 2 3 4 5 6 c1 2 3 4 5 6 -a1.gifb1c1 a 2.gifb 2c 2 a 3.gifb 3c 3 a 4.gifb 4c 4 a 5.gifb 5c 5 a 6.gifb 6c 6 abc +a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6 +a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 ### Test -m with 60000 args 98c94dcab1efedab3f820935d230bc77 - 12 180011 1286837 diff --git a/unittest/wanted-results/test18 b/unittest/wanted-results/test18 index eae7e07f..381e7f39 100644 --- a/unittest/wanted-results/test18 +++ b/unittest/wanted-results/test18 @@ -37,16 +37,16 @@ Warning: --cleanup ignored as there are no remote --sshlogin 9 10 ### Check forced number of CPUs being respected +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk +nlv.pi.dk alpha -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk -nlv.pi.dk alpha alpha alpha diff --git a/unittest/wanted-results/test19 b/unittest/wanted-results/test19 index e7014af4..64df1ae7 100644 --- a/unittest/wanted-results/test19 +++ b/unittest/wanted-results/test19 @@ -75,17 +75,17 @@ OK ls: cannot access tmp/parallel.file*: No such file or directory OK Input for ssh -parallel-server1 rsync --server -lDErRze.iLsf . ./ -parallel-server1 cat tmp/parallel.file.' +parallel-server3 rsync --server -lDErRze.iLsf . ./ +parallel-server3 cat tmp/parallel.file.' 'newline2 > tmp/parallel.file.' 'newline2.out;cat tmp/parallel.file.' 'newline2 > tmp/parallel.file.' 'newline2.out2 -parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' +parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' 'newline2.out -parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' +parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' 'newline2.out2 -parallel-server1 rm -f tmp/parallel.file.' +parallel-server3 rm -f tmp/parallel.file.' 'newline2 -l parallel parallel-server2 rsync --server -lDErRze.iLsf . ./ parallel@parallel-server2 cat tmp/parallel.file.' diff --git a/unittest/wanted-results/test21 b/unittest/wanted-results/test21 index 690ea1ac..fb071aab 100644 --- a/unittest/wanted-results/test21 +++ b/unittest/wanted-results/test21 @@ -2,10 +2,10 @@ 1 ssh -l parallel parallel-server2 echo\ 1; 1 -ssh parallel-server1 echo\ 2; +ssh parallel-server3 echo\ 2; 2 ### Test ~/.parallelrc ssh -l parallel parallel-server2 echo\ 1; 1 -ssh parallel-server1 echo\ 2; +ssh parallel-server3 echo\ 2; 2