Some systems does not receive the SIGCHLD (maybe when running under

/bin/dash?).
Unittest failed if /bin/sh was dash.
This commit is contained in:
Ole Tange 2010-06-16 05:03:52 +02:00
parent 167332902b
commit c81a15677d
27 changed files with 190 additions and 326 deletions

View file

@ -1,6 +1,6 @@
@%:@! /bin/sh @%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles. @%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100601. @%:@ Generated by GNU Autoconf 2.65 for parallel 20100616.
@%:@ @%:@
@%:@ Report bugs to <bug-parallel@gnu.org>. @%:@ Report bugs to <bug-parallel@gnu.org>.
@%:@ @%:@
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package. # Identity of this package.
PACKAGE_NAME='parallel' PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel' PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100601' PACKAGE_VERSION='20100616'
PACKAGE_STRING='parallel 20100601' PACKAGE_STRING='parallel 20100616'
PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL='' PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing. # Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh. # This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF cat <<_ACEOF
\`configure' configures parallel 20100601 to adapt to many kinds of systems. \`configure' configures parallel 20100616 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]... Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then if test -n "$ac_init_help"; then
case $ac_init_help in case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100601:";; short | recursive ) echo "Configuration of parallel 20100616:";;
esac esac
cat <<\_ACEOF cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then if $ac_init_version; then
cat <<\_ACEOF cat <<\_ACEOF
parallel configure 20100601 parallel configure 20100616
generated by GNU Autoconf 2.65 generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc. Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake. running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100601, which was It was created by parallel $as_me 20100616, which was
generated by GNU Autoconf 2.65. Invocation command line was generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@ $ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package. # Define the identity of the package.
PACKAGE='parallel' PACKAGE='parallel'
VERSION='20100601' VERSION='20100616'
cat >>confdefs.h <<_ACEOF cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their # report actual input values of CONFIG_FILES etc. instead of their
# values after options handling. # values after options handling.
ac_log=" ac_log="
This file was extended by parallel $as_me 20100601, which was This file was extended by parallel $as_me 20100616, which was
generated by GNU Autoconf 2.65. Invocation command line was generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\ ac_cs_version="\\
parallel config.status 20100601 parallel config.status 20100616
configured by $0, generated by GNU Autoconf 2.65, configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\" with options \\"\$ac_cs_config\\"

View file

@ -1,6 +1,6 @@
@%:@! /bin/sh @%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles. @%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100601. @%:@ Generated by GNU Autoconf 2.65 for parallel 20100616.
@%:@ @%:@
@%:@ Report bugs to <bug-parallel@gnu.org>. @%:@ Report bugs to <bug-parallel@gnu.org>.
@%:@ @%:@
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package. # Identity of this package.
PACKAGE_NAME='parallel' PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel' PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100601' PACKAGE_VERSION='20100616'
PACKAGE_STRING='parallel 20100601' PACKAGE_STRING='parallel 20100616'
PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL='' PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing. # Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh. # This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF cat <<_ACEOF
\`configure' configures parallel 20100601 to adapt to many kinds of systems. \`configure' configures parallel 20100616 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]... Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then if test -n "$ac_init_help"; then
case $ac_init_help in case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100601:";; short | recursive ) echo "Configuration of parallel 20100616:";;
esac esac
cat <<\_ACEOF cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then if $ac_init_version; then
cat <<\_ACEOF cat <<\_ACEOF
parallel configure 20100601 parallel configure 20100616
generated by GNU Autoconf 2.65 generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc. Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake. running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100601, which was It was created by parallel $as_me 20100616, which was
generated by GNU Autoconf 2.65. Invocation command line was generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@ $ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package. # Define the identity of the package.
PACKAGE='parallel' PACKAGE='parallel'
VERSION='20100601' VERSION='20100616'
cat >>confdefs.h <<_ACEOF cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their # report actual input values of CONFIG_FILES etc. instead of their
# values after options handling. # values after options handling.
ac_log=" ac_log="
This file was extended by parallel $as_me 20100601, which was This file was extended by parallel $as_me 20100616, which was
generated by GNU Autoconf 2.65. Invocation command line was generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\ ac_cs_version="\\
parallel config.status 20100601 parallel config.status 20100616
configured by $0, generated by GNU Autoconf 2.65, configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\" with options \\"\$ac_cs_config\\"

View file

@ -1,4 +1,4 @@
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100601], [bug-parallel@gnu.org]) m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100616], [bug-parallel@gnu.org])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS']) m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS'])

20
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh #! /bin/sh
# Guess values for system-dependent variables and create Makefiles. # Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.65 for parallel 20100601. # Generated by GNU Autoconf 2.65 for parallel 20100616.
# #
# Report bugs to <bug-parallel@gnu.org>. # Report bugs to <bug-parallel@gnu.org>.
# #
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package. # Identity of this package.
PACKAGE_NAME='parallel' PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel' PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100601' PACKAGE_VERSION='20100616'
PACKAGE_STRING='parallel 20100601' PACKAGE_STRING='parallel 20100616'
PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL='' PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing. # Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh. # This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF cat <<_ACEOF
\`configure' configures parallel 20100601 to adapt to many kinds of systems. \`configure' configures parallel 20100616 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]... Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then if test -n "$ac_init_help"; then
case $ac_init_help in case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100601:";; short | recursive ) echo "Configuration of parallel 20100616:";;
esac esac
cat <<\_ACEOF cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then if $ac_init_version; then
cat <<\_ACEOF cat <<\_ACEOF
parallel configure 20100601 parallel configure 20100616
generated by GNU Autoconf 2.65 generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc. Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake. running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100601, which was It was created by parallel $as_me 20100616, which was
generated by GNU Autoconf 2.65. Invocation command line was generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@ $ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package. # Define the identity of the package.
PACKAGE='parallel' PACKAGE='parallel'
VERSION='20100601' VERSION='20100616'
cat >>confdefs.h <<_ACEOF cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their # report actual input values of CONFIG_FILES etc. instead of their
# values after options handling. # values after options handling.
ac_log=" ac_log="
This file was extended by parallel $as_me 20100601, which was This file was extended by parallel $as_me 20100616, which was
generated by GNU Autoconf 2.65. Invocation command line was generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\ ac_cs_version="\\
parallel config.status 20100601 parallel config.status 20100616
configured by $0, generated by GNU Autoconf 2.65, configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\" with options \\"\$ac_cs_config\\"

View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20100601], [bug-parallel@gnu.org]) AC_INIT([parallel], [20100616], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h]) AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([ AC_CONFIG_FILES([

View file

@ -1,6 +1,6 @@
--progress Some systems does not receive the SIGCHLD (maybe when running under
Fixed bug if transfered file contains : dash?).
Unittest failed if /bin/sh was dash.
* 100% options complete with xargs. All options for xargs can now * 100% options complete with xargs. All options for xargs can now
be used in GNU Parallel - even the more exotic. be used in GNU Parallel - even the more exotic.
@ -21,6 +21,8 @@ Fixed bug if transfered file contains :
* New video showing the new options. * New video showing the new options.
=head1 YouTube video =head1 YouTube video
GNU Parallel is a tool with lots of uses in shell. Every time you use GNU Parallel is a tool with lots of uses in shell. Every time you use
@ -32,17 +34,28 @@ it easy to distribute jobs to these.
terminal2: ssh parallel@vh2.pi.dk terminal2: ssh parallel@vh2.pi.dk
ssh parallel@vh2.pi.dk ssh parallel@vh2.pi.dk
and
PS1="\[\e[7m\]GNU Parallel:\[\033[01;34m\]\w\[\033[00m\e[27m\]$ "
PS1="\e[7mGNU Parallel:\[\033[01;34m\]\w\[\033[00m\]\e[27m$ "
gunzip logs/*gz gunzip logs/*gz
rm logs/*bz2 rm -f logs/*bz2*
rm -rf zip/*[^p] rm -rf zip/*[^p]
rm -rf dirs/*
rm -rf parallel-*bz2
xvidcap
ffmpeg -i 20100615_002.mp4 -ab 320k -ar 44100 speak.mp3
# Merge video using youtube
#ffmpeg -i speak.mp3 -i xvidcap.mpeg -target mpeg -hq -minrate 8000000 \
#-title "GNU Parallel" -author "Ole Tange" -copyright "(CC-By-SA) 2010" -comment "Intro video of GNU Parallel 20100615" videoaudio.mpg
# GNU PARALLEL - BASIC USAGE
# A GNU tool for parallelizing shell commands
# GET GNU PARALLEL # GET GNU PARALLEL
wget ftp://ftp.gnu.org/gnu/parallel/parallel-20100601.tar.bz2 wget ftp://ftp.gnu.org/gnu/parallel/parallel-20100615.tar.bz2
tar xvjf parallel-20100601.tar.bz2 tar xjf parallel-20100615.tar.bz2
cd parallel-20100601 cd parallel-20100615
./configure && make ## ./configure && make ##
su su
make install make install
@ -52,20 +65,39 @@ cd
# YOUR FIRST PARALLEL JOBS # YOUR FIRST PARALLEL JOBS
cd logs cd logs
du du
/usr/bin/time gzip * /usr/bin/time gzip -1 *
## 24 sek - 22 sek
/usr/bin/time gunzip * /usr/bin/time gunzip *
ls | time parallel gzip ## 24 sek - 18
ls | time parallel gzip -1
## 17 sek - 10
ls | time parallel gunzip ls | time parallel gunzip
## 25 sek - 19
# RECOMPRESS gz TO bz2 # RECOMPRESS gz TO bz2
ls | time parallel gzip ls | time parallel gzip -1
ls *.gz | time parallel 'zcat {} | bzip2 > {.}.bz2' ls *.gz | time parallel -j+0 --progress 'zcat {} | bzip2 -1 > {.}.bz2'
## Explain command line
## vis top local ## vis top local
# RECOMPRESS gz TO bz2 USING local(:) AND REMOTE server1 ## Man that is boring
ls *.gz | time parallel -S server1,: --trc {.}.bz2 \ ## 2m41s - 2m
'zcat {} | bzip2 > {.}.bz2'
# RECOMPRESS gz TO bz2 USING local(:) AND REMOTE server1-3
ls *.gz |time parallel -j+0 --progress -Sserver1,server2,server3,: \
--transfer --return {.}.bz2 --cleanup 'zcat {} | bzip2 -1 > {.}.bz2'
## Explain command line
## Explain server config
## vis top local ## vis top local
## vis top remote ## vis top remote1-3
## 49 sek
# RECOMPRESS gz TO bz2 USING A SCRIPT ON local AND REMOTE server1-3
# (imagine the script is way more complex)
cp ../recompress /tmp
cat /tmp/recompress
ls *.gz |time parallel -j+0 --progress -Sserver1,server2,server3,: \
--trc {.}.bz2 --basefile /tmp/recompress '/tmp/recompress {} {.}.bz2'
# MAKING SMALL SCRIPTS # MAKING SMALL SCRIPTS
cd ../zip cd ../zip
@ -76,38 +108,55 @@ ls -l
# GROUP OUTPUT # GROUP OUTPUT
traceroute debian.org traceroute debian.org
traceroute debian.org & traceroute freenetproject.org ### traceroute debian.org & traceroute freenetproject.org ###
(echo debian.org; echo freenetproject.org) \ (echo debian.org; echo freenetproject.org) | parallel traceroute ###
| parallel traceroute
# KEEP ORDER # KEEP ORDER
(echo debian.org; echo freenetproject.org) \ (echo debian.org; echo freenetproject.org) | parallel -k traceroute ###
| parallel -k traceroute
# RUN MANY JOBS. USE OUTPUT # RUN MANY JOBS. USE OUTPUT
# Find the number of hosts responding to ping # Find the number of hosts responding to ping
ping -c 1 178.63.11.1
ping -c 1 178.63.11.1 | grep '64 bytes'
seq 1 255 | parallel -j255 ping -c 1 178.63.11.{} 2>&1 \ seq 1 255 | parallel -j255 ping -c 1 178.63.11.{} 2>&1 \
| grep '64 bytes' | wc -l | grep '64 bytes' | wc -l
seq 1 255 | parallel -j0 ping -c 1 178.63.11.{} 2>&1 \ seq 1 255 | parallel -j0 ping -c 1 178.63.11.{} 2>&1 \
| grep '64 bytes' | wc -l | grep '64 bytes' | wc -l
# MULTIPLE ARGUMENTS # MULTIPLE ARGUMENTS
# make dir: (1-20000).dir # make dir: test-(1-20000).dir
cd ../dirs cd ../dirs
rm -rf *; sync rm -rf *; sync
seq 1 20000 | time parallel mkdir {}.dir seq 1 10 | parallel echo mkdir test-{}.dir
seq 1 5000 | time parallel mkdir test-{}.dir
## 15 sek
rm -rf *; sync rm -rf *; sync
seq 1 20000 | time parallel -X mkdir {}.dir seq 1 10 | parallel -X echo mkdir test-{}.dir
seq 1 5000 | time parallel -X mkdir test-{}.dir
# CALLING GNU PARALLEL FROM GNU PARALLEL # CALLING GNU PARALLEL FROM GNU PARALLEL
# make dir: top-(1-100)/sub-(1-300) # make dir: top-(1-100)/sub-(1-100)
rm -rf *; sync rm -rf *; sync
seq 1 100 | time parallel -I /// \ seq 1 100 | time parallel -I @@ \
'mkdir top-///;cd top-///; seq 1 300 | parallel -X mkdir sub-{}' 'mkdir top-@@; seq 1 100 | parallel -X mkdir top-@@/sub-{}'
find | wc -l find | wc -l
# Thanks for watching cd
# Thank you for watching
#
# If you like GNU Parallel:
# * Post this video on your blog/Twitter/Facebook/Linkedin
# * Join the mailing list http://lists.gnu.org/mailman/listinfo/parallel
# * Request or write a review for your favourite magazine
# * Request or build a package for your favourite distribution
# * Invite me for your next conference (Contact http://ole.tange.dk)
#
# If GNU Parallel saves you money:
# * Donate to FSF https://my.fsf.org/donate/
#
# Find GNU Parallel at http://www.gnu.org/software/parallel/ # Find GNU Parallel at http://www.gnu.org/software/parallel/
# GIVE ME THE FIRST RESULT # GIVE ME THE FIRST RESULT
(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -H2 traceroute {}";false" (echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -H2 traceroute {}";false"

View file

@ -37,7 +37,7 @@ make dist-bzip2
YYYYMMDD=`yyyymmdd` YYYYMMDD=`yyyymmdd`
cp doc/parallel.directive parallel-$YYYYMMDD.tar.bz2.directive cp doc/parallel.directive parallel-$YYYYMMDD.tar.bz2.directive
perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-*.tar.*directive perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-*.tar.*directive
gpg --clearsign parallel-$YYYYMMDD.tar.bz2.directive gpg --clearsign parallel-$YYYYMMDD.tar.bz2.directive
YYYYMMDD=`yyyymmdd` YYYYMMDD=`yyyymmdd`

View file

@ -30,12 +30,15 @@ possible to use output from GNU B<parallel> as input for other programs.
For each line of input GNU B<parallel> will execute I<command> with For each line of input GNU B<parallel> will execute I<command> with
the line as arguments. If no I<command> is given, the line of input is the line as arguments. If no I<command> is given, the line of input is
executed. Several lines will be run in parallel. GNU B<parallel> can executed. Several lines will be run in parallel. GNU B<parallel> can
often be used as a substitute for B<xargs> or B<cat | sh>. often be used as a substitute for B<xargs> or B<cat | bash>.
Before looking at the options you may want to check out the examples Before looking at the options you may want to check out the examples
after the list of options. That will give you an idea of what GNU after the list of options. That will give you an idea of what GNU
B<parallel> is capable of. B<parallel> is capable of.
You can also watch the intro video for a quick introduction:
http://www.youtube.com/watch?v=LlXDtd_pRaY
=head1 OPTIONS =head1 OPTIONS
=over 9 =over 9
@ -411,7 +414,8 @@ times:
B<--return> is often used with B<--transfer> and B<--cleanup>. B<--return> is often used with B<--transfer> and B<--cleanup>.
B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. B<--return> is ignored when used with B<--sshlogin :> or when not used
with B<--sshlogin>.
=item B<--max-chars>=I<max-chars> =item B<--max-chars>=I<max-chars>
@ -1599,7 +1603,7 @@ if($::opt_halt_on_error) {
sub parse_options { sub parse_options {
# Defaults: # Defaults:
$Global::version = 20100601; $Global::version = 20100616;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::debug = 0; $Global::debug = 0;
$Global::verbose = 0; $Global::verbose = 0;
@ -2520,6 +2524,7 @@ sub drain_job_queue {
while($Global::total_running > 0) { while($Global::total_running > 0) {
debug("jobs running: $Global::total_running Memory usage:".my_memory_usage()."\n"); debug("jobs running: $Global::total_running Memory usage:".my_memory_usage()."\n");
sleep 1; sleep 1;
Reaper(); # Some systems fail to catch the SIGCHLD
if($::opt_progress) { if($::opt_progress) {
my %progress = progress(); my %progress = progress();
if($last_header ne $progress{'header'}) { if($last_header ne $progress{'header'}) {
@ -2628,7 +2633,7 @@ sub columns {
if(not $Global::columns) { if(not $Global::columns) {
$Global::columns = $ENV{'COLUMNS'}; $Global::columns = $ENV{'COLUMNS'};
if(not $Global::columns) { if(not $Global::columns) {
my $resize = qx{ resize }; my $resize = qx{ resize 2>/dev/null };
$resize =~ /COLUMNS=(\d+);/ and do { $Global::columns = $1; }; $resize =~ /COLUMNS=(\d+);/ and do { $Global::columns = $1; };
} }
$Global::columns ||= 80; $Global::columns ||= 80;

View file

@ -2,6 +2,11 @@ unittest: ../src/parallel tests-to-run/* wanted-results/*
echo | mop || (echo mop is required for unittest; /bin/false) echo | mop || (echo mop is required for unittest; /bin/false)
seq 1 2 | mop || (echo seq is required for unittest; /bin/false) seq 1 2 | mop || (echo seq is required for unittest; /bin/false)
stdout echo || (echo stdout is required for unittest; /bin/false) stdout echo || (echo stdout is required for unittest; /bin/false)
convert | mop || (echo convert is required for unittest; /bin/false)
ren 2>&1 | mop || (echo ren is required for unittest; /bin/false)
echo | buffer | mop || (echo buffer is required for unittest; /bin/false)
echo 1+2 | bc | mop || (echo bc is required for unittest; /bin/false)
stdout gawk | mop || (echo gawk is required for parallel; /bin/false)
sh Start.sh sh Start.sh
clean: clean:

View file

@ -1,13 +1,14 @@
### Test filenames containing UTF-8
. .
./1-col.txt
./2-col.txt
./a ./a
./a/foo2
./a/bar ./a/bar
./a/foo ./a/foo
./a/foo2
./b ./b
./b/bar ./b/bar
./b/foo ./b/foo
./2-col.txt
./中国 (Zhōngguó) ./中国 (Zhōngguó)
./中国 (Zhōngguó)/China's (中国) road.jpg ./中国 (Zhōngguó)/China's (中国) road.jpg
./中国 (Zhōngguó)/thumb_China's (中国) road.jpg ./中国 (Zhōngguó)/thumb_China's (中国) road.jpg
./1-col.txt

View file

@ -60,8 +60,8 @@ rm -- 中国\ \(Zhōngguó\)/abc-中国\ \(Zhōngguó\)-中国\ \(Zhōngguó\)
### Test -m ### Test -m
1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC 1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC
1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC 1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC
a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif b1 2 3 4 5 6 c1 2 3 4 5 6 a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6
a1.gifb1c1 a 2.gifb 2c 2 a 3.gifb 3c 3 a 4.gifb 4c 4 a 5.gifb 5c 5 a 6.gifb 6c 6 abc a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6
### Test -m with 60000 args ### Test -m with 60000 args
98c94dcab1efedab3f820935d230bc77 - 98c94dcab1efedab3f820935d230bc77 -
12 180011 1286837 12 180011 1286837

View file

@ -18,14 +18,12 @@ Warning: --transfer ignored as there are no remote --sshlogin
### Check warning if --return but not --sshlogin ### Check warning if --return but not --sshlogin
Warning: --return ignored as there are no remote --sshlogin Warning: --return ignored as there are no remote --sshlogin
### Check warning if --cleanup but not --sshlogin ### Check warning if --cleanup but not --sshlog10
Warning: --cleanup ignored as there are no remote --sshlogin ### Check forced number of CPUs being respected
alpha
### Test --sshlogin -S --sshloginfile ### Check more than 9 simultaneous sshlogins
1 ### Check more than 9(relative) simultaneous sshlogins
2 ### Check -S syntax
3
### Test --sshloginfile with extra content
1 1
2 2
3 3
@ -36,17 +34,18 @@ Warning: --cleanup ignored as there are no remote --sshlogin
8 8
9 9
10 10
### Check forced number of CPUs being respected 11
PUs being respected
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
alpha alpha
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
alpha alpha
alpha alpha
alpha alpha

View file

@ -1,202 +1 @@
### Test --transfer --return --cleanup - files with newline ### Test --transfer --return --cleanup - files with newline
### --transfer - file with newline
newline
newline
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
ls: cannot access tmp/parallel.file*: No such file or directory
### --transfer --cleanup - file with newline
newline
newline
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
ls: cannot access tmp/parallel.file*: No such file or directory
### --return - file with newline
tmp/parallel.file.
newline1.out
tmp/parallel.file.
newline2.out
### --return --cleanup - file with newline
tmp/parallel.file.
newline1.out
tmp/parallel.file.
newline2.out
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
OK
ls: cannot access tmp/parallel.file*: No such file or directory
OK
### --transfer --return --cleanup - file with newline
tmp/parallel.file.
newline1.out
tmp/parallel.file.
newline2.out
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
OK
ls: cannot access tmp/parallel.file*: No such file or directory
OK
### --trc - file with newline
tmp/parallel.file.
newline1.out
tmp/parallel.file.
newline2.out
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
OK
ls: cannot access tmp/parallel.file*: No such file or directory
OK
### --trc - multiple file with newline
tmp/parallel.file.
newline1.out
tmp/parallel.file.
newline1.out2
tmp/parallel.file.
newline2.out
tmp/parallel.file.
newline2.out2
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
OK
ls: cannot access tmp/parallel.file*: No such file or directory
OK
### Test use special ssh
tmp/parallel.file.
newline1.out
tmp/parallel.file.
newline1.out2
tmp/parallel.file.
newline2.out
tmp/parallel.file.
newline2.out2
good if no file
ls: cannot access tmp/parallel.file*: No such file or directory
OK
ls: cannot access tmp/parallel.file*: No such file or directory
OK
Input for ssh
parallel-server1 rsync --server -lDErRze.iLsf . ./
parallel-server1 cat tmp/parallel.file.'
'newline2 > tmp/parallel.file.'
'newline2.out;cat tmp/parallel.file.'
'newline2 > tmp/parallel.file.'
'newline2.out2
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
'newline2.out
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
'newline2.out2
parallel-server1 rm -f tmp/parallel.file.'
'newline2
-l parallel parallel-server2 rsync --server -lDErRze.iLsf . ./
parallel@parallel-server2 cat tmp/parallel.file.'
'newline1 > tmp/parallel.file.'
'newline1.out;cat tmp/parallel.file.'
'newline1 > tmp/parallel.file.'
'newline1.out2
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
'newline1.out
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
'newline1.out2
parallel@parallel-server2 rm -f tmp/parallel.file.'
'newline1
### Test use special ssh with > 9 simultaneous
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

View file

@ -2,10 +2,10 @@
1 1
ssh -l parallel parallel-server2 echo\ 1; ssh -l parallel parallel-server2 echo\ 1;
1 1
ssh parallel-server1 echo\ 2; ssh parallel-server3 echo\ 2;
2 2
### Test ~/.parallelrc ### Test ~/.parallelrc
ssh -l parallel parallel-server2 echo\ 1; ssh -l parallel parallel-server2 echo\ 1;
1 1
ssh parallel-server1 echo\ 2; ssh parallel-server3 echo\ 2;
2 2

View file

@ -6,10 +6,11 @@ rm -rf tmp 2>/dev/null
cp -a input-files/testdir2 tmp cp -a input-files/testdir2 tmp
cd tmp cd tmp
echo '### Test filenames containing UTF-8'
find . -name '*.jpg' | $PAR -j +0 convert -geometry 120 {} {}_thumb.jpg find . -name '*.jpg' | $PAR -j +0 convert -geometry 120 {} {}_thumb.jpg
find . -name '*_thumb.jpg' | ren 's:/([^/]+)_thumb.jpg$:/thumb_$1:' find . -name '*_thumb.jpg' | ren 's:/([^/]+)_thumb.jpg$:/thumb_$1:'
find |grep -v CVS find |grep -v CVS | sort
cd .. cd ..
rm -rf tmp rm -rf tmp

View file

@ -17,7 +17,7 @@ echo '### Test {.} with files that have no . but dir does'
mkdir -p /tmp/test-of-{.}-parallel/subdir mkdir -p /tmp/test-of-{.}-parallel/subdir
touch /tmp/test-of-{.}-parallel/subdir/file touch /tmp/test-of-{.}-parallel/subdir/file
touch /tmp/test-of-{.}-parallel/subdir/file{.}.funkyextension}} touch /tmp/test-of-{.}-parallel/subdir/file{.}.funkyextension}}
find /tmp/test-of-{.}-parallel -type f | $PAR echo {.} find /tmp/test-of-{.}-parallel -type f | $PAR echo {.} | sort
rm -rf /tmp/test-of-{.}-parallel/subdir rm -rf /tmp/test-of-{.}-parallel/subdir
@ -34,8 +34,8 @@ ls | $PAR -kv rm -- {.}/abc-{.}-{} 2>&1
echo '### Test -m' echo '### Test -m'
(echo foo;echo bar;echo joe.gif) | $PAR -km echo 1{}2{.}3 A{.}B{.}C (echo foo;echo bar;echo joe.gif) | $PAR -km echo 1{}2{.}3 A{.}B{.}C
(echo foo;echo bar;echo joe.gif) | $PAR -kX echo 1{}2{.}3 A{.}B{.}C (echo foo;echo bar;echo joe.gif) | $PAR -kX echo 1{}2{.}3 A{.}B{.}C
seq 1 6 | $PAR -kX echo -e '{}.gif\\n' | $PAR -km echo a{}b{.}c{.} seq 1 6 | $PAR -k printf '{}.gif\\n' | $PAR -km echo a{}b{.}c{.}
seq 1 6 | $PAR -kX echo -e '{}.gif\\n' | $PAR -kX echo a{}b{.}c{.} seq 1 6 | $PAR -k printf '{}.gif\\n' | $PAR -kX echo a{}b{.}c{.}
echo '### Test -m with 60000 args' echo '### Test -m with 60000 args'
seq 1 60000 | perl -pe 's/$/.gif\n/' | $PAR -km echo a{}b{.}c{.} | mop -d 4 "|md5sum" "| wc" seq 1 60000 | perl -pe 's/$/.gif\n/' | $PAR -km echo a{}b{.}c{.} | mop -d 4 "|md5sum" "| wc"
echo '### Test -X with 60000 args' echo '### Test -X with 60000 args'

View file

@ -2,7 +2,7 @@
PAR=parallel PAR=parallel
SERVER1=parallel-server1 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
# Make sure sort order is the same # Make sure sort order is the same
@ -13,12 +13,12 @@ echo '### Test --transfer --return --cleanup'
rm -rf /tmp/parallel.file* rm -rf /tmp/parallel.file*
stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*'
stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*'
(seq 1 3;echo '>fire';seq 5 10; echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17 (seq 1 3;echo '>fire';seq 5 10; /bin/echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17
echo '# Create some weirdly files in /tmp' echo '# Create some weirdly files in /tmp'
mkdir -p /tmp/parallel.file mkdir -p /tmp/parallel.file
cat /tmp/test17 | $PAR -k echo file{} '>'/tmp/parallel.file{}.file cat /tmp/test17 | $PAR -k /bin/echo file{} '>'/tmp/parallel.file{}.file
cat /tmp/test17 | $PAR -k echo /tmp/parallel.file{}.file >/tmp/test17abs cat /tmp/test17 | $PAR -k /bin/echo /tmp/parallel.file{}.file >/tmp/test17abs
cat /tmp/test17 | $PAR -k echo tmp/parallel.file{}.file >/tmp/test17rel cat /tmp/test17 | $PAR -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel
echo '### --transfer - abspath' echo '### --transfer - abspath'
stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' stdout ssh $SERVER1 'rm -rf tmp/parallel.file*'

View file

@ -2,7 +2,7 @@
PAR=parallel PAR=parallel
SERVER1=parallel-server1 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
echo '### Check warning if --transfer but file not found' echo '### Check warning if --transfer but file not found'

View file

@ -4,7 +4,7 @@
PAR=parallel PAR=parallel
SERVER1=parallel-server1 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
echo '### Test --transfer --return --cleanup - files with newline' echo '### Test --transfer --return --cleanup - files with newline'

View file

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
PAR=parallel PAR=parallel
SERVER1=parallel-server1 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
echo '### Test $PARALLEL' echo '### Test $PARALLEL'

View file

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
PAR=parallel PAR=parallel
SERVER1=parallel-server1 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
cd /tmp cd /tmp
@ -25,6 +25,6 @@ rm -f parallel_*.test parallel_*.out
seq 1 13 | parallel echo {} '>' parallel_{}.test seq 1 13 | parallel echo {} '>' parallel_{}.test
ls parallel_*.test | parallel -j+0 --trc {.}.out -B my_script \ ls parallel_*.test | parallel -j+0 --trc {.}.out -B my_script \
-S parallel-server1,parallel@parallel-server2,: "./my_script {} > {.}.out" -S $SERVER1,parallel@$SERVER2,: "./my_script {} > {.}.out"
cat parallel_*.test parallel_*.out cat parallel_*.test parallel_*.out

View file

@ -0,0 +1,4 @@
#!/bin/bash
echo '### 64-bit wierdness - this did not complete on a 64-bit machine'
seq 1 2 | parallel -j1 'seq 1 1 | parallel true'

View file

@ -1,13 +1,14 @@
### Test filenames containing UTF-8
. .
./1-col.txt
./2-col.txt
./a ./a
./a/foo2
./a/bar ./a/bar
./a/foo ./a/foo
./a/foo2
./b ./b
./b/bar ./b/bar
./b/foo ./b/foo
./2-col.txt
./中国 (Zhōngguó) ./中国 (Zhōngguó)
./中国 (Zhōngguó)/China's (中国) road.jpg ./中国 (Zhōngguó)/China's (中国) road.jpg
./中国 (Zhōngguó)/thumb_China's (中国) road.jpg ./中国 (Zhōngguó)/thumb_China's (中国) road.jpg
./1-col.txt

View file

@ -60,8 +60,8 @@ rm -- 中国\ \(Zhōngguó\)/abc-中国\ \(Zhōngguó\)-中国\ \(Zhōngguó\)
### Test -m ### Test -m
1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC 1foo bar joe.gif2foo bar joe3 Afoo bar joeBfoo bar joeC
1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC 1foo2foo3 1bar2bar3 1joe.gif2joe3 AfooBfooC AbarBbarC AjoeBjoeC
a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif b1 2 3 4 5 6 c1 2 3 4 5 6 a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6
a1.gifb1c1 a 2.gifb 2c 2 a 3.gifb 3c 3 a 4.gifb 4c 4 a 5.gifb 5c 5 a 6.gifb 6c 6 abc a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6
### Test -m with 60000 args ### Test -m with 60000 args
98c94dcab1efedab3f820935d230bc77 - 98c94dcab1efedab3f820935d230bc77 -
12 180011 1286837 12 180011 1286837

View file

@ -37,16 +37,16 @@ Warning: --cleanup ignored as there are no remote --sshlogin
9 9
10 10
### Check forced number of CPUs being respected ### Check forced number of CPUs being respected
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
alpha alpha
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
nlv.pi.dk
alpha alpha
alpha alpha
alpha alpha

View file

@ -75,17 +75,17 @@ OK
ls: cannot access tmp/parallel.file*: No such file or directory ls: cannot access tmp/parallel.file*: No such file or directory
OK OK
Input for ssh Input for ssh
parallel-server1 rsync --server -lDErRze.iLsf . ./ parallel-server3 rsync --server -lDErRze.iLsf . ./
parallel-server1 cat tmp/parallel.file.' parallel-server3 cat tmp/parallel.file.'
'newline2 > tmp/parallel.file.' 'newline2 > tmp/parallel.file.'
'newline2.out;cat tmp/parallel.file.' 'newline2.out;cat tmp/parallel.file.'
'newline2 > tmp/parallel.file.' 'newline2 > tmp/parallel.file.'
'newline2.out2 'newline2.out2
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
'newline2.out 'newline2.out
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.' parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
'newline2.out2 'newline2.out2
parallel-server1 rm -f tmp/parallel.file.' parallel-server3 rm -f tmp/parallel.file.'
'newline2 'newline2
-l parallel parallel-server2 rsync --server -lDErRze.iLsf . ./ -l parallel parallel-server2 rsync --server -lDErRze.iLsf . ./
parallel@parallel-server2 cat tmp/parallel.file.' parallel@parallel-server2 cat tmp/parallel.file.'

View file

@ -2,10 +2,10 @@
1 1
ssh -l parallel parallel-server2 echo\ 1; ssh -l parallel parallel-server2 echo\ 1;
1 1
ssh parallel-server1 echo\ 2; ssh parallel-server3 echo\ 2;
2 2
### Test ~/.parallelrc ### Test ~/.parallelrc
ssh -l parallel parallel-server2 echo\ 1; ssh -l parallel parallel-server2 echo\ 1;
1 1
ssh parallel-server1 echo\ 2; ssh parallel-server3 echo\ 2;
2 2