mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
Merge branch 'master' of git://git.savannah.gnu.org/parallel
This commit is contained in:
gfxmonk
commit
6f05588657
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -15,3 +15,5 @@ src/sem.1
|
||||||
src/sem.html
|
src/sem.html
|
||||||
src/sql.1
|
src/sql.1
|
||||||
src/sql.html
|
src/sql.html
|
||||||
|
src/niceload.1
|
||||||
|
src/niceload.html
|
||||||
|
|
|
||||||
58
NEWS
58
NEWS
|
|
@ -1,3 +1,61 @@
|
||||||
|
20101222
|
||||||
|
|
||||||
|
* GNU niceload is now part of GNU Parallel. GNU niceload slows down a
|
||||||
|
program if the load average is above a certain limit.
|
||||||
|
|
||||||
|
* Implemented --tmpdir to buffer standard output and standard error in
|
||||||
|
a different place.
|
||||||
|
|
||||||
|
* Implemented --load to wait until the load is below a limit before
|
||||||
|
starting another job on that computer.
|
||||||
|
|
||||||
|
* Implemented --nice set the niceness of jobs running both locally and
|
||||||
|
remotely.
|
||||||
|
|
||||||
|
* Implemented --dry-run to print the job without running it.
|
||||||
|
|
||||||
|
* Implemented --tty as the old default of assigning a tty to the first
|
||||||
|
job causes problems.
|
||||||
|
|
||||||
|
* Review with focus on clusters. Thanks to Taylor Gillespie
|
||||||
|
http://www.unixpronews.com/unixpronews-49-20101019GNUParallelSpeedUpProcessingWithMulticoresClusters.html
|
||||||
|
|
||||||
|
* Review with focus on protein similarity.
|
||||||
|
http://kevinformatics.tumblr.com/post/2142473893/cluster-like-computing-using-gnu-parallel
|
||||||
|
|
||||||
|
* Review in Spanish.
|
||||||
|
http://gr3p.com/2010/12/gnu-parallel-acelera-tus-scripts-en-linux
|
||||||
|
|
||||||
|
* Quite a few bug fixes and man page updates.
|
||||||
|
|
||||||
|
|
||||||
|
20101202
|
||||||
|
|
||||||
|
* Implemented {/} for the input line with the path removed (basename).
|
||||||
|
|
||||||
|
* Implemented {/.} for the input line with extension and path removed
|
||||||
|
(basename).
|
||||||
|
|
||||||
|
* Output from --progress is now sent to standard error instead of
|
||||||
|
standard output.
|
||||||
|
|
||||||
|
* --eta was broken and counted down from 0. Now fixed.
|
||||||
|
|
||||||
|
* Standard output and standard error are flushed after every job so if
|
||||||
|
standard output does not end with a newline it will still be grouped
|
||||||
|
with the rest of standard output.
|
||||||
|
|
||||||
|
* --command, -c, --file, and -f are now removed as options. They were
|
||||||
|
never used in practice.
|
||||||
|
|
||||||
|
* GetOptionsFromArray rewritten to work with old Perl libraries.
|
||||||
|
|
||||||
|
* The file COPYING now contains the GNU General Public License 3
|
||||||
|
|
||||||
|
* Major rewrite to make the code more object oriented and easier to
|
||||||
|
maintain in the future.
|
||||||
|
|
||||||
|
|
||||||
20101113
|
20101113
|
||||||
|
|
||||||
* Using -j myfile the number of jobs can be changed while GNU Parallel
|
* Using -j myfile the number of jobs can be changed while GNU Parallel
|
||||||
|
|
|
||||||
2
README
2
README
|
|
@ -6,7 +6,7 @@ Please send problems and feedback to bug-parallel@gnu.org.
|
||||||
= Presentation of GNU Parallel =
|
= Presentation of GNU Parallel =
|
||||||
|
|
||||||
GNU Parallel is a shell tool for executing jobs in parallel using one
|
GNU Parallel is a shell tool for executing jobs in parallel using one
|
||||||
or more machines. A job is typically a single command or a small
|
or more computers. A job is typically a single command or a small
|
||||||
script that has to be run for each of the lines in the input. The
|
script that has to be run for each of the lines in the input. The
|
||||||
typical input is a list of files, a list of hosts, a list of users, or
|
typical input is a list of files, a list of hosts, a list of users, or
|
||||||
a list of tables.
|
a list of tables.
|
||||||
|
|
|
||||||
20
configure
vendored
20
configure
vendored
|
|
@ -1,6 +1,6 @@
|
||||||
#! /bin/sh
|
#! /bin/sh
|
||||||
# Guess values for system-dependent variables and create Makefiles.
|
# Guess values for system-dependent variables and create Makefiles.
|
||||||
# Generated by GNU Autoconf 2.67 for parallel 20101122.
|
# Generated by GNU Autoconf 2.67 for parallel 20101222.
|
||||||
#
|
#
|
||||||
# Report bugs to <bug-parallel@gnu.org>.
|
# Report bugs to <bug-parallel@gnu.org>.
|
||||||
#
|
#
|
||||||
|
|
@ -551,8 +551,8 @@ MAKEFLAGS=
|
||||||
# Identity of this package.
|
# Identity of this package.
|
||||||
PACKAGE_NAME='parallel'
|
PACKAGE_NAME='parallel'
|
||||||
PACKAGE_TARNAME='parallel'
|
PACKAGE_TARNAME='parallel'
|
||||||
PACKAGE_VERSION='20101122'
|
PACKAGE_VERSION='20101222'
|
||||||
PACKAGE_STRING='parallel 20101122'
|
PACKAGE_STRING='parallel 20101222'
|
||||||
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
||||||
PACKAGE_URL=''
|
PACKAGE_URL=''
|
||||||
|
|
||||||
|
|
@ -1168,7 +1168,7 @@ if test "$ac_init_help" = "long"; then
|
||||||
# Omit some internal or obsolete options to make the list less imposing.
|
# Omit some internal or obsolete options to make the list less imposing.
|
||||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||||
cat <<_ACEOF
|
cat <<_ACEOF
|
||||||
\`configure' configures parallel 20101122 to adapt to many kinds of systems.
|
\`configure' configures parallel 20101222 to adapt to many kinds of systems.
|
||||||
|
|
||||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||||
|
|
||||||
|
|
@ -1234,7 +1234,7 @@ fi
|
||||||
|
|
||||||
if test -n "$ac_init_help"; then
|
if test -n "$ac_init_help"; then
|
||||||
case $ac_init_help in
|
case $ac_init_help in
|
||||||
short | recursive ) echo "Configuration of parallel 20101122:";;
|
short | recursive ) echo "Configuration of parallel 20101222:";;
|
||||||
esac
|
esac
|
||||||
cat <<\_ACEOF
|
cat <<\_ACEOF
|
||||||
|
|
||||||
|
|
@ -1301,7 +1301,7 @@ fi
|
||||||
test -n "$ac_init_help" && exit $ac_status
|
test -n "$ac_init_help" && exit $ac_status
|
||||||
if $ac_init_version; then
|
if $ac_init_version; then
|
||||||
cat <<\_ACEOF
|
cat <<\_ACEOF
|
||||||
parallel configure 20101122
|
parallel configure 20101222
|
||||||
generated by GNU Autoconf 2.67
|
generated by GNU Autoconf 2.67
|
||||||
|
|
||||||
Copyright (C) 2010 Free Software Foundation, Inc.
|
Copyright (C) 2010 Free Software Foundation, Inc.
|
||||||
|
|
@ -1318,7 +1318,7 @@ cat >config.log <<_ACEOF
|
||||||
This file contains any messages produced by compilers while
|
This file contains any messages produced by compilers while
|
||||||
running configure, to aid debugging if configure makes a mistake.
|
running configure, to aid debugging if configure makes a mistake.
|
||||||
|
|
||||||
It was created by parallel $as_me 20101122, which was
|
It was created by parallel $as_me 20101222, which was
|
||||||
generated by GNU Autoconf 2.67. Invocation command line was
|
generated by GNU Autoconf 2.67. Invocation command line was
|
||||||
|
|
||||||
$ $0 $@
|
$ $0 $@
|
||||||
|
|
@ -2133,7 +2133,7 @@ fi
|
||||||
|
|
||||||
# Define the identity of the package.
|
# Define the identity of the package.
|
||||||
PACKAGE='parallel'
|
PACKAGE='parallel'
|
||||||
VERSION='20101122'
|
VERSION='20101222'
|
||||||
|
|
||||||
|
|
||||||
cat >>confdefs.h <<_ACEOF
|
cat >>confdefs.h <<_ACEOF
|
||||||
|
|
@ -2684,7 +2684,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
||||||
# report actual input values of CONFIG_FILES etc. instead of their
|
# report actual input values of CONFIG_FILES etc. instead of their
|
||||||
# values after options handling.
|
# values after options handling.
|
||||||
ac_log="
|
ac_log="
|
||||||
This file was extended by parallel $as_me 20101122, which was
|
This file was extended by parallel $as_me 20101222, which was
|
||||||
generated by GNU Autoconf 2.67. Invocation command line was
|
generated by GNU Autoconf 2.67. Invocation command line was
|
||||||
|
|
||||||
CONFIG_FILES = $CONFIG_FILES
|
CONFIG_FILES = $CONFIG_FILES
|
||||||
|
|
@ -2746,7 +2746,7 @@ _ACEOF
|
||||||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||||
ac_cs_version="\\
|
ac_cs_version="\\
|
||||||
parallel config.status 20101122
|
parallel config.status 20101222
|
||||||
configured by $0, generated by GNU Autoconf 2.67,
|
configured by $0, generated by GNU Autoconf 2.67,
|
||||||
with options \\"\$ac_cs_config\\"
|
with options \\"\$ac_cs_config\\"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
AC_INIT([parallel], [20101122], [bug-parallel@gnu.org])
|
AC_INIT([parallel], [20101222], [bug-parallel@gnu.org])
|
||||||
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
|
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
|
||||||
AC_CONFIG_HEADERS([config.h])
|
AC_CONFIG_HEADERS([config.h])
|
||||||
AC_CONFIG_FILES([
|
AC_CONFIG_FILES([
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,34 @@
|
||||||
--command, -c, --file, and -f now obsoleted. --eta works again.
|
Testsuite: sem without ~/.parallel
|
||||||
Bugfix in testsuite for --retries.
|
|
||||||
Lots of dead code removed.
|
|
||||||
|
|
||||||
== Bug? ==
|
cleanup of transferred files in workdir fixed.
|
||||||
|
-T implemented as ssh/rsync sometimes hang due to getting a tty.
|
||||||
|
|
||||||
|
Til QUOTING:
|
||||||
|
|
||||||
|
FN="two spaces"
|
||||||
|
echo 1 | parallel -q echo {} "$FN"
|
||||||
|
# Prints 2 spaces between 'two' and 'spaces'
|
||||||
|
|
||||||
|
-q will not work with composed commands as it will quote the ; as
|
||||||
|
well. So composed commands have to be quoted by hand:
|
||||||
|
|
||||||
|
# Using export:
|
||||||
|
FN2="two spaces"
|
||||||
|
export FN2
|
||||||
|
echo 1 | parallel echo {} \"\$FN2\" \; echo \"\$FN2\" {}
|
||||||
|
# Prints 2 spaces between 'two' and 'spaces'
|
||||||
|
|
||||||
|
# Without export:
|
||||||
|
FN3="two spaces"
|
||||||
|
echo 1 | parallel echo {} \""$FN3"\" \; echo \'"$FN3"\' {}
|
||||||
|
|
||||||
|
# By quoting the space in the variable
|
||||||
|
FN4='two\ \ spaces'
|
||||||
|
echo 1 | parallel echo {} $FN4 \; echo $FN4 {}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
= Bug? ==
|
||||||
|
|
||||||
locate .gz | parallel -X find {} -size +1000 -size -2000 | parallel --workdir ... -S .. --trc {/}.bz2 'zcat {} | bzip2 > {/}.bz2'
|
locate .gz | parallel -X find {} -size +1000 -size -2000 | parallel --workdir ... -S .. --trc {/}.bz2 'zcat {} | bzip2 > {/}.bz2'
|
||||||
|
|
||||||
|
|
@ -12,30 +38,16 @@ locate .gz | parallel -X find {} -size +1000 -size -2000 | parallel --workdir ..
|
||||||
http://code.google.com/p/spawntool/
|
http://code.google.com/p/spawntool/
|
||||||
http://code.google.com/p/push/
|
http://code.google.com/p/push/
|
||||||
|
|
||||||
== Basename ==
|
|
||||||
|
|
||||||
find mp3/ -iname "*.mp3" | parallel -j+0 lame -S -q1 --vbr-new -V6 -b 32 -B128 {} /mnt/sda1/tmp-zik/{/}
|
|
||||||
|
|
||||||
== Bug? ==
|
== Bug? ==
|
||||||
|
|
||||||
.parallel/config with --long-options
|
.parallel/config with --long-options
|
||||||
|
|
||||||
time find . -type f | parallel -j+0 --eta -S..,: --progress --trc {}.gz gzip {}
|
time find . -type f | parallel -j+0 --eta -S..,: --progress --trc {}.gz gzip {}
|
||||||
|
|
||||||
== Bug --eta ==
|
|
||||||
|
|
||||||
Only negative.
|
|
||||||
|
|
||||||
== Workdir ==
|
== Workdir ==
|
||||||
|
|
||||||
Bug: {2} in --return
|
|
||||||
parallel -N2 --workdir ... --cleanup --transfer --return {2}.2 -S .. -v echo {} ">{2}.2" ::: ./tmp/foo ./tmp/bar
|
parallel -N2 --workdir ... --cleanup --transfer --return {2}.2 -S .. -v echo {} ">{2}.2" ::: ./tmp/foo ./tmp/bar
|
||||||
|
|
||||||
== load ==
|
|
||||||
|
|
||||||
Include niceload
|
|
||||||
Have an option to not spawn new tasks until load is below a limit computed like number of jobs.
|
|
||||||
|
|
||||||
== SQL ==
|
== SQL ==
|
||||||
|
|
||||||
Example with %0a as newline
|
Example with %0a as newline
|
||||||
|
|
@ -302,7 +314,8 @@ distribute the arguments evenly if running -X.
|
||||||
|
|
||||||
=head1 options
|
=head1 options
|
||||||
|
|
||||||
One char options not used: A F G K O Q R T Z
|
One char options not used: A F G K O Q R T Z c f
|
||||||
|
-T terminal
|
||||||
|
|
||||||
=head1 sem
|
=head1 sem
|
||||||
|
|
||||||
|
|
|
||||||
80
doc/promo
Normal file
80
doc/promo
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
=head1 search terms
|
||||||
|
|
||||||
|
run commands in parallel
|
||||||
|
|
||||||
|
Parallel shell loops
|
||||||
|
|
||||||
|
multi threading in bash xargs
|
||||||
|
|
||||||
|
# TAGS: parallel | parallel processing | multicore | multiprocessor | Clustering/Distributed Networks
|
||||||
|
# job control | multiple jobs | parallelization | text processing | cluster | filters
|
||||||
|
# Clustering Tools | Command Line Tools | Utilities | System Administration
|
||||||
|
# Bash parallel
|
||||||
|
|
||||||
|
GNU parallel execution shell bash script simultaneous concurrent linux
|
||||||
|
scripting run xargs ppss code.google.com/p/ppss/
|
||||||
|
|
||||||
|
@vvuksan @ychaker @ncb000gt
|
||||||
|
xargs can lead to nasty surprises caused by the separator problem
|
||||||
|
http://nd.gd/0t GNU Parallel http://nd.gd/0s may be better.
|
||||||
|
|
||||||
|
Comments:
|
||||||
|
|
||||||
|
http://nd.gd/0u http://www.gnu.org/software/parallel/
|
||||||
|
http://nd.gd/0s http://www.youtube.com/watch?v=OpaiGYxkSuQ
|
||||||
|
http://nd.gd/0t http://en.wikipedia.org/wiki/Xargs#The_separator_problem
|
||||||
|
http://nd.gd/3k http://www.gnu.org/software/parallel/man.html#differences_between_xargs_and_gnu_parallel
|
||||||
|
|
||||||
|
If you like xargs you may love GNU Parallel: http://nd.gd/0s
|
||||||
|
|
||||||
|
|
||||||
|
With GNU Parallel (http://nd.gd/0s) you can do:
|
||||||
|
ls | grep jpeg | parallel mv {} {.}.jpg
|
||||||
|
|
||||||
|
Watch the intro video for GNU Parallel: http://nd.gd/0s
|
||||||
|
|
||||||
|
If your input file names are generated by users, you need to deal with
|
||||||
|
surprising file names containing space, ', or " in the filename.
|
||||||
|
|
||||||
|
xargs can give nasty surprises due to the separator problem
|
||||||
|
http://en.wikipedia.org/wiki/Xargs#The_separator_problem
|
||||||
|
http://nd.gd/0t
|
||||||
|
|
||||||
|
GNU Parallel http://nd.gd/0s may be better.
|
||||||
|
|
||||||
|
@jaylyerly @stevenf xargs will bite you if file names contain
|
||||||
|
space http://nd.gd/0t. Use GNU Parallel instead: http://nd.gd/0s
|
||||||
|
|
||||||
|
Please repay by spreading the word about GNU Parallel to your
|
||||||
|
contacts/blog/facebook/linkedin/mailing lists/user group
|
||||||
|
|
||||||
|
Your use of xargs can lead to nasty surprises because of the separator
|
||||||
|
problem http://en.wikipedia.org/wiki/Xargs#The_separator_problem
|
||||||
|
|
||||||
|
GNU Parallel http://www.gnu.org/software/parallel/ does not have that
|
||||||
|
problem.
|
||||||
|
|
||||||
|
Watch the intro video for GNU Parallel:
|
||||||
|
http://www.youtube.com/watch?v=OpaiGYxkSuQ
|
||||||
|
|
||||||
|
If you have GNU Parallel http://www.gnu.org/software/parallel/ installed you can do this:
|
||||||
|
|
||||||
|
Watch the intro video for GNU Parallel to learn more:
|
||||||
|
http://www.youtube.com/watch?v=OpaiGYxkSuQ
|
||||||
|
|
||||||
|
|
||||||
|
GNU Parallel also makes it possible to run small scripts. Try this:
|
||||||
|
|
||||||
|
ls *.zip | parallel ‘mkdir {.}; cd {.}; unzip ../{}’
|
||||||
|
|
||||||
|
= Other people says =
|
||||||
|
|
||||||
|
fawxtin: Using GNU Parallel to run things simultaneously, neat! ;)
|
||||||
|
cowsandmilk: if you haven't quit xargs for gnu parallel yet, you're really missing out
|
||||||
|
afader: GNU Parallel is cool
|
||||||
|
ecyrd: Okay, why hasn't anyone told me of GNU parallel before?
|
||||||
|
cashion: very cool! Did not know about GNU parallel.
|
||||||
|
toorghezi: Writing shell scripts? You should know about GNU Parallel!
|
||||||
|
Racecar564 : GNU Parallel rocks!
|
||||||
|
arclight : Very nice! I especially like the {} -> {.}.bz2 syntax.
|
||||||
|
nwp8861 : I surprised that it's so comfortable!
|
||||||
|
|
@ -15,6 +15,7 @@ src/parallel: $Global::version = 20100422;
|
||||||
YYYYMMDD=`yyyymmdd`
|
YYYYMMDD=`yyyymmdd`
|
||||||
perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parallel
|
perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parallel
|
||||||
perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql
|
perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql
|
||||||
|
perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/niceload
|
||||||
|
|
||||||
=== Autoconf/automake ===
|
=== Autoconf/automake ===
|
||||||
|
|
||||||
|
|
@ -22,7 +23,7 @@ rm -fr autom4te.cache aclocal.m4 config.h config.h.in config.log Makefile.in mis
|
||||||
rm -rf src/Makefile.in
|
rm -rf src/Makefile.in
|
||||||
autoreconf --install -W gnu
|
autoreconf --install -W gnu
|
||||||
./configure
|
./configure
|
||||||
make && sudo make install
|
make -j && sudo make install
|
||||||
|
|
||||||
== Testsuite ==
|
== Testsuite ==
|
||||||
|
|
||||||
|
|
@ -40,6 +41,7 @@ make dist-bzip2
|
||||||
|
|
||||||
== Test the package ==
|
== Test the package ==
|
||||||
|
|
||||||
|
YYYYMMDD=`yyyymmdd`
|
||||||
cp parallel-$YYYYMMDD.tar.bz2 /tmp
|
cp parallel-$YYYYMMDD.tar.bz2 /tmp
|
||||||
cd /tmp
|
cd /tmp
|
||||||
tar xjvf parallel-$YYYYMMDD.tar.bz2
|
tar xjvf parallel-$YYYYMMDD.tar.bz2
|
||||||
|
|
@ -80,16 +82,22 @@ Content from release mail:
|
||||||
https://savannah.gnu.org/news/submit.php?group=parallel
|
https://savannah.gnu.org/news/submit.php?group=parallel
|
||||||
https://savannah.gnu.org/news/approve.php?group=parallel
|
https://savannah.gnu.org/news/approve.php?group=parallel
|
||||||
|
|
||||||
doc/pod2savannah_publicinfo src/parallel | klipper-stdin
|
# No longer updated. Too long and too buggy
|
||||||
https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel
|
# doc/pod2savannah_publicinfo src/parallel | klipper-stdin
|
||||||
|
# https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel
|
||||||
|
|
||||||
== Update website ==
|
== Update website ==
|
||||||
|
|
||||||
http://www.gnu.org/software/parallel/
|
http://www.gnu.org/software/parallel/
|
||||||
http://www.gnu.org/software/parallel/man.html
|
http://www.gnu.org/software/parallel/man.html
|
||||||
|
http://www.gnu.org/software/parallel/sql.html
|
||||||
|
http://www.gnu.org/software/parallel/sem.html
|
||||||
|
http://www.gnu.org/software/parallel/niceload.html
|
||||||
|
|
||||||
pod2html src/parallel > ../parallel-web/parallel/man.html
|
pod2html src/parallel.pod > ../parallel-web/parallel/man.html
|
||||||
pod2html src/sql > ../parallel-web/parallel/sql.html
|
pod2html src/sql > ../parallel-web/parallel/sql.html
|
||||||
|
pod2html src/niceload > ../parallel-web/parallel/niceload.html
|
||||||
|
pod2html src/sem.pod > ../parallel-web/parallel/sem.html
|
||||||
cd ../parallel-web/parallel
|
cd ../parallel-web/parallel
|
||||||
cvs up
|
cvs up
|
||||||
cvs ci
|
cvs ci
|
||||||
|
|
@ -100,7 +108,7 @@ http://freshmeat.net/projects/parallel/releases/new
|
||||||
|
|
||||||
== Update Twitter ==
|
== Update Twitter ==
|
||||||
|
|
||||||
#GNU Parallel 20101113 released. See what is new in this release
|
New version of #GNU Parallel released. See what is new in this release
|
||||||
http://nd.gd/2j Watch the intro video http://nd.gd/0s
|
http://nd.gd/2j Watch the intro video http://nd.gd/0s
|
||||||
|
|
||||||
https://savannah.gnu.org/news/?group=parallel
|
https://savannah.gnu.org/news/?group=parallel
|
||||||
|
|
@ -123,84 +131,49 @@ cc:Peter Simons <simons@cryp.to>, Sandro Cazzaniga <kharec@mandriva.org>,
|
||||||
Andrew McFague <amcfague@wgen.net>, Steven M. Christensen <sunfreeware@gmail.com>,
|
Andrew McFague <amcfague@wgen.net>, Steven M. Christensen <sunfreeware@gmail.com>,
|
||||||
Chris Howey <howeyc@gmail.com>, Fethican Coşkuner <fethicanc@gmail.com>,
|
Chris Howey <howeyc@gmail.com>, Fethican Coşkuner <fethicanc@gmail.com>,
|
||||||
Rogério Brito <rbrito@ime.usp.br>, Jonathan Palardy <jonathan.palardy@gmail.com>,
|
Rogério Brito <rbrito@ime.usp.br>, Jonathan Palardy <jonathan.palardy@gmail.com>,
|
||||||
Koen Vervloesem <koen@vervloesem.eu>, R. Tyler Croy <tyler@monkeypox.org>
|
Koen Vervloesem <koen@vervloesem.eu>, R. Tyler Croy <tyler@monkeypox.org>,
|
||||||
|
ryoichiro.suzuki@gmail.com,kerick@shiftedbit.net,
|
||||||
|
Christian Faulhammer <fauli@gentoo.org>
|
||||||
|
|
||||||
Subject: GNU Parallel 20101113 released
|
Subject: GNU Parallel 20101222 released
|
||||||
|
|
||||||
GNU Parallel 20101113 has been released. It is available for
|
GNU Parallel 20101222 has been released. It is available for
|
||||||
download at: http://ftp.gnu.org/gnu/parallel/
|
download at: http://ftp.gnu.org/gnu/parallel/
|
||||||
|
|
||||||
New in this release:
|
New in this release:
|
||||||
|
|
||||||
* Using -j myfile the number of jobs can be changed while GNU Parallel
|
* GNU niceload is now part of GNU Parallel. GNU niceload slows down a
|
||||||
is running simply by changing the content of myfile.
|
program if the load average is above a certain limit.
|
||||||
|
|
||||||
* Implemented --profile to use different .parallel/config for
|
* Implemented --tmpdir to buffer standard output and standard error in
|
||||||
different situations.
|
a different place.
|
||||||
|
|
||||||
* Ugly newlines in $PARALLEL and .parallel/config are no longer
|
* Implemented --load to wait until the load is below a limit before
|
||||||
needed. Instead you have to use \ in front of special shell
|
starting another job on that computer.
|
||||||
characters.
|
|
||||||
|
|
||||||
* --workdir puts the files transferred to remote machines in a
|
* Implemented --nice set the niceness of jobs running both locally and
|
||||||
specified directory.
|
remotely.
|
||||||
|
|
||||||
* $PARALLEL_PID is set to the process id of GNU Parallel.
|
* Implemented --dry-run to print the job without running it.
|
||||||
|
|
||||||
* $PARALLEL_SEQ is set to the sequence number of the job.
|
* Implemented --tty as the old default of assigning a tty to the first
|
||||||
|
job causes problems.
|
||||||
|
|
||||||
* -v now only shows the command to run. Use -v -v to show the
|
* Review with focus on clusters. Thanks to Taylor Gillespie
|
||||||
ssh/rsync wrapping.
|
http://www.unixpronews.com/unixpronews-49-20101019GNUParallelSpeedUpProcessingWithMulticoresClusters.html
|
||||||
|
|
||||||
* Slow spawning error is now only a warning.
|
* Review with focus on protein similarity.
|
||||||
|
http://kevinformatics.tumblr.com/post/2142473893/cluster-like-computing-using-gnu-parallel
|
||||||
|
|
||||||
* If stdin is a tty and input is read from stdin you now get a
|
* Review in Spanish.
|
||||||
warning.
|
http://gr3p.com/2010/12/gnu-parallel-acelera-tus-scripts-en-linux
|
||||||
|
|
||||||
* GNU sql: \n and \x0a in arguments is replaced with newline.
|
|
||||||
|
|
||||||
* Patch for Debian package and spelling mistakes. Thanks to Rogério
|
|
||||||
Brito <rbrito at ime dot usp dot br>
|
|
||||||
|
|
||||||
* Mac OS X Homebrew package. Thanks to Jonathan Palardy <jonathan dot
|
|
||||||
palardy at gmail dot com>
|
|
||||||
|
|
||||||
* FreeBSD port. Thanks to Chris Howey <howeyc at gmail dot com>
|
|
||||||
|
|
||||||
* Pardus package. Thanks to Fethican Coşkuner
|
|
||||||
<fethicanc at gmail dot com>
|
|
||||||
|
|
||||||
* First review in Chinese. Thanks to 曾義峰:
|
|
||||||
http://antbsd.twbbs.org/~ant/wordpress/?p=2876
|
|
||||||
|
|
||||||
* First review in print:
|
|
||||||
http://www.linux-magazine.com/Issues/2010 Nov 2010
|
|
||||||
|
|
||||||
* First review in Spanish:
|
|
||||||
http://www.muylinux.com/2010/10/18/gnu-parallel-computacion-paralela-a-golpe-de-comando
|
|
||||||
|
|
||||||
* First review in Dutch thanks to Koen Vervloesem <koen at vervloesem
|
|
||||||
dot eu>:
|
|
||||||
http://techworld.nl/technologie/33493/gebruik-al-je-processorkernen-met-gnu-parallel.html
|
|
||||||
|
|
||||||
* Blog review thanks to R. Tyler Croy <tyler at monkeypox dot org>:
|
|
||||||
http://unethicalblogger.com/posts/2010/11/gnuparallel_changed_my_life
|
|
||||||
|
|
||||||
* 5000 views of the intro video:
|
|
||||||
http://www.youtube.com/watch?v=OpaiGYxkSuQ
|
|
||||||
|
|
||||||
* As usual a bunch of bugfixes and more usage examples in the man
|
|
||||||
page.
|
|
||||||
|
|
||||||
* GNU Parallel was presented at FSCONS 2010-11-07:
|
|
||||||
http://www.fscons.org/fs/gnu-parallel Hopefully the
|
|
||||||
video will be online soon.
|
|
||||||
|
|
||||||
|
* Quite a few bug fixes and man page updates.
|
||||||
|
|
||||||
= About GNU Parallel =
|
= About GNU Parallel =
|
||||||
|
|
||||||
GNU Parallel is a shell tool for executing jobs in parallel using one
|
GNU Parallel is a shell tool for executing jobs in parallel using one
|
||||||
or more machines. A job is typically a single command or a small
|
or more computers. A job is typically a single command or a small
|
||||||
script that has to be run for each of the lines in the input. The
|
script that has to be run for each of the lines in the input. The
|
||||||
typical input is a list of files, a list of hosts, a list of users, a
|
typical input is a list of files, a list of hosts, a list of users, a
|
||||||
list of URLs, or a list of tables.
|
list of URLs, or a list of tables.
|
||||||
|
|
|
||||||
|
|
@ -1 +1,2 @@
|
||||||
make
|
make
|
||||||
|
Edit 20100906-2 to the version number
|
||||||
|
|
|
||||||
|
|
@ -1,46 +1,3 @@
|
||||||
parallel (20100922-4) unstable; urgency=low
|
|
||||||
|
|
||||||
* Change package to be non-native.
|
|
||||||
* Move to format "3.0 (quilt)".
|
|
||||||
* debian/compat:
|
|
||||||
+ update to 7.
|
|
||||||
* debian/rules:
|
|
||||||
+ don't configure the package in the clean target.
|
|
||||||
+ create a configure (and configure-stamp) target.
|
|
||||||
* debian/control:
|
|
||||||
+ update debhelper dependency to >= 7.
|
|
||||||
+ add dh-autoreconf to build-dependency.
|
|
||||||
+ indicate that the package conforms to Policy 3.9.1.
|
|
||||||
+ set myself as maintainer and Ole as uploader.
|
|
||||||
* debian/rules:
|
|
||||||
+ use dh(7) style file with autoreconf add-on.
|
|
||||||
* debian/doc-base:
|
|
||||||
+ properly register the documentation with doc-base.
|
|
||||||
* debian/patches:
|
|
||||||
+ include some patches to fix spelling errors detected by lintian.
|
|
||||||
* debian/dirs:
|
|
||||||
+ remove, not needed.
|
|
||||||
|
|
||||||
-- Rogério Brito <rbrito@ime.usp.br> Mon, 25 Oct 2010 03:44:37 -0200
|
|
||||||
|
|
||||||
parallel (20100922-3) unstable; urgency=low
|
|
||||||
|
|
||||||
* New upstream version - 3rd try
|
|
||||||
|
|
||||||
-- Ole Tange <ole@tange.dk> Tue, 21 Sep 2010 22:15:05 +0200
|
|
||||||
|
|
||||||
parallel (20100922-2) unstable; urgency=low
|
|
||||||
|
|
||||||
* New upstream version
|
|
||||||
|
|
||||||
-- Ole Tange <ole@tange.dk> Tue, 21 Sep 2010 22:13:53 +0200
|
|
||||||
|
|
||||||
parallel (20100922-1) unstable; urgency=low
|
|
||||||
|
|
||||||
* New upstream version
|
|
||||||
|
|
||||||
-- Ole Tange <ole@tange.dk> Tue, 21 Sep 2010 22:12:22 +0200
|
|
||||||
|
|
||||||
parallel (20100906-1) unstable; urgency=low
|
parallel (20100906-1) unstable; urgency=low
|
||||||
|
|
||||||
* New upstream version
|
* New upstream version
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
7
|
5
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
Source: parallel
|
Source: parallel
|
||||||
Section: utils
|
Section: utils
|
||||||
Priority: extra
|
Priority: extra
|
||||||
Maintainer: Rogério Brito <rbrito@ime.usp.br>
|
Maintainer: Ole Tange <ole@tange.dk>
|
||||||
Uploaders: Ole Tange <ole@tange.dk>
|
Build-Depends: debhelper (>= 5)
|
||||||
Build-Depends: debhelper (>= 7.0.50~), dh-autoreconf
|
Standards-Version: 3.8.3
|
||||||
Standards-Version: 3.9.1
|
|
||||||
Homepage: https://www.gnu.org/software/parallel/
|
Homepage: https://www.gnu.org/software/parallel/
|
||||||
|
|
||||||
Package: parallel
|
Package: parallel
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,7 @@ License:
|
||||||
|
|
||||||
The Debian packaging is:
|
The Debian packaging is:
|
||||||
|
|
||||||
Copyright © 2010 Ole Tange <ole@tange.dk>
|
Copyright (C) 2010 Ole Tange <ole@tange.dk>
|
||||||
Copyright © 2010 Rogério Theodoro de Brito <rbrito@ime.usp.br>
|
|
||||||
|
|
||||||
and is licensed under the GPL version 3,
|
and is licensed under the GPL version 3,
|
||||||
see `/usr/share/common-licenses/GPL-3'.
|
see `/usr/share/common-licenses/GPL-3'.
|
||||||
|
|
|
||||||
2
packager/debian/debian-template/dirs
Normal file
2
packager/debian/debian-template/dirs
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
usr/bin
|
||||||
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
Document: parallel
|
|
||||||
Title: Manual of GNU parallel
|
|
||||||
Author: Ole Tange
|
|
||||||
Abstract: This manual describes the use of GNU parallel,
|
|
||||||
provides an introduction to the subject and also presents
|
|
||||||
comparisons against similar utilities.
|
|
||||||
Section: System/Administration
|
|
||||||
|
|
||||||
Format: HTML
|
|
||||||
Index: /usr/share/doc/parallel/parallel.html
|
|
||||||
Files: /usr/share/doc/parallel/*.html
|
|
||||||
1
packager/debian/debian-template/files
Normal file
1
packager/debian/debian-template/files
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
parallel_20100906-6_all.deb utils extra
|
||||||
13
packager/debian/debian-template/parallel.debhelper.log
Normal file
13
packager/debian/debian-template/parallel.debhelper.log
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
dh_prep
|
||||||
|
dh_installdirs
|
||||||
|
dh_installchangelogs
|
||||||
|
dh_installdocs
|
||||||
|
dh_installexamples
|
||||||
|
dh_installman
|
||||||
|
dh_link
|
||||||
|
dh_compress
|
||||||
|
dh_fixperms
|
||||||
|
dh_installdeb
|
||||||
|
dh_gencontrol
|
||||||
|
dh_md5sums
|
||||||
|
dh_builddeb
|
||||||
1
packager/debian/debian-template/parallel.substvars
Normal file
1
packager/debian/debian-template/parallel.substvars
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
misc:Depends=
|
||||||
|
|
@ -1,52 +0,0 @@
|
||||||
Description: Fix spelling errors in some manpages
|
|
||||||
Author: Rogério Brito <rbrito@ime.usp.br>
|
|
||||||
Forwarded: no
|
|
||||||
Last-Update: 2010-10-24
|
|
||||||
|
|
||||||
--- a/src/parallel
|
|
||||||
+++ b/src/parallel
|
|
||||||
@@ -1095,7 +1095,7 @@
|
|
||||||
|
|
||||||
=head1 EXAMPLE: Group output lines
|
|
||||||
|
|
||||||
-When runnning jobs that output data, you often do not want the output
|
|
||||||
+When running jobs that output data, you often do not want the output
|
|
||||||
of multiple jobs to run together. GNU B<parallel> defaults to grouping the
|
|
||||||
output of each job, so the output is printed when the job finishes. If
|
|
||||||
you want the output to be printed while the job is running you can use
|
|
||||||
@@ -1311,12 +1311,12 @@
|
|
||||||
B<parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1}>
|
|
||||||
|
|
||||||
|
|
||||||
-=head1 EXAMPLE: Working as cat | sh. Ressource inexpensive jobs and evaluation
|
|
||||||
+=head1 EXAMPLE: Working as cat | sh. Resource inexpensive jobs and evaluation
|
|
||||||
|
|
||||||
GNU B<parallel> can work similar to B<cat | sh>.
|
|
||||||
|
|
||||||
-A ressource inexpensive job is a job that takes very little CPU, disk
|
|
||||||
-I/O and network I/O. Ping is an example of a ressource inexpensive
|
|
||||||
+A resource inexpensive job is a job that takes very little CPU, disk
|
|
||||||
+I/O and network I/O. Ping is an example of a resource inexpensive
|
|
||||||
job. wget is too - if the webpages are small.
|
|
||||||
|
|
||||||
The content of the file jobs_to_run:
|
|
||||||
@@ -2976,7 +2976,7 @@
|
|
||||||
# Number of parallel processes to run
|
|
||||||
|
|
||||||
sub compute_number_of_processes {
|
|
||||||
- # Number of processes wanted and limited by system ressources
|
|
||||||
+ # Number of processes wanted and limited by system resources
|
|
||||||
# Returns:
|
|
||||||
# Number of processes
|
|
||||||
my $opt_P = shift;
|
|
||||||
--- a/src/sql
|
|
||||||
+++ b/src/sql
|
|
||||||
@@ -118,7 +118,7 @@
|
|
||||||
|
|
||||||
=item B<-s> I<string>
|
|
||||||
|
|
||||||
-Field separator. Use I<string> as seperator between columns.
|
|
||||||
+Field separator. Use I<string> as separator between columns.
|
|
||||||
|
|
||||||
|
|
||||||
=item B<--skip-first-line>
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
01-fix-spelling.patch
|
|
||||||
|
|
@ -1,7 +1,64 @@
|
||||||
#!/usr/bin/make -f
|
#!/usr/bin/make -f
|
||||||
%:
|
# -*- makefile -*-
|
||||||
dh $@ --with autoreconf
|
# Sample debian/rules that uses debhelper.
|
||||||
|
# This file was originally written by Joey Hess and Craig Small.
|
||||||
|
# As a special exception, when this file is copied by dh-make into a
|
||||||
|
# dh-make output file, you may use that output file without restriction.
|
||||||
|
# This special exception was added by Craig Small in version 0.37 of dh-make.
|
||||||
|
|
||||||
override_dh_clean:
|
# Uncomment this to turn on verbose mode.
|
||||||
rm -f src/sem
|
#export DH_VERBOSE=1
|
||||||
dh_clean
|
|
||||||
|
build: build-stamp
|
||||||
|
|
||||||
|
build-stamp:
|
||||||
|
dh_testdir
|
||||||
|
|
||||||
|
# Add here commands to compile the package.
|
||||||
|
$(MAKE)
|
||||||
|
|
||||||
|
touch $@
|
||||||
|
|
||||||
|
clean:
|
||||||
|
dh_testdir
|
||||||
|
dh_testroot
|
||||||
|
dh_installdirs
|
||||||
|
rm -f build-stamp
|
||||||
|
|
||||||
|
# Add here commands to clean up after the build process.
|
||||||
|
./configure --prefix=/usr
|
||||||
|
$(MAKE) clean
|
||||||
|
|
||||||
|
dh_clean
|
||||||
|
|
||||||
|
install: build
|
||||||
|
dh_testdir
|
||||||
|
dh_testroot
|
||||||
|
dh_prep || dh_clean -k
|
||||||
|
dh_installdirs
|
||||||
|
|
||||||
|
# Add here commands to install the package into debian/parallel.
|
||||||
|
$(MAKE) DESTDIR=$(CURDIR)/debian/parallel install
|
||||||
|
|
||||||
|
|
||||||
|
# Build architecture-independent files here.
|
||||||
|
binary-indep: install
|
||||||
|
dh_testdir
|
||||||
|
dh_testroot
|
||||||
|
dh_installchangelogs
|
||||||
|
dh_installdocs
|
||||||
|
dh_installexamples
|
||||||
|
dh_installman
|
||||||
|
dh_link
|
||||||
|
dh_compress
|
||||||
|
dh_fixperms
|
||||||
|
dh_installdeb
|
||||||
|
dh_gencontrol
|
||||||
|
dh_md5sums
|
||||||
|
dh_builddeb
|
||||||
|
|
||||||
|
# Build architecture-dependent files here.
|
||||||
|
binary-arch: install
|
||||||
|
|
||||||
|
binary: binary-indep binary-arch
|
||||||
|
.PHONY: build clean binary-indep binary-arch binary install
|
||||||
|
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
3.0 (quilt)
|
|
||||||
|
|
@ -1,7 +1,5 @@
|
||||||
all:
|
all:
|
||||||
# cd ../debian/ && make
|
cd ../debian/ && make
|
||||||
# cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/
|
cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/
|
||||||
cp `ls ../../parallel-*.tar.bz2|tail -n1` home\:tange/parallel/
|
cp `ls ../../parallel-*.tar.bz2|tail -n1` home\:tange/parallel/
|
||||||
cd home\:tange/parallel/ && osc up && osc add *.dsc *.tar.gz && osc ci
|
cd home\:tange/parallel/ && osc up && osc add *.dsc *.tar.gz *.spec *.tar.bz2 && osc ci
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1 +1,4 @@
|
||||||
https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
|
make
|
||||||
|
Edit parallel.spec
|
||||||
|
Then go to https://build.opensuse.org/package/files?package=parallel&project=home%3Atange
|
||||||
|
and add all the files
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
bin_SCRIPTS = parallel sem sql
|
bin_SCRIPTS = parallel sem sql niceload
|
||||||
man_MANS = parallel.1 sem.1 sql.1
|
man_MANS = parallel.1 sem.1 sql.1 niceload.1
|
||||||
doc_DATA = parallel.html sem.html sql.html
|
doc_DATA = parallel.html sem.html sql.html niceload.html
|
||||||
|
|
||||||
parallel.1: parallel Makefile
|
parallel.1: parallel.pod Makefile
|
||||||
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
--section=1 $(srcdir)/parallel > $(srcdir)/parallel.1
|
--section=1 $(srcdir)/parallel.pod > $(srcdir)/parallel.1
|
||||||
|
|
||||||
sem.1: sem.pod Makefile
|
sem.1: sem.pod Makefile
|
||||||
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
|
|
@ -14,20 +14,31 @@ sql.1: sql Makefile
|
||||||
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
--section=1 $(srcdir)/sql > $(srcdir)/sql.1
|
--section=1 $(srcdir)/sql > $(srcdir)/sql.1
|
||||||
|
|
||||||
parallel.html: parallel Makefile
|
niceload.1: niceload Makefile
|
||||||
pod2html $(srcdir)/parallel > $(srcdir)/parallel.html
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
rm -f $(srcdir)/pod2htm*
|
--section=1 $(srcdir)/niceload > $(srcdir)/niceload.1
|
||||||
|
|
||||||
sem.html: sem.pod Makefile
|
parallel.html: parallel Makefile sem
|
||||||
pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html
|
./sem --fg --id pod2html pod2html $(srcdir)/parallel.pod > $(srcdir)/parallel.html
|
||||||
rm -f $(srcdir)/pod2htm*
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
sql.html: sql Makefile
|
sem.html: sem.pod Makefile sem
|
||||||
pod2html $(srcdir)/sql > $(srcdir)/sql.html
|
./sem --fg --id pod2html pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html
|
||||||
rm -f $(srcdir)/pod2htm*
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
|
sql.html: sql Makefile sem
|
||||||
|
./sem --fg --id pod2html pod2html $(srcdir)/sql > $(srcdir)/sql.html
|
||||||
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
|
niceload.html: niceload Makefile sem
|
||||||
|
./sem --fg --id pod2html pod2html $(srcdir)/niceload > $(srcdir)/niceload.html
|
||||||
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
sem: parallel
|
sem: parallel
|
||||||
ln -fs parallel sem
|
ln -fs parallel sem
|
||||||
|
|
||||||
DISTCLEANFILES = parallel.1 sem.1 sql.1 parallel.html sem.html sql.html
|
DISTCLEANFILES = parallel.1 sem.1 sql.1 niceload.1 parallel.html sem.html sql.html niceload.html
|
||||||
EXTRA_DIST = parallel sem sql parallel.1 sem.1 sql.1 parallel.html sem.html sem.pod sql.html
|
EXTRA_DIST = parallel sem sql niceload \
|
||||||
|
parallel.1 sem.1 sql.1 niceload.1 \
|
||||||
|
parallel.html sem.html sql.html niceload.html \
|
||||||
|
sem.pod parallel.pod
|
||||||
|
|
|
||||||
|
|
@ -146,11 +146,15 @@ target_alias = @target_alias@
|
||||||
top_build_prefix = @top_build_prefix@
|
top_build_prefix = @top_build_prefix@
|
||||||
top_builddir = @top_builddir@
|
top_builddir = @top_builddir@
|
||||||
top_srcdir = @top_srcdir@
|
top_srcdir = @top_srcdir@
|
||||||
bin_SCRIPTS = parallel sem sql
|
bin_SCRIPTS = parallel sem sql niceload
|
||||||
man_MANS = parallel.1 sem.1 sql.1
|
man_MANS = parallel.1 sem.1 sql.1 niceload.1
|
||||||
doc_DATA = parallel.html sem.html sql.html
|
doc_DATA = parallel.html sem.html sql.html niceload.html
|
||||||
DISTCLEANFILES = parallel.1 sem.1 sql.1 parallel.html sem.html sql.html
|
DISTCLEANFILES = parallel.1 sem.1 sql.1 niceload.1 parallel.html sem.html sql.html niceload.html
|
||||||
EXTRA_DIST = parallel sem sql parallel.1 sem.1 sql.1 parallel.html sem.html sem.pod sql.html
|
EXTRA_DIST = parallel sem sql niceload \
|
||||||
|
parallel.1 sem.1 sql.1 niceload.1 \
|
||||||
|
parallel.html sem.html sql.html niceload.html \
|
||||||
|
sem.pod parallel.pod
|
||||||
|
|
||||||
all: all-am
|
all: all-am
|
||||||
|
|
||||||
.SUFFIXES:
|
.SUFFIXES:
|
||||||
|
|
@ -443,9 +447,9 @@ uninstall-man: uninstall-man1
|
||||||
uninstall-man1
|
uninstall-man1
|
||||||
|
|
||||||
|
|
||||||
parallel.1: parallel Makefile
|
parallel.1: parallel.pod Makefile
|
||||||
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
--section=1 $(srcdir)/parallel > $(srcdir)/parallel.1
|
--section=1 $(srcdir)/parallel.pod > $(srcdir)/parallel.1
|
||||||
|
|
||||||
sem.1: sem.pod Makefile
|
sem.1: sem.pod Makefile
|
||||||
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
|
|
@ -455,17 +459,25 @@ sql.1: sql Makefile
|
||||||
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
--section=1 $(srcdir)/sql > $(srcdir)/sql.1
|
--section=1 $(srcdir)/sql > $(srcdir)/sql.1
|
||||||
|
|
||||||
parallel.html: parallel Makefile
|
niceload.1: niceload Makefile
|
||||||
pod2html $(srcdir)/parallel > $(srcdir)/parallel.html
|
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
|
||||||
rm -f $(srcdir)/pod2htm*
|
--section=1 $(srcdir)/niceload > $(srcdir)/niceload.1
|
||||||
|
|
||||||
sem.html: sem.pod Makefile
|
parallel.html: parallel Makefile sem
|
||||||
pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html
|
./sem --fg --id pod2html pod2html $(srcdir)/parallel.pod > $(srcdir)/parallel.html
|
||||||
rm -f $(srcdir)/pod2htm*
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
sql.html: sql Makefile
|
sem.html: sem.pod Makefile sem
|
||||||
pod2html $(srcdir)/sql > $(srcdir)/sql.html
|
./sem --fg --id pod2html pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html
|
||||||
rm -f $(srcdir)/pod2htm*
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
|
sql.html: sql Makefile sem
|
||||||
|
./sem --fg --id pod2html pod2html $(srcdir)/sql > $(srcdir)/sql.html
|
||||||
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
|
niceload.html: niceload Makefile sem
|
||||||
|
./sem --fg --id pod2html pod2html $(srcdir)/niceload > $(srcdir)/niceload.html
|
||||||
|
./sem --fg --id pod2html rm -f $(srcdir)/pod2htm*
|
||||||
|
|
||||||
sem: parallel
|
sem: parallel
|
||||||
ln -fs parallel sem
|
ln -fs parallel sem
|
||||||
|
|
|
||||||
483
src/niceload
Executable file
483
src/niceload
Executable file
|
|
@ -0,0 +1,483 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
|
||||||
|
=head1 NAME
|
||||||
|
|
||||||
|
niceload - slow down a program when the load average is above a certain limit
|
||||||
|
|
||||||
|
=head1 SYNOPSIS
|
||||||
|
|
||||||
|
B<niceload> [-v] [-n nice] [-l load] [-t time] [-s time|-f factor] command
|
||||||
|
|
||||||
|
B<niceload> [-v] [-n nice] [-l load] [-t time] [-s time|-f factor] -p=PID
|
||||||
|
|
||||||
|
=head1 DESCRIPTION
|
||||||
|
|
||||||
|
GNU B<niceload> will run a program when the load average is below a
|
||||||
|
certain limit. When the limit is reached the program will be suspended
|
||||||
|
for some time. Then resumed again for some time. Then the load load
|
||||||
|
average is checked again and we start over.
|
||||||
|
|
||||||
|
If the load is 3.00 then the default settings will run a program
|
||||||
|
like this:
|
||||||
|
|
||||||
|
run 1 second, suspend (3.00-1.00) seconds, run 1 second, suspend
|
||||||
|
(3.00-1.00) seconds, run 1 second, ...
|
||||||
|
|
||||||
|
=head1 OPTIONS
|
||||||
|
|
||||||
|
=over 9
|
||||||
|
|
||||||
|
=item B<-n> I<niceness>
|
||||||
|
|
||||||
|
Sets niceness. See B<nice>(1).
|
||||||
|
|
||||||
|
=item B<-l> I<maxload>
|
||||||
|
|
||||||
|
Max load. The maximal load average before suspending command. Default
|
||||||
|
is 1.00.
|
||||||
|
|
||||||
|
=item B<-t> I<SEC>
|
||||||
|
|
||||||
|
Recheck load time. Sleep SEC seconds before checking load
|
||||||
|
again. Default is 1 second.
|
||||||
|
|
||||||
|
=item B<-s> I<SEC>
|
||||||
|
|
||||||
|
Suspend time. Suspend the command this many seconds when the max load
|
||||||
|
average is reached.
|
||||||
|
|
||||||
|
=item B<-f> I<FACTOR>
|
||||||
|
|
||||||
|
Suspend time factor. Dynamically set B<-s> as max load average over limit * factor. Default is 1.
|
||||||
|
|
||||||
|
=item B<-p> I<PID>
|
||||||
|
|
||||||
|
Process ID of process to suspend.
|
||||||
|
|
||||||
|
=item B<-v>
|
||||||
|
|
||||||
|
Verbose. Print some extra output on what is happening. Use B<-v> until
|
||||||
|
you know what your are doing.
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
=head1 EXAMPLE: See niceload in action
|
||||||
|
|
||||||
|
In terminal 1 run: top
|
||||||
|
|
||||||
|
In terminal 2 run:
|
||||||
|
|
||||||
|
B<niceload perl -e '$|=1;do{$l==$r or print "."; $l=$r}until(($r=time-$^T)>>B<50)'>
|
||||||
|
|
||||||
|
This will print a '.' every second for 50 seconds and eat a lot of
|
||||||
|
CPU. When the load rises to 1.0 the process is suspended.
|
||||||
|
|
||||||
|
|
||||||
|
=head1 EXAMPLE: Run updatedb
|
||||||
|
|
||||||
|
Running updatedb can often starve the system for disk I/O and thus result in a high load.
|
||||||
|
|
||||||
|
Run updatedb but suspend updatedb if the load is above 2.00:
|
||||||
|
|
||||||
|
B<niceload -l=2 updatedb>
|
||||||
|
|
||||||
|
|
||||||
|
=head1 EXAMPLE: Run rsync
|
||||||
|
|
||||||
|
rsync can just like updatedb starve the system for disk I/O and thus result in a high load.
|
||||||
|
|
||||||
|
Run rsync but keep load below 3.4. If load reaches 7 sleep for
|
||||||
|
(7-3.4)*12 seconds:
|
||||||
|
|
||||||
|
B<niceload -l=3.4 -f=12 rsync -Ha /home/ /backup/home/>
|
||||||
|
|
||||||
|
|
||||||
|
=head1 ENVIRONMENT VARIABLES
|
||||||
|
|
||||||
|
None. In future versions $NICELOAD will be able to contain default settings.
|
||||||
|
|
||||||
|
=head1 EXIT STATUS
|
||||||
|
|
||||||
|
Exit status should be the same as the command being run (untested).
|
||||||
|
|
||||||
|
=head1 REPORTING BUGS
|
||||||
|
|
||||||
|
Report bugs to <bug-parallel@gnu.org>.
|
||||||
|
|
||||||
|
=head1 AUTHOR
|
||||||
|
|
||||||
|
Copyright (C) 2004-11-19 Ole Tange, http://ole.tange.dk
|
||||||
|
|
||||||
|
Copyright (C) 2005,2006,2006,2008,2009,2010 Ole Tange, http://ole.tange.dk
|
||||||
|
|
||||||
|
Copyright (C) 2010 Ole Tange, http://ole.tange.dk and Free Software
|
||||||
|
Foundation, Inc.
|
||||||
|
|
||||||
|
=head1 LICENSE
|
||||||
|
|
||||||
|
Copyright (C) 2010 Free Software Foundation, Inc.
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation; either version 3 of the License, or
|
||||||
|
at your option any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
=head2 Documentation license I
|
||||||
|
|
||||||
|
Permission is granted to copy, distribute and/or modify this documentation
|
||||||
|
under the terms of the GNU Free Documentation License, Version 1.3 or
|
||||||
|
any later version published by the Free Software Foundation; with no
|
||||||
|
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
|
||||||
|
Texts. A copy of the license is included in the file fdl.txt.
|
||||||
|
|
||||||
|
=head2 Documentation license II
|
||||||
|
|
||||||
|
You are free:
|
||||||
|
|
||||||
|
=over 9
|
||||||
|
|
||||||
|
=item B<to Share>
|
||||||
|
|
||||||
|
to copy, distribute and transmit the work
|
||||||
|
|
||||||
|
=item B<to Remix>
|
||||||
|
|
||||||
|
to adapt the work
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
Under the following conditions:
|
||||||
|
|
||||||
|
=over 9
|
||||||
|
|
||||||
|
=item B<Attribution>
|
||||||
|
|
||||||
|
You must attribute the work in the manner specified by the author or
|
||||||
|
licensor (but not in any way that suggests that they endorse you or
|
||||||
|
your use of the work).
|
||||||
|
|
||||||
|
=item B<Share Alike>
|
||||||
|
|
||||||
|
If you alter, transform, or build upon this work, you may distribute
|
||||||
|
the resulting work only under the same, similar or a compatible
|
||||||
|
license.
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
With the understanding that:
|
||||||
|
|
||||||
|
=over 9
|
||||||
|
|
||||||
|
=item B<Waiver>
|
||||||
|
|
||||||
|
Any of the above conditions can be waived if you get permission from
|
||||||
|
the copyright holder.
|
||||||
|
|
||||||
|
=item B<Public Domain>
|
||||||
|
|
||||||
|
Where the work or any of its elements is in the public domain under
|
||||||
|
applicable law, that status is in no way affected by the license.
|
||||||
|
|
||||||
|
=item B<Other Rights>
|
||||||
|
|
||||||
|
In no way are any of the following rights affected by the license:
|
||||||
|
|
||||||
|
=over 2
|
||||||
|
|
||||||
|
=item *
|
||||||
|
|
||||||
|
Your fair dealing or fair use rights, or other applicable
|
||||||
|
copyright exceptions and limitations;
|
||||||
|
|
||||||
|
=item *
|
||||||
|
|
||||||
|
The author's moral rights;
|
||||||
|
|
||||||
|
=item *
|
||||||
|
|
||||||
|
Rights other persons may have either in the work itself or in
|
||||||
|
how the work is used, such as publicity or privacy rights.
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
=over 9
|
||||||
|
|
||||||
|
=item B<Notice>
|
||||||
|
|
||||||
|
For any reuse or distribution, you must make clear to others the
|
||||||
|
license terms of this work.
|
||||||
|
|
||||||
|
=back
|
||||||
|
|
||||||
|
A copy of the full license is included in the file as cc-by-sa.txt.
|
||||||
|
|
||||||
|
=head1 DEPENDENCIES
|
||||||
|
|
||||||
|
GNU B<niceload> uses Perl, and the Perl modules POSIX, and
|
||||||
|
Getopt::Long.
|
||||||
|
|
||||||
|
=head1 SEE ALSO
|
||||||
|
|
||||||
|
B<parallel>(1), B<nice>(1)
|
||||||
|
|
||||||
|
|
||||||
|
=cut
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use Getopt::Long;
|
||||||
|
$Global::progname="niceload";
|
||||||
|
$Global::version = 20101222;
|
||||||
|
Getopt::Long::Configure("bundling","require_order");
|
||||||
|
get_options_from_array(\@ARGV) || die_usage();
|
||||||
|
if($::opt_version) {
|
||||||
|
version();
|
||||||
|
exit 0;
|
||||||
|
}
|
||||||
|
if($::opt_help) {
|
||||||
|
help();
|
||||||
|
exit 0;
|
||||||
|
}
|
||||||
|
if($::opt_factor and $::opt_suspend) {
|
||||||
|
# You cannot have --suspend and --factor
|
||||||
|
help();
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $nice = $::opt_nice || 0; # -n=0 Nice level (Default: 0)
|
||||||
|
my $max_load = $::opt_load || 1; # -l=1 Max acceptable load average (Default: 1)
|
||||||
|
my $check_time = $::opt_recheck || 1; # -t=1 Seconds between checking load average (Default: 1)
|
||||||
|
my $wait_factor;
|
||||||
|
my $wait_time;
|
||||||
|
if($::opt_suspend) {
|
||||||
|
# --suspend=sec Seconds to suspend process when load average is too high
|
||||||
|
$wait_time = $::opt_suspend;
|
||||||
|
} else {
|
||||||
|
# --factor=1 compute wait_time dynamically as (load - limit) * factor
|
||||||
|
$wait_factor=$::opt_factor || 1;
|
||||||
|
}
|
||||||
|
my $processid = $::opt_pid; # Control this PID (Default: control the command)
|
||||||
|
my $verbose = $::opt_verbose || $::opt_debug;
|
||||||
|
my @program = @ARGV;
|
||||||
|
$SIG{CHLD} = \&REAPER;
|
||||||
|
|
||||||
|
if($processid) {
|
||||||
|
$Child::fork = $processid;
|
||||||
|
$::opt_verbose and print STDERR "Control $processid\n";
|
||||||
|
init_signal_handling_attached_child();
|
||||||
|
my $child_pgrp = getpgrp $Child::fork;
|
||||||
|
suspend_resume($max_load,$check_time,$wait_time,$wait_factor,$child_pgrp);
|
||||||
|
} elsif(@ARGV) {
|
||||||
|
if($Child::fork = fork) {
|
||||||
|
sleep 1; # Give child time to setpgrp(0,0);
|
||||||
|
init_signal_handling_my_child();
|
||||||
|
my $child_pgrp = getpgrp $Child::fork;
|
||||||
|
suspend_resume($max_load,$check_time,$wait_time,$wait_factor,$child_pgrp);
|
||||||
|
} else {
|
||||||
|
setpgrp(0,0);
|
||||||
|
debug("Child pid: $$, pgrp: ",getpgrp $$,"\n");
|
||||||
|
if($nice) {
|
||||||
|
unshift(@program,"nice","-n",$nice);
|
||||||
|
}
|
||||||
|
debug("@program\n");
|
||||||
|
system(@program);
|
||||||
|
debug("Child exit\n");
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
help();
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub get_options_from_array {
|
||||||
|
# Run GetOptions on @array
|
||||||
|
# Returns:
|
||||||
|
# true if parsing worked
|
||||||
|
# false if parsing failed
|
||||||
|
# @array is changed
|
||||||
|
my $array_ref = shift;
|
||||||
|
# A bit of shuffling of @ARGV needed as GetOptionsFromArray is not
|
||||||
|
# supported everywhere
|
||||||
|
my @save_argv;
|
||||||
|
my $this_is_ARGV = (\@::ARGV == $array_ref);
|
||||||
|
if(not $this_is_ARGV) {
|
||||||
|
@save_argv = @::ARGV;
|
||||||
|
@::ARGV = @{$array_ref};
|
||||||
|
}
|
||||||
|
my @retval = GetOptions
|
||||||
|
("debug|D" => \$::opt_debug,
|
||||||
|
"load|l=s" => \$::opt_load,
|
||||||
|
"factor|f=s" => \$::opt_factor,
|
||||||
|
"suspend|s=s" => \$::opt_suspend,
|
||||||
|
"recheck|t=s" => \$::opt_recheck,
|
||||||
|
"nice|n=i" => \$::opt_nice,
|
||||||
|
"help|h" => \$::opt_help,
|
||||||
|
"process|pid|p=s" => \$::opt_pid,
|
||||||
|
"verbose|v" => \$::opt_verbose,
|
||||||
|
"version|V" => \$::opt_version,
|
||||||
|
);
|
||||||
|
if(not $this_is_ARGV) {
|
||||||
|
@{$array_ref} = @::ARGV;
|
||||||
|
@::ARGV = @save_argv;
|
||||||
|
}
|
||||||
|
return @retval;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub die_usage {
|
||||||
|
help();
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub help {
|
||||||
|
print q{
|
||||||
|
Usage:
|
||||||
|
niceload [-v] [-n=niceness] [-l=loadavg] [-t=recheck_sec] [-s=suspend_sec|-f=factor] command
|
||||||
|
niceload [-v] [-n=niceness] [-l=loadavg] [-t=recheck_sec] [-s=suspend_sec|-f=factor] command
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sub debug {
|
||||||
|
if($::opt_debug) {
|
||||||
|
print STDERR @_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub version {
|
||||||
|
# Returns: N/A
|
||||||
|
print join("\n",
|
||||||
|
"GNU $Global::progname $Global::version",
|
||||||
|
"Copyright (C) 2004,2005,2006,2007,2008,2009 Ole Tange",
|
||||||
|
"Copyright (C) 2010 Ole Tange and Free Software Foundation, Inc.",
|
||||||
|
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
|
||||||
|
"This is free software: you are free to change and redistribute it.",
|
||||||
|
"GNU $Global::progname comes with no warranty.",
|
||||||
|
"",
|
||||||
|
"Web site: http://www.gnu.org/software/parallel\n"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
sub init_signal_handling_attached_child {
|
||||||
|
$SIG{INT}=\&sigint_attached_child;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub sigint_attached_child {
|
||||||
|
# Let the attached child continue when detaching
|
||||||
|
kill_child_CONT();
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub init_signal_handling_my_child {
|
||||||
|
$SIG{INT}=\&kill_child_INT;
|
||||||
|
$SIG{TSTP}=\&kill_child_TSTP;
|
||||||
|
$SIG{CONT}=\&kill_child_CONT;
|
||||||
|
}
|
||||||
|
|
||||||
|
use POSIX ":sys_wait_h";
|
||||||
|
|
||||||
|
sub REAPER {
|
||||||
|
my $stiff;
|
||||||
|
while (($stiff = waitpid(-1, &WNOHANG)) > 0) {
|
||||||
|
# do something with $stiff if you want
|
||||||
|
}
|
||||||
|
$SIG{CHLD} = \&REAPER; # install *after* calling waitpid
|
||||||
|
}
|
||||||
|
|
||||||
|
sub kill_child_CONT {
|
||||||
|
debug("SIGCONT received. Killing $Child::fork\n");
|
||||||
|
kill CONT => -getpgrp($Child::fork);
|
||||||
|
}
|
||||||
|
|
||||||
|
sub kill_child_TSTP {
|
||||||
|
debug("SIGTSTP received. Killing $Child::fork and self\n");
|
||||||
|
kill TSTP => -getpgrp($Child::fork);
|
||||||
|
kill STOP => -$$;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub kill_child_INT {
|
||||||
|
debug("SIGINT received. Killing $Child::fork Exit\n");
|
||||||
|
kill INT => -getpgrp($Child::fork);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub suspend_resume {
|
||||||
|
my ($max_load,$check_time,$wait_time,$wait_factor,@pids) = @_;
|
||||||
|
debug("suspend_resume these @pids\n");
|
||||||
|
resume_pids(@pids);
|
||||||
|
while (pids_exist(@pids)) {
|
||||||
|
if ( loadavg() > $max_load ) {
|
||||||
|
if($wait_factor) {
|
||||||
|
$wait_time = (loadavg()-$max_load) * $wait_factor;
|
||||||
|
}
|
||||||
|
$::opt_verbose and print STDERR "suspending for $wait_time seconds\n";
|
||||||
|
suspend_pids(@pids);
|
||||||
|
sleep 1; # for some reason this statement is skipped
|
||||||
|
sleep $wait_time;
|
||||||
|
resume_pids(@pids);
|
||||||
|
}
|
||||||
|
$::opt_verbose and print STDERR "running for $check_time second(s)\n";
|
||||||
|
sleep($check_time);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub pids_exist {
|
||||||
|
my (@pids) = @_;
|
||||||
|
my ($exists) = 0;
|
||||||
|
for my $pid (@pids) {
|
||||||
|
if(-e "/proc/".$pid) { $exists++ }
|
||||||
|
#if(kill 0 => $Child::fork) { $exists++ }
|
||||||
|
}
|
||||||
|
return $exists;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub loadavg {
|
||||||
|
my ($loadavg);
|
||||||
|
if(open(IN,"/proc/loadavg")) {
|
||||||
|
# Linux specific (but fast)
|
||||||
|
my $upString = <IN>;
|
||||||
|
if($upString =~ m/^(\d+\.\d+)/) {
|
||||||
|
$loadavg = $1;
|
||||||
|
} else {
|
||||||
|
die;
|
||||||
|
}
|
||||||
|
close IN;
|
||||||
|
} elsif (open(IN,"uptime|")) {
|
||||||
|
my $upString = <IN>;
|
||||||
|
if($upString =~ m/average.\s*(\d+\.\d+)/) {
|
||||||
|
$loadavg = $1;
|
||||||
|
} else {
|
||||||
|
die;
|
||||||
|
}
|
||||||
|
close IN;
|
||||||
|
}
|
||||||
|
return $loadavg;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub suspend_pids {
|
||||||
|
my @pids = @_;
|
||||||
|
signal_pids("STOP",@pids);
|
||||||
|
}
|
||||||
|
|
||||||
|
sub resume_pids {
|
||||||
|
my @pids = @_;
|
||||||
|
signal_pids("CONT",@pids);
|
||||||
|
}
|
||||||
|
|
||||||
|
sub signal_pids {
|
||||||
|
my ($signal,@pids) = @_;
|
||||||
|
|
||||||
|
# local $SIG{$signal} = 'IGNORE';
|
||||||
|
for my $pid (@pids) {
|
||||||
|
kill $signal => -$pid; # stop PID group
|
||||||
|
}
|
||||||
|
}
|
||||||
3743
src/parallel
3743
src/parallel
File diff suppressed because it is too large
Load diff
2584
src/parallel.pod
Normal file
2584
src/parallel.pod
Normal file
File diff suppressed because it is too large
Load diff
225
src/sem.pod
225
src/sem.pod
|
|
@ -81,6 +81,17 @@ available.
|
||||||
done
|
done
|
||||||
sem --wait
|
sem --wait
|
||||||
|
|
||||||
|
=head1 EXAMPLE: Protecting pod2html from itself
|
||||||
|
|
||||||
|
pod2html creates two files: pod2htmd.tmp and pod2htmi.tmp which it
|
||||||
|
does not clean up. It uses these two files for a short time. But if
|
||||||
|
you run multiple pod2html in parallel (e.g. in a Makefile with make
|
||||||
|
-j) you need to protect pod2html from running twice at the same
|
||||||
|
time. B<sem> running as a mutex will do just that:
|
||||||
|
|
||||||
|
sem --fg --id pod2html pod2html foo.pod > foo.html
|
||||||
|
sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp
|
||||||
|
|
||||||
|
|
||||||
=head1 BUGS
|
=head1 BUGS
|
||||||
|
|
||||||
|
|
@ -217,217 +228,3 @@ Symbol, Fcntl.
|
||||||
B<parallel>(1)
|
B<parallel>(1)
|
||||||
|
|
||||||
=cut
|
=cut
|
||||||
|
|
||||||
use strict;
|
|
||||||
use Symbol qw(gensym);
|
|
||||||
use Getopt::Long;
|
|
||||||
|
|
||||||
Getopt::Long::Configure ("bundling","require_order");
|
|
||||||
GetOptions("debug|D" => \$::opt_D,
|
|
||||||
"id|i=s" => \$::opt_id,
|
|
||||||
"count|j=i" => \$::opt_count,
|
|
||||||
"fg" => \$::opt_fg,
|
|
||||||
"timeout|t=i" => \$::opt_timeout,
|
|
||||||
"version" => \$::opt_version,
|
|
||||||
"wait|w" => \$::opt_wait,
|
|
||||||
) || die_usage();
|
|
||||||
$Global::debug = $::opt_D;
|
|
||||||
$Global::version = 20100814;
|
|
||||||
$Global::progname = 'sem';
|
|
||||||
|
|
||||||
my $count = 1; # Default 1 = mutex
|
|
||||||
if($::opt_count) {
|
|
||||||
$count = $::opt_count + 1;
|
|
||||||
}
|
|
||||||
if($::opt_wait) {
|
|
||||||
$count = 1;
|
|
||||||
}
|
|
||||||
my $id = $::opt_id;
|
|
||||||
my $fg = $::opt_fg || $::opt_wait;
|
|
||||||
$::opt_timeout = $::opt_timeout;
|
|
||||||
if(defined $::opt_version) {
|
|
||||||
version();
|
|
||||||
}
|
|
||||||
|
|
||||||
if(not defined $id) {
|
|
||||||
# $id = getppid();
|
|
||||||
# does not work with:
|
|
||||||
# find . -name '*linux*' -exec sem -j1000 "sleep 3; echo `tty` '{}'" \; ; sem --wait echo done
|
|
||||||
$id = `tty`;
|
|
||||||
}
|
|
||||||
$id = "id-$id";
|
|
||||||
$id=~s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex
|
|
||||||
my $sem = Semaphore->new($id,$count);
|
|
||||||
$sem->acquire();
|
|
||||||
debug("run");
|
|
||||||
if($fg) {
|
|
||||||
system @ARGV;
|
|
||||||
$sem->release();
|
|
||||||
} else {
|
|
||||||
# If run in the background, the PID will change
|
|
||||||
# therefore release and re-acquire the semaphore
|
|
||||||
$sem->release();
|
|
||||||
if(not fork()) {
|
|
||||||
# child
|
|
||||||
# Get a semaphore for this pid
|
|
||||||
my $child_sem = Semaphore->new($id,$count);
|
|
||||||
$child_sem->acquire();
|
|
||||||
system @ARGV;
|
|
||||||
$child_sem->release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sub version {
|
|
||||||
# Returns: N/A
|
|
||||||
print join("\n",
|
|
||||||
"GNU $Global::progname $Global::version",
|
|
||||||
"Copyright (C) 2010 Ole Tange and Free Software Foundation, Inc.",
|
|
||||||
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
|
|
||||||
"This is free software: you are free to change and redistribute it.",
|
|
||||||
"GNU $Global::progname comes with no warranty.",
|
|
||||||
"",
|
|
||||||
"Web site: http://www.gnu.org/software/parallel\n"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
sub usage {
|
|
||||||
# Returns: N/A
|
|
||||||
print "Usage:\n";
|
|
||||||
print "$Global::progname [options] [command [arguments]] < list_of_arguments)\n";
|
|
||||||
print "$Global::progname [options] [command [arguments]] ::: arguments\n";
|
|
||||||
print "$Global::progname [options] [command [arguments]] :::: argfile(s)\n";
|
|
||||||
print "\n";
|
|
||||||
print "See 'man $Global::progname' for the options\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
sub die_usage {
|
|
||||||
usage();
|
|
||||||
exit(255);
|
|
||||||
}
|
|
||||||
|
|
||||||
sub debug {
|
|
||||||
# Returns: N/A
|
|
||||||
$Global::debug or return;
|
|
||||||
@_ = grep { defined $_ ? $_ : "" } @_;
|
|
||||||
print map {$_,"\n" } @_;
|
|
||||||
}
|
|
||||||
|
|
||||||
package Semaphore;
|
|
||||||
|
|
||||||
# This package provides a counting semaphore
|
|
||||||
#
|
|
||||||
# If a process dies without releasing the semaphore the next process
|
|
||||||
# that needs that entry will clean up dead semaphores
|
|
||||||
#
|
|
||||||
# The semaphores are stored in ~/.parallel/semaphores/id-<name> Each
|
|
||||||
# file in ~/.parallel/semaphores/id-<name>/ is the process ID of the
|
|
||||||
# process holding the entry. If the process dies, the entry can be
|
|
||||||
# taken by another process.
|
|
||||||
|
|
||||||
use Fcntl qw(:DEFAULT :flock);
|
|
||||||
|
|
||||||
sub new {
|
|
||||||
my $class = shift;
|
|
||||||
my $id = shift;
|
|
||||||
my $count = shift;
|
|
||||||
my $parallel_locks = $ENV{'HOME'}."/.parallel/semaphores";
|
|
||||||
-d $parallel_locks or mkdir $parallel_locks;
|
|
||||||
my $lockdir = "$parallel_locks/$id";
|
|
||||||
my $lockfile = $lockdir.".lock";
|
|
||||||
return bless {
|
|
||||||
'lockfile' => $lockfile,
|
|
||||||
'lockfh' => Symbol::gensym(),
|
|
||||||
'lockdir' => $lockdir,
|
|
||||||
'id' => $id,
|
|
||||||
'idfile' => $lockdir."/".$id,
|
|
||||||
'pid' => $$,
|
|
||||||
'pidfile' => $lockdir."/".$$,
|
|
||||||
'count' => $count
|
|
||||||
}, ref($class) || $class;
|
|
||||||
}
|
|
||||||
|
|
||||||
sub acquire {
|
|
||||||
my $self = shift;
|
|
||||||
while(1) {
|
|
||||||
$self->atomic_link_if_count_less_than() and last;
|
|
||||||
::debug("Remove dead locks");
|
|
||||||
my $lockdir = $self->{'lockdir'};
|
|
||||||
for my $d (<$lockdir/*>) {
|
|
||||||
$d =~ m:$lockdir/([0-9]+):o or next;
|
|
||||||
if(not kill 0, $1) {
|
|
||||||
::debug("Dead: $d");
|
|
||||||
unlink $d;
|
|
||||||
} else {
|
|
||||||
::debug("Alive: $d");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# try again
|
|
||||||
$self->atomic_link_if_count_less_than() and last;
|
|
||||||
sleep 1;
|
|
||||||
# TODO if timeout: last
|
|
||||||
}
|
|
||||||
::debug("got $self->{'pid'}");
|
|
||||||
}
|
|
||||||
|
|
||||||
sub release {
|
|
||||||
my ($self) = shift;
|
|
||||||
unlink $self->{'pidfile'};
|
|
||||||
if($self->nlinks() == 1) {
|
|
||||||
# This is the last link, so atomic cleanup
|
|
||||||
$self->lock();
|
|
||||||
if($self->nlinks() == 1) {
|
|
||||||
unlink $self->{'idfile'};
|
|
||||||
rmdir $self->{'lockdir'};
|
|
||||||
}
|
|
||||||
$self->unlock();
|
|
||||||
}
|
|
||||||
::debug("released $self->{'pid'}");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sub atomic_link_if_count_less_than {
|
|
||||||
# Link $file1 to $file2 if nlinks to $file1 < $count
|
|
||||||
my ($self) = shift;
|
|
||||||
my ($retval) = 0;
|
|
||||||
$self->lock();
|
|
||||||
if($self->nlinks() < $count) {
|
|
||||||
-d $self->{'lockdir'} || mkdir $self->{'lockdir'};
|
|
||||||
if(not -e $self->{'idfile'}) {
|
|
||||||
open (A, ">", $self->{'idfile'}) or die ">$self->{'idfile'}";
|
|
||||||
close A;
|
|
||||||
}
|
|
||||||
$retval = link $self->{'idfile'}, $self->{'pidfile'};
|
|
||||||
}
|
|
||||||
$self->unlock();
|
|
||||||
::debug("atomic $retval");
|
|
||||||
return $retval;
|
|
||||||
}
|
|
||||||
|
|
||||||
sub nlinks {
|
|
||||||
my $self = shift;
|
|
||||||
if(-e $self->{'idfile'}) {
|
|
||||||
return (stat(_))[3];
|
|
||||||
} else {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sub lock {
|
|
||||||
my ($self) = shift;
|
|
||||||
open $self->{'lockfh'}, ">", $self->{'lockfile'}
|
|
||||||
or die "Can't open semaphore file $self->{'lockfile'}: $!";
|
|
||||||
chmod 0666, $self->{'lockfile'}; # assuming you want it a+rw
|
|
||||||
while(not flock $self->{'lockfh'}, LOCK_EX()|LOCK_NB()) {
|
|
||||||
::debug("Cannot lock $self->{'lockfile'}");
|
|
||||||
# TODO if timeout: last
|
|
||||||
sleep 1;
|
|
||||||
}
|
|
||||||
::debug("locked $self->{'lockfile'}");
|
|
||||||
}
|
|
||||||
|
|
||||||
sub unlock {
|
|
||||||
my $self = shift;
|
|
||||||
unlink $self->{'lockfile'};
|
|
||||||
close $self->{'lockfh'};
|
|
||||||
::debug("unlocked");
|
|
||||||
}
|
|
||||||
|
|
|
||||||
2
src/sql
2
src/sql
|
|
@ -528,7 +528,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
||||||
exit ($err);
|
exit ($err);
|
||||||
|
|
||||||
sub parse_options {
|
sub parse_options {
|
||||||
$Global::version = 20101122;
|
$Global::version = 20101222;
|
||||||
$Global::progname = 'sql';
|
$Global::progname = 'sql';
|
||||||
|
|
||||||
# This must be done first as this may exec myself
|
# This must be done first as this may exec myself
|
||||||
|
|
|
||||||
11
testsuite/tests-to-run/niceload01.sh
Normal file
11
testsuite/tests-to-run/niceload01.sh
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo '### Test niceload'
|
||||||
|
niceload -s 1 perl -e '$|=1;do{$l==$r or print "."; $l=$r}until(($r=time-$^T)>10)'
|
||||||
|
echo
|
||||||
|
|
||||||
|
#echo '### Test niceload -p'
|
||||||
|
#sleep 3 &
|
||||||
|
#nice-load -v -p $!
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# Test if we can deal with output > 4 GB
|
# Test if we can deal with output > 4 GB
|
||||||
echo | niceload -l=1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
|
echo | niceload -l 1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
|
||||||
# dd does not work with niceload (no idea why)
|
# dd does not work with niceload (no idea why)
|
||||||
#echo | parallel 'dd if=/dev/zero count=43 bs=100000k; echo 1; echo 2' | md5sum
|
#echo | parallel 'dd if=/dev/zero count=43 bs=100000k; echo 1; echo 2' | md5sum
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ echo 3 | xargs -P 1 -n 1 -a files cat -
|
||||||
echo 'parallel Expect: 3 1 via psedotty 2'
|
echo 'parallel Expect: 3 1 via psedotty 2'
|
||||||
cat >/tmp/parallel-script-for-script <<EOF
|
cat >/tmp/parallel-script-for-script <<EOF
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
echo 3 | parallel -k -P 1 -n 1 -a files cat -
|
echo 3 | parallel -T -k -P 1 -n 1 -a files cat -
|
||||||
EOF
|
EOF
|
||||||
chmod 755 /tmp/parallel-script-for-script
|
chmod 755 /tmp/parallel-script-for-script
|
||||||
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script /dev/null
|
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script /dev/null
|
||||||
|
|
@ -102,7 +102,7 @@ echo 3 | xargs -I {} -P 1 -n 1 -a files cat {} -
|
||||||
echo 'parallel Expect: 1 3 2 via pseudotty'
|
echo 'parallel Expect: 1 3 2 via pseudotty'
|
||||||
cat >/tmp/parallel-script-for-script2 <<EOF
|
cat >/tmp/parallel-script-for-script2 <<EOF
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
echo 3 | parallel -k -I {} -P 1 -n 1 -a files cat {} -
|
echo 3 | parallel -T -k -I {} -P 1 -n 1 -a files cat {} -
|
||||||
EOF
|
EOF
|
||||||
chmod 755 /tmp/parallel-script-for-script2
|
chmod 755 /tmp/parallel-script-for-script2
|
||||||
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script2 /dev/null
|
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script2 /dev/null
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ parallel --argsep .--- -kv .--- 'echo a' 'echo b'
|
||||||
echo '### Test stdin goes to first command only ("-" as argument)'
|
echo '### Test stdin goes to first command only ("-" as argument)'
|
||||||
cat >/tmp/parallel-script-for-script <<EOF
|
cat >/tmp/parallel-script-for-script <<EOF
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
echo via first cat |parallel -kv cat ::: - -
|
echo via first cat |parallel -T -kv cat ::: - -
|
||||||
EOF
|
EOF
|
||||||
chmod 755 /tmp/parallel-script-for-script
|
chmod 755 /tmp/parallel-script-for-script
|
||||||
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script /dev/null
|
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script /dev/null
|
||||||
|
|
@ -21,7 +21,7 @@ sleep 1
|
||||||
echo '### Test stdin goes to first command only ("cat" as argument)'
|
echo '### Test stdin goes to first command only ("cat" as argument)'
|
||||||
cat >/tmp/parallel-script-for-script2 <<EOF
|
cat >/tmp/parallel-script-for-script2 <<EOF
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
echo no output |parallel -kv ::: 'echo a' 'cat'
|
echo no output |parallel -T -kv ::: 'echo a' 'cat'
|
||||||
EOF
|
EOF
|
||||||
chmod 755 /tmp/parallel-script-for-script2
|
chmod 755 /tmp/parallel-script-for-script2
|
||||||
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script2 /dev/null
|
echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script2 /dev/null
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,37 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
echo '### Test of --retries'
|
|
||||||
seq 1 10 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo
|
|
||||||
|
|
||||||
echo '### Test of --retries - it should run 13 jobs in total'
|
echo '### Test of --retries - it should run 13 jobs in total'
|
||||||
seq 0 12 | stdout parallel --progress -kj100% --retries 1 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
|
seq 0 12 | stdout parallel --progress -kj100% --retries 1 -S 12/localhost,1/:,parallel@server2 -vq \
|
||||||
perl -e 'sleep 1;print "job{}\n";exit({})' | \
|
perl -e 'sleep 1;print "job{}\n";exit({})' | \
|
||||||
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
|
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
|
||||||
|
|
||||||
echo '### Test of --retries - it should run 25 jobs in total'
|
echo '### Test of --retries - it should run 25 jobs in total'
|
||||||
seq 0 12 | stdout parallel --progress -kj100% --retries 2 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
|
seq 0 12 | stdout parallel --progress -kj100% --retries 2 -S 12/localhost,1/:,parallel@server2 -vq \
|
||||||
perl -e 'sleep 1;print "job{}\n";exit({})' | \
|
perl -e 'sleep 1;print "job{}\n";exit({})' | \
|
||||||
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
|
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
|
||||||
|
|
||||||
echo '### Test of --retries - it should run 49 jobs in total'
|
echo '### Test of --retries - it should run 49 jobs in total'
|
||||||
seq 0 12 | stdout parallel --progress -kj100% --retries 4 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
|
seq 0 12 | stdout parallel --progress -kj100% --retries 4 -S 12/localhost,1/:,parallel@server2 -vq \
|
||||||
perl -e 'sleep 1;print "job{}\n";exit({})' | \
|
perl -e 'sleep 1;print "job{}\n";exit({})' | \
|
||||||
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
|
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
|
||||||
|
|
||||||
|
echo '### Bug with --retries'
|
||||||
|
seq 1 8 | parallel --retries 2 --sshlogin 8/localhost,8/: -j+0 "hostname; false" | wc -l
|
||||||
|
seq 1 8 | parallel --retries 2 --sshlogin 8/localhost,8/: -j+1 "hostname; false" | wc -l
|
||||||
|
seq 1 2 | parallel --retries 2 --sshlogin 8/localhost,8/: -j-1 "hostname; false" | wc -l
|
||||||
|
seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j1 "hostname; false" | wc -l
|
||||||
|
seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j9 "hostname; false" | wc -l
|
||||||
|
seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j0 "hostname; false" | wc -l
|
||||||
|
seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j-1 "hostname; false" | wc -l
|
||||||
|
|
||||||
|
echo '### These were not affected by the bug'
|
||||||
|
seq 1 8 | parallel --retries 2 --sshlogin 1/localhost,9/: -j-1 "hostname; false" | wc -l
|
||||||
|
seq 1 8 | parallel --retries 2 --sshlogin 8/localhost,8/: -j-1 "hostname; false" | wc -l
|
||||||
|
seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: "hostname; false" | wc -l
|
||||||
|
seq 1 4 | parallel --retries 2 --sshlogin 2/localhost,2/: -j-1 "hostname; false" | wc -l
|
||||||
|
seq 1 4 | parallel --retries 2 --sshlogin 2/localhost,2/: -j1 "hostname; false" | wc -l
|
||||||
|
seq 1 4 | parallel --retries 2 --sshlogin 1/localhost,1/: -j1 "hostname; false" | wc -l
|
||||||
|
seq 1 2 | parallel --retries 2 --sshlogin 1/localhost,1/: -j1 "hostname; false" | wc -l
|
||||||
|
|
||||||
|
echo '### Test of --retries on unreachable host'
|
||||||
|
seq 1 10 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
SERVER1=parallel-server3
|
SERVER1=parallel-server3
|
||||||
SERVER2=parallel-server2
|
SERVER2=parallel-server2
|
||||||
|
|
||||||
echo '### Test -M'
|
echo '### Test -M (--retries to avoid false errors)'
|
||||||
|
|
||||||
seq 1 20 | parallel -k -M -S 9/$SERVER1,9/parallel@$SERVER2 echo
|
seq 1 30 | parallel -j5 --retries 3 -k -M -S $SERVER1,parallel@$SERVER2 echo
|
||||||
|
seq 1 30 | parallel -j10 --retries 3 -k -M -S $SERVER1,parallel@$SERVER2 echo
|
||||||
|
|
|
||||||
38
testsuite/tests-to-run/test42.sh
Normal file
38
testsuite/tests-to-run/test42.sh
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "### Test --basenamereplace"
|
||||||
|
parallel -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b
|
||||||
|
parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b
|
||||||
|
|
||||||
|
echo "### Test --basenameextensionreplace"
|
||||||
|
parallel -k -X --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b
|
||||||
|
parallel -k --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b
|
||||||
|
|
||||||
|
echo "### Test {/}"
|
||||||
|
parallel -k -X echo {/} ::: /a/b.c a/b.c b.c /a/b a/b b
|
||||||
|
|
||||||
|
echo "### Test {/.}"
|
||||||
|
parallel -k -X echo {/.} ::: /a/b.c a/b.c b.c /a/b a/b b
|
||||||
|
|
||||||
|
echo "### Test {#/.}"
|
||||||
|
parallel -k -X echo {2/.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6
|
||||||
|
|
||||||
|
echo "### Test {#/}"
|
||||||
|
parallel -k -X echo {2/} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6
|
||||||
|
|
||||||
|
echo "### Test {#.}"
|
||||||
|
parallel -k -X echo {2.} ::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6
|
||||||
|
|
||||||
|
SERVER1=parallel-server3
|
||||||
|
SERVER2=parallel-server2
|
||||||
|
rm -rf tmp
|
||||||
|
|
||||||
|
echo "### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"
|
||||||
|
stdout parallel -k -Xv --cleanup --return tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file -S parallel@$SERVER2 \
|
||||||
|
mkdir -p tmp/{/}_{/.}_{2/.}_{2/}_{2.} \;touch tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file \
|
||||||
|
::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 'number 7' 'number <8|8>'
|
||||||
|
find tmp
|
||||||
|
rm -rf tmp
|
||||||
|
|
||||||
|
echo "### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"
|
||||||
|
echo "### But we will wait for a real world scenario"
|
||||||
25
testsuite/tests-to-run/test43.sh
Normal file
25
testsuite/tests-to-run/test43.sh
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Assume /dev/shm is easy to fill up
|
||||||
|
mkdir -p /dev/shm/parallel
|
||||||
|
|
||||||
|
echo '### Test $TMPDIR'
|
||||||
|
TMPDIR=/dev/shm/parallel stdout timeout -k 1 6 parallel head -c 2000m '<'{} >/dev/null ::: /dev/zero &
|
||||||
|
seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 1" \
|
||||||
|
| stdout timeout -k 1 10 perl -ne 'BEGIN{$a=<>} $b=<>; if ($a-1000 > $b) { die "More than 1 MB gone. Good!" }'
|
||||||
|
wait
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
echo '### Test --tmpdir'
|
||||||
|
stdout timeout -k 1 6 parallel --tmpdir /dev/shm/parallel head -c 2000m '<'{} >/dev/null ::: /dev/zero &
|
||||||
|
seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 1" \
|
||||||
|
| stdout timeout -k 1 10 perl -ne 'BEGIN{$a=<>} $b=<>; if ($a-1000 > $b) { die "More than 1 MB gone. Good!" }'
|
||||||
|
wait
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
echo '### Test $TMPDIR and --tmpdir'
|
||||||
|
TMPDIR=/tmp stdout timeout -k 1 6 parallel --tmpdir /dev/shm/parallel head -c 2000m '<'{} >/dev/null ::: /dev/zero &
|
||||||
|
seq 1 20 | parallel -j1 "df /dev/shm | parallel -k --colsep ' +' echo {4}|tail -n 1;sleep 1" \
|
||||||
|
| stdout timeout -k 1 10 perl -ne 'BEGIN{$a=<>} $b=<>; if ($a-1000 > $b) { die "More than 1 MB gone. Good!" }'
|
||||||
|
wait
|
||||||
|
sleep 1
|
||||||
10
testsuite/tests-to-run/test44.sh
Normal file
10
testsuite/tests-to-run/test44.sh
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Test --nice
|
||||||
|
echo '### Test --nice locally'
|
||||||
|
parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b
|
||||||
|
|
||||||
|
echo '### Test --nice remote'
|
||||||
|
stdout parallel --nice 1 -S .. -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b \
|
||||||
|
| perl -pe 's/\S*parallel-server\S*/one-server/'
|
||||||
|
|
||||||
12
testsuite/tests-to-run/test45.sh
Normal file
12
testsuite/tests-to-run/test45.sh
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
SERVER1=parallel-server3
|
||||||
|
SERVER2=parallel-server2
|
||||||
|
|
||||||
|
echo '### Test --load locally'
|
||||||
|
seq 1 300 | nice timeout -k 1 14 parallel -j0 burnP6
|
||||||
|
stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||||
|
|
||||||
|
echo '### Test --load remote'
|
||||||
|
seq 1 300 | ssh parallel@$SERVER2 nice timeout -k 1 14 parallel -j0 burnP6
|
||||||
|
stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
|
||||||
19
testsuite/tests-to-run/test46.sh
Normal file
19
testsuite/tests-to-run/test46.sh
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
SERVER1=parallel-server3
|
||||||
|
SERVER2=parallel-server2
|
||||||
|
|
||||||
|
echo '### Test --trc with space added in filename'
|
||||||
|
echo original > '/tmp/parallel space file'
|
||||||
|
echo '/tmp/parallel space file' | parallel --trc "{} more space" -S $SERVER1 cat {} ">{}\\ more\\ space"
|
||||||
|
cat '/tmp/parallel space file more space'
|
||||||
|
|
||||||
|
echo '### Test --trc with >|< added in filename'
|
||||||
|
echo original > '/tmp/parallel space file'
|
||||||
|
echo '/tmp/parallel space file' | parallel --trc "{} >|<" -S $SERVER1 cat {} ">{}\\ \\>\\|\\<"
|
||||||
|
cat '/tmp/parallel space file >|<'
|
||||||
|
|
||||||
|
echo '### Test --return with fixed string (Gave undef warnings)'
|
||||||
|
touch a
|
||||||
|
echo a | stdout parallel --return b -S .. echo ">b" && echo OK
|
||||||
|
rm a b
|
||||||
170
testsuite/tests-to-run/test47.sh
Normal file
170
testsuite/tests-to-run/test47.sh
Normal file
|
|
@ -0,0 +1,170 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
SERVER1=parallel-server3
|
||||||
|
SERVER2=parallel-server2
|
||||||
|
|
||||||
|
# Minimal version of test17
|
||||||
|
|
||||||
|
# Make sure sort order is the same
|
||||||
|
export LANG=C
|
||||||
|
|
||||||
|
echo '### Test --transfer --return --cleanup'
|
||||||
|
|
||||||
|
rm -rf /tmp/parallel.file*
|
||||||
|
stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*'
|
||||||
|
stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*'
|
||||||
|
(seq 1 2) >/tmp/test17
|
||||||
|
echo '# Create some weirdly files in /tmp'
|
||||||
|
mkdir -p /tmp/parallel.file
|
||||||
|
cat /tmp/test17 | parallel -k /bin/echo file{} '>'/tmp/parallel.file{}.file
|
||||||
|
cat /tmp/test17 | parallel -k /bin/echo /tmp/parallel.file{}.file >/tmp/test17abs
|
||||||
|
cat /tmp/test17 | parallel -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel
|
||||||
|
|
||||||
|
echo '### --transfer - abspath'
|
||||||
|
stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*'
|
||||||
|
stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*'
|
||||||
|
cat /tmp/test17abs | parallel -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {}
|
||||||
|
# One of these should give the empty dir /tmp/parallel.file
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls '/tmp/parallel.file*'
|
||||||
|
# The other: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*'
|
||||||
|
|
||||||
|
echo '### --transfer - relpath'
|
||||||
|
stdout ssh $SERVER1 'rm -rf tmp/parallel.file*'
|
||||||
|
stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*'
|
||||||
|
cd /
|
||||||
|
cat /tmp/test17rel | parallel -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {}
|
||||||
|
# Should give: No such file or directory
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*'
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*'
|
||||||
|
|
||||||
|
echo '### --transfer --cleanup - abspath'
|
||||||
|
cat /tmp/test17abs | parallel -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {}
|
||||||
|
echo good if no file
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh $SERVER1 ls '/tmp/parallel.file*'
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*'
|
||||||
|
|
||||||
|
echo '### --transfer --cleanup - relpath'
|
||||||
|
cat /tmp/test17rel | parallel -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {}
|
||||||
|
# Should give: No such file or directory
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --return - abspath'
|
||||||
|
rm -rf /tmp/parallel.file*out
|
||||||
|
cat /tmp/test17abs | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 echo {} ">"{.}.out
|
||||||
|
ls /tmp/parallel.file*out
|
||||||
|
|
||||||
|
echo '### --return - relpath'
|
||||||
|
rm -rf /tmp/parallel.file*out
|
||||||
|
cat /tmp/test17rel | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out
|
||||||
|
ls tmp/parallel.file*out
|
||||||
|
|
||||||
|
echo '### --return - multiple files'
|
||||||
|
rm -rf tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
|
||||||
|
ls tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
|
||||||
|
echo '### --return --cleanup - abspath'
|
||||||
|
rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
cat /tmp/test17abs | parallel -k --return {.}.out --return {}.done --cleanup \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --return --cleanup - relpath'
|
||||||
|
rm -rf tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
|
||||||
|
ls tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --return --cleanup - multiple returns'
|
||||||
|
rm -rf tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --transfer --return --cleanup - abspath'
|
||||||
|
rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
cat /tmp/test17abs | parallel -k --transfer --return {.}.out --return {}.done --cleanup \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
|
||||||
|
echo '### --transfer --return --cleanup - relpath'
|
||||||
|
rm -rf tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --transfer --return --cleanup - multiple files'
|
||||||
|
rm -rf tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --trc - abspath'
|
||||||
|
rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --trc - relpath'
|
||||||
|
rm -rf tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
cat /tmp/test17rel | parallel -k --trc {.}.out --trc {}.done \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
|
||||||
|
ls tmp/parallel.file*out tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --trc - multiple files'
|
||||||
|
rm -rf /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \
|
||||||
|
--sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';'
|
||||||
|
ls /tmp/parallel.file*out /tmp/parallel.file*done
|
||||||
|
echo good if no file
|
||||||
|
stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
|
echo '### --transfer --cleanup - multiple argument files'
|
||||||
|
parallel -kv --transfer --cleanup -Sparallel@$SERVER2 cat {2} {1} :::: /tmp/test17rel <(sort -r /tmp/test17abs)
|
||||||
|
# Should give: No such file or directory
|
||||||
|
stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK
|
||||||
|
|
||||||
2
testsuite/wanted-results/niceload01
Normal file
2
testsuite/wanted-results/niceload01
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
### Test niceload
|
||||||
|
.........
|
||||||
|
|
@ -85,7 +85,7 @@ parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./
|
||||||
'newline2.out
|
'newline2.out
|
||||||
parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ././tmp/parallel.file.'
|
parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ././tmp/parallel.file.'
|
||||||
'newline2.out2
|
'newline2.out2
|
||||||
parallel-server3 rm -f tmp/parallel.file.'
|
parallel-server3 rm -f ./tmp/parallel.file.'
|
||||||
'newline2; rmdir 2>/dev/null ./tmp
|
'newline2; rmdir 2>/dev/null ./tmp
|
||||||
-l parallel parallel-server2 rsync --server -lDErRze.iLsf . .
|
-l parallel parallel-server2 rsync --server -lDErRze.iLsf . .
|
||||||
parallel@parallel-server2 PARALLEL_SEQ=1;export PARALLEL_SEQ;PARALLEL_PID=00000;export PARALLEL_PID; cat tmp/parallel.file.'
|
parallel@parallel-server2 PARALLEL_SEQ=1;export PARALLEL_SEQ;PARALLEL_PID=00000;export PARALLEL_PID; cat tmp/parallel.file.'
|
||||||
|
|
@ -97,7 +97,7 @@ parallel@parallel-server2 PARALLEL_SEQ=1;export PARALLEL_SEQ;PARALLEL_PID=00000;
|
||||||
'newline1.out
|
'newline1.out
|
||||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ././tmp/parallel.file.'
|
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ././tmp/parallel.file.'
|
||||||
'newline1.out2
|
'newline1.out2
|
||||||
parallel@parallel-server2 rm -f tmp/parallel.file.'
|
parallel@parallel-server2 rm -f ./tmp/parallel.file.'
|
||||||
'newline1; rmdir 2>/dev/null ./tmp
|
'newline1; rmdir 2>/dev/null ./tmp
|
||||||
### Test use special ssh with > 9 simultaneous
|
### Test use special ssh with > 9 simultaneous
|
||||||
1
|
1
|
||||||
|
|
|
||||||
|
|
@ -11,9 +11,9 @@ sleep 2;false
|
||||||
1
|
1
|
||||||
sh: non_exist: command not found
|
sh: non_exist: command not found
|
||||||
2
|
2
|
||||||
|
sh: non_exist: command not found
|
||||||
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
|
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
|
||||||
sleep 2;false
|
sleep 2;false
|
||||||
sh: non_exist: command not found
|
|
||||||
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
|
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
|
||||||
sleep 4; non_exist
|
sleep 4; non_exist
|
||||||
127
|
127
|
||||||
|
|
@ -23,31 +23,31 @@ sleep 2;false
|
||||||
### Test last dying print --halt-on-error
|
### Test last dying print --halt-on-error
|
||||||
0
|
0
|
||||||
1
|
1
|
||||||
parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
|
|
||||||
2
|
2
|
||||||
parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2
|
|
||||||
3
|
3
|
||||||
parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
|
|
||||||
4
|
4
|
||||||
parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4
|
|
||||||
5
|
5
|
||||||
parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5
|
|
||||||
6
|
6
|
||||||
parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6
|
|
||||||
7
|
7
|
||||||
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
|
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
|
|
||||||
8
|
8
|
||||||
0
|
0
|
||||||
|
9
|
||||||
|
parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
|
||||||
|
parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2
|
||||||
|
parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
|
||||||
|
parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4
|
||||||
|
parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5
|
||||||
|
parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6
|
||||||
|
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
|
||||||
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
|
||||||
parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed:
|
parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed:
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8
|
||||||
9
|
|
||||||
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
|
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
|
||||||
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9
|
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9
|
||||||
9
|
9
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,26 @@
|
||||||
### Test of --retries
|
### Test of --retries - it should run 13 jobs in total
|
||||||
|
13
|
||||||
|
### Test of --retries - it should run 25 jobs in total
|
||||||
|
25
|
||||||
|
### Test of --retries - it should run 49 jobs in total
|
||||||
|
49
|
||||||
|
### Bug with --retries
|
||||||
|
8
|
||||||
|
8
|
||||||
|
2
|
||||||
|
1
|
||||||
|
1
|
||||||
|
1
|
||||||
|
1
|
||||||
|
### These were not affected by the bug
|
||||||
|
8
|
||||||
|
8
|
||||||
|
1
|
||||||
|
4
|
||||||
|
4
|
||||||
|
4
|
||||||
|
2
|
||||||
|
### Test of --retries on unreachable host
|
||||||
echo 1
|
echo 1
|
||||||
1
|
1
|
||||||
echo 2
|
echo 2
|
||||||
|
|
@ -19,9 +41,3 @@ echo 9
|
||||||
9
|
9
|
||||||
echo 10
|
echo 10
|
||||||
10
|
10
|
||||||
### Test of --retries - it should run 13 jobs in total
|
|
||||||
13
|
|
||||||
### Test of --retries - it should run 25 jobs in total
|
|
||||||
25
|
|
||||||
### Test of --retries - it should run 49 jobs in total
|
|
||||||
49
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
### Test of -j filename
|
### Test of -j filename
|
||||||
Parsing of --jobs/-j/--max-procs/-P failed
|
|
||||||
Usage:
|
Usage:
|
||||||
parallel [options] [command [arguments]] < list_of_arguments
|
parallel [options] [command [arguments]] < list_of_arguments
|
||||||
parallel [options] [command [arguments]] ::: arguments
|
parallel [options] [command [arguments]] ::: arguments
|
||||||
parallel [options] [command [arguments]] :::: argfile(s)
|
parallel [options] [command [arguments]] :::: argfile(s)
|
||||||
|
|
||||||
See 'man parallel' for the options
|
See 'man parallel' for the options
|
||||||
|
Parsing of --jobs/-j/--max-procs/-P failed
|
||||||
### Test of -j filename
|
### Test of -j filename
|
||||||
sleep 0.7
|
sleep 0.7
|
||||||
sleep 0.8
|
sleep 0.8
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
### Test -M
|
### Test -M (--retries to avoid false errors)
|
||||||
1
|
1
|
||||||
2
|
2
|
||||||
3
|
3
|
||||||
|
|
@ -19,3 +19,43 @@
|
||||||
18
|
18
|
||||||
19
|
19
|
||||||
20
|
20
|
||||||
|
21
|
||||||
|
22
|
||||||
|
23
|
||||||
|
24
|
||||||
|
25
|
||||||
|
26
|
||||||
|
27
|
||||||
|
28
|
||||||
|
29
|
||||||
|
30
|
||||||
|
1
|
||||||
|
2
|
||||||
|
3
|
||||||
|
4
|
||||||
|
5
|
||||||
|
6
|
||||||
|
7
|
||||||
|
8
|
||||||
|
9
|
||||||
|
10
|
||||||
|
11
|
||||||
|
12
|
||||||
|
13
|
||||||
|
14
|
||||||
|
15
|
||||||
|
16
|
||||||
|
17
|
||||||
|
18
|
||||||
|
19
|
||||||
|
20
|
||||||
|
21
|
||||||
|
22
|
||||||
|
23
|
||||||
|
24
|
||||||
|
25
|
||||||
|
26
|
||||||
|
27
|
||||||
|
28
|
||||||
|
29
|
||||||
|
30
|
||||||
|
|
|
||||||
56
testsuite/wanted-results/test42
Normal file
56
testsuite/wanted-results/test42
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
### Test --basenamereplace
|
||||||
|
b.c b.c b.c b b b
|
||||||
|
b.c
|
||||||
|
b.c
|
||||||
|
b.c
|
||||||
|
b
|
||||||
|
b
|
||||||
|
b
|
||||||
|
### Test --basenameextensionreplace
|
||||||
|
b b b b b b
|
||||||
|
b
|
||||||
|
b
|
||||||
|
b
|
||||||
|
b
|
||||||
|
b
|
||||||
|
b
|
||||||
|
### Test {/}
|
||||||
|
b.c b.c b.c b b b
|
||||||
|
### Test {/.}
|
||||||
|
b b b b b b
|
||||||
|
### Test {#/.}
|
||||||
|
number2
|
||||||
|
### Test {#/}
|
||||||
|
number2.c
|
||||||
|
### Test {#.}
|
||||||
|
a/number2
|
||||||
|
### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
|
||||||
|
Warning: using -X or -m with --sshlogin may fail
|
||||||
|
mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 tmp/number\ 7_number\ 7_number2_number2.c_a/number2 tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file tmp/number\ 7_number\ 7_number2_number2.c_a/number2/file tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2/file
|
||||||
|
tmp
|
||||||
|
tmp/number6_number6_number2_number2.c_a
|
||||||
|
tmp/number6_number6_number2_number2.c_a/number2
|
||||||
|
tmp/number6_number6_number2_number2.c_a/number2/file
|
||||||
|
tmp/number <8|8>_number <8|8>_number2_number2.c_a
|
||||||
|
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2
|
||||||
|
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2/file
|
||||||
|
tmp/number3.c_number3_number2_number2.c_a
|
||||||
|
tmp/number3.c_number3_number2_number2.c_a/number2
|
||||||
|
tmp/number3.c_number3_number2_number2.c_a/number2/file
|
||||||
|
tmp/number5_number5_number2_number2.c_a
|
||||||
|
tmp/number5_number5_number2_number2.c_a/number2
|
||||||
|
tmp/number5_number5_number2_number2.c_a/number2/file
|
||||||
|
tmp/number2.c_number2_number2_number2.c_a
|
||||||
|
tmp/number2.c_number2_number2_number2.c_a/number2
|
||||||
|
tmp/number2.c_number2_number2_number2.c_a/number2/file
|
||||||
|
tmp/number1.c_number1_number2_number2.c_a
|
||||||
|
tmp/number1.c_number1_number2_number2.c_a/number2
|
||||||
|
tmp/number1.c_number1_number2_number2.c_a/number2/file
|
||||||
|
tmp/number4_number4_number2_number2.c_a
|
||||||
|
tmp/number4_number4_number2_number2.c_a/number2
|
||||||
|
tmp/number4_number4_number2_number2.c_a/number2/file
|
||||||
|
tmp/number 7_number 7_number2_number2.c_a
|
||||||
|
tmp/number 7_number 7_number2_number2.c_a/number2
|
||||||
|
tmp/number 7_number 7_number2_number2.c_a/number2/file
|
||||||
|
### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
|
||||||
|
### But we will wait for a real world scenario
|
||||||
6
testsuite/wanted-results/test43
Normal file
6
testsuite/wanted-results/test43
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
### Test $TMPDIR
|
||||||
|
More than 1 MB gone. Good! at -e line 1, <> line 3.
|
||||||
|
### Test --tmpdir
|
||||||
|
More than 1 MB gone. Good! at -e line 1, <> line 3.
|
||||||
|
### Test $TMPDIR and --tmpdir
|
||||||
|
More than 1 MB gone. Good! at -e line 1, <> line 3.
|
||||||
6
testsuite/wanted-results/test44
Normal file
6
testsuite/wanted-results/test44
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
### Test --nice locally
|
||||||
|
nice -n1 bash -c PAR=a\ bash\ -c\ \"echo\ \ \\\$PAR\ b\"
|
||||||
|
a b
|
||||||
|
### Test --nice remote
|
||||||
|
ssh one-server PARALLEL_SEQ=$PARALLEL_SEQ\;export PARALLEL_SEQ\;PARALLEL_PID=$PARALLEL_PID\;export PARALLEL_PID\; nice\ -n1\ bash\ -c\ PAR=a\\\ bash\\\ -c\\\ \\\"echo\\\ \\\ \\\\\\\$PAR\\\ b\\\";
|
||||||
|
a b
|
||||||
4
testsuite/wanted-results/test45
Normal file
4
testsuite/wanted-results/test45
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
### Test --load locally
|
||||||
|
OK
|
||||||
|
### Test --load remote
|
||||||
|
OK
|
||||||
6
testsuite/wanted-results/test46
Normal file
6
testsuite/wanted-results/test46
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
### Test --trc with space added in filename
|
||||||
|
original
|
||||||
|
### Test --trc with >|< added in filename
|
||||||
|
original
|
||||||
|
### Test --return with fixed string (Gave undef warnings)
|
||||||
|
OK
|
||||||
127
testsuite/wanted-results/test47
Normal file
127
testsuite/wanted-results/test47
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
### Test --transfer --return --cleanup
|
||||||
|
# Create some weirdly files in /tmp
|
||||||
|
### --transfer - abspath
|
||||||
|
file1
|
||||||
|
file2
|
||||||
|
good if no file
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
### --transfer - relpath
|
||||||
|
file1
|
||||||
|
file2
|
||||||
|
good if no file
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
### --transfer --cleanup - abspath
|
||||||
|
file1
|
||||||
|
file2
|
||||||
|
good if no file
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
### --transfer --cleanup - relpath
|
||||||
|
file1
|
||||||
|
file2
|
||||||
|
good if no file
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --return - abspath
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
### --return - relpath
|
||||||
|
tmp/parallel.file1.out
|
||||||
|
tmp/parallel.file2.out
|
||||||
|
### --return - multiple files
|
||||||
|
tmp/parallel.file1.file.done
|
||||||
|
tmp/parallel.file1.out
|
||||||
|
tmp/parallel.file2.file.done
|
||||||
|
tmp/parallel.file2.out
|
||||||
|
### --return --cleanup - abspath
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --return --cleanup - relpath
|
||||||
|
tmp/parallel.file1.file.done
|
||||||
|
tmp/parallel.file1.out
|
||||||
|
tmp/parallel.file2.file.done
|
||||||
|
tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --return --cleanup - multiple returns
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --transfer --return --cleanup - abspath
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --transfer --return --cleanup - relpath
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --transfer --return --cleanup - multiple files
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --trc - abspath
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --trc - relpath
|
||||||
|
tmp/parallel.file1.file.done
|
||||||
|
tmp/parallel.file1.out
|
||||||
|
tmp/parallel.file2.file.done
|
||||||
|
tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --trc - multiple files
|
||||||
|
/tmp/parallel.file1.file.done
|
||||||
|
/tmp/parallel.file1.out
|
||||||
|
/tmp/parallel.file2.file.done
|
||||||
|
/tmp/parallel.file2.out
|
||||||
|
good if no file
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
|
### --transfer --cleanup - multiple argument files
|
||||||
|
cat /tmp/parallel.file2.file tmp/parallel.file1.file
|
||||||
|
file2
|
||||||
|
file1
|
||||||
|
cat /tmp/parallel.file1.file tmp/parallel.file2.file
|
||||||
|
file1
|
||||||
|
file2
|
||||||
|
ls: cannot access /tmp/parallel.file*: No such file or directory
|
||||||
|
OK
|
||||||
Loading…
Reference in a new issue