mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 20:57:53 +00:00
Code cleanup. POD file for mini man page for sem
This commit is contained in:
Ole Tange
parent
07a2682099
commit
47cf0f0128
|
|
@ -1,6 +1,6 @@
|
|||
@%:@! /bin/sh
|
||||
@%:@ Guess values for system-dependent variables and create Makefiles.
|
||||
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722.
|
||||
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100817.
|
||||
@%:@
|
||||
@%:@ Report bugs to <bug-parallel@gnu.org>.
|
||||
@%:@
|
||||
|
|
@ -551,8 +551,8 @@ MAKEFLAGS=
|
|||
# Identity of this package.
|
||||
PACKAGE_NAME='parallel'
|
||||
PACKAGE_TARNAME='parallel'
|
||||
PACKAGE_VERSION='20100722'
|
||||
PACKAGE_STRING='parallel 20100722'
|
||||
PACKAGE_VERSION='20100817'
|
||||
PACKAGE_STRING='parallel 20100817'
|
||||
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
||||
PACKAGE_URL=''
|
||||
|
||||
|
|
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
|
|||
# Omit some internal or obsolete options to make the list less imposing.
|
||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||
cat <<_ACEOF
|
||||
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
|
||||
\`configure' configures parallel 20100817 to adapt to many kinds of systems.
|
||||
|
||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||
|
||||
|
|
@ -1233,7 +1233,7 @@ fi
|
|||
|
||||
if test -n "$ac_init_help"; then
|
||||
case $ac_init_help in
|
||||
short | recursive ) echo "Configuration of parallel 20100722:";;
|
||||
short | recursive ) echo "Configuration of parallel 20100817:";;
|
||||
esac
|
||||
cat <<\_ACEOF
|
||||
|
||||
|
|
@ -1300,7 +1300,7 @@ fi
|
|||
test -n "$ac_init_help" && exit $ac_status
|
||||
if $ac_init_version; then
|
||||
cat <<\_ACEOF
|
||||
parallel configure 20100722
|
||||
parallel configure 20100817
|
||||
generated by GNU Autoconf 2.65
|
||||
|
||||
Copyright (C) 2009 Free Software Foundation, Inc.
|
||||
|
|
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
|
|||
This file contains any messages produced by compilers while
|
||||
running configure, to aid debugging if configure makes a mistake.
|
||||
|
||||
It was created by parallel $as_me 20100722, which was
|
||||
It was created by parallel $as_me 20100817, which was
|
||||
generated by GNU Autoconf 2.65. Invocation command line was
|
||||
|
||||
$ $0 $@
|
||||
|
|
@ -2125,7 +2125,7 @@ fi
|
|||
|
||||
# Define the identity of the package.
|
||||
PACKAGE='parallel'
|
||||
VERSION='20100722'
|
||||
VERSION='20100817'
|
||||
|
||||
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
|
|
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
|||
# report actual input values of CONFIG_FILES etc. instead of their
|
||||
# values after options handling.
|
||||
ac_log="
|
||||
This file was extended by parallel $as_me 20100722, which was
|
||||
This file was extended by parallel $as_me 20100817, which was
|
||||
generated by GNU Autoconf 2.65. Invocation command line was
|
||||
|
||||
CONFIG_FILES = $CONFIG_FILES
|
||||
|
|
@ -2737,7 +2737,7 @@ _ACEOF
|
|||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||
ac_cs_version="\\
|
||||
parallel config.status 20100722
|
||||
parallel config.status 20100817
|
||||
configured by $0, generated by GNU Autoconf 2.65,
|
||||
with options \\"\$ac_cs_config\\"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
@%:@! /bin/sh
|
||||
@%:@ Guess values for system-dependent variables and create Makefiles.
|
||||
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722.
|
||||
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100817.
|
||||
@%:@
|
||||
@%:@ Report bugs to <bug-parallel@gnu.org>.
|
||||
@%:@
|
||||
|
|
@ -551,8 +551,8 @@ MAKEFLAGS=
|
|||
# Identity of this package.
|
||||
PACKAGE_NAME='parallel'
|
||||
PACKAGE_TARNAME='parallel'
|
||||
PACKAGE_VERSION='20100722'
|
||||
PACKAGE_STRING='parallel 20100722'
|
||||
PACKAGE_VERSION='20100817'
|
||||
PACKAGE_STRING='parallel 20100817'
|
||||
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
||||
PACKAGE_URL=''
|
||||
|
||||
|
|
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
|
|||
# Omit some internal or obsolete options to make the list less imposing.
|
||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||
cat <<_ACEOF
|
||||
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
|
||||
\`configure' configures parallel 20100817 to adapt to many kinds of systems.
|
||||
|
||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||
|
||||
|
|
@ -1233,7 +1233,7 @@ fi
|
|||
|
||||
if test -n "$ac_init_help"; then
|
||||
case $ac_init_help in
|
||||
short | recursive ) echo "Configuration of parallel 20100722:";;
|
||||
short | recursive ) echo "Configuration of parallel 20100817:";;
|
||||
esac
|
||||
cat <<\_ACEOF
|
||||
|
||||
|
|
@ -1300,7 +1300,7 @@ fi
|
|||
test -n "$ac_init_help" && exit $ac_status
|
||||
if $ac_init_version; then
|
||||
cat <<\_ACEOF
|
||||
parallel configure 20100722
|
||||
parallel configure 20100817
|
||||
generated by GNU Autoconf 2.65
|
||||
|
||||
Copyright (C) 2009 Free Software Foundation, Inc.
|
||||
|
|
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
|
|||
This file contains any messages produced by compilers while
|
||||
running configure, to aid debugging if configure makes a mistake.
|
||||
|
||||
It was created by parallel $as_me 20100722, which was
|
||||
It was created by parallel $as_me 20100817, which was
|
||||
generated by GNU Autoconf 2.65. Invocation command line was
|
||||
|
||||
$ $0 $@
|
||||
|
|
@ -2125,7 +2125,7 @@ fi
|
|||
|
||||
# Define the identity of the package.
|
||||
PACKAGE='parallel'
|
||||
VERSION='20100722'
|
||||
VERSION='20100817'
|
||||
|
||||
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
|
|
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
|||
# report actual input values of CONFIG_FILES etc. instead of their
|
||||
# values after options handling.
|
||||
ac_log="
|
||||
This file was extended by parallel $as_me 20100722, which was
|
||||
This file was extended by parallel $as_me 20100817, which was
|
||||
generated by GNU Autoconf 2.65. Invocation command line was
|
||||
|
||||
CONFIG_FILES = $CONFIG_FILES
|
||||
|
|
@ -2737,7 +2737,7 @@ _ACEOF
|
|||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||
ac_cs_version="\\
|
||||
parallel config.status 20100722
|
||||
parallel config.status 20100817
|
||||
configured by $0, generated by GNU Autoconf 2.65,
|
||||
with options \\"\$ac_cs_config\\"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100722], [bug-parallel@gnu.org])
|
||||
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100817], [bug-parallel@gnu.org])
|
||||
m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_])
|
||||
m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_])
|
||||
m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS'])
|
||||
|
|
|
|||
20
configure
vendored
20
configure
vendored
|
|
@ -1,6 +1,6 @@
|
|||
#! /bin/sh
|
||||
# Guess values for system-dependent variables and create Makefiles.
|
||||
# Generated by GNU Autoconf 2.65 for parallel 20100722.
|
||||
# Generated by GNU Autoconf 2.65 for parallel 20100817.
|
||||
#
|
||||
# Report bugs to <bug-parallel@gnu.org>.
|
||||
#
|
||||
|
|
@ -551,8 +551,8 @@ MAKEFLAGS=
|
|||
# Identity of this package.
|
||||
PACKAGE_NAME='parallel'
|
||||
PACKAGE_TARNAME='parallel'
|
||||
PACKAGE_VERSION='20100722'
|
||||
PACKAGE_STRING='parallel 20100722'
|
||||
PACKAGE_VERSION='20100817'
|
||||
PACKAGE_STRING='parallel 20100817'
|
||||
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
||||
PACKAGE_URL=''
|
||||
|
||||
|
|
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
|
|||
# Omit some internal or obsolete options to make the list less imposing.
|
||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||
cat <<_ACEOF
|
||||
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
|
||||
\`configure' configures parallel 20100817 to adapt to many kinds of systems.
|
||||
|
||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||
|
||||
|
|
@ -1233,7 +1233,7 @@ fi
|
|||
|
||||
if test -n "$ac_init_help"; then
|
||||
case $ac_init_help in
|
||||
short | recursive ) echo "Configuration of parallel 20100722:";;
|
||||
short | recursive ) echo "Configuration of parallel 20100817:";;
|
||||
esac
|
||||
cat <<\_ACEOF
|
||||
|
||||
|
|
@ -1300,7 +1300,7 @@ fi
|
|||
test -n "$ac_init_help" && exit $ac_status
|
||||
if $ac_init_version; then
|
||||
cat <<\_ACEOF
|
||||
parallel configure 20100722
|
||||
parallel configure 20100817
|
||||
generated by GNU Autoconf 2.65
|
||||
|
||||
Copyright (C) 2009 Free Software Foundation, Inc.
|
||||
|
|
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
|
|||
This file contains any messages produced by compilers while
|
||||
running configure, to aid debugging if configure makes a mistake.
|
||||
|
||||
It was created by parallel $as_me 20100722, which was
|
||||
It was created by parallel $as_me 20100817, which was
|
||||
generated by GNU Autoconf 2.65. Invocation command line was
|
||||
|
||||
$ $0 $@
|
||||
|
|
@ -2125,7 +2125,7 @@ fi
|
|||
|
||||
# Define the identity of the package.
|
||||
PACKAGE='parallel'
|
||||
VERSION='20100722'
|
||||
VERSION='20100817'
|
||||
|
||||
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
|
|
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
|||
# report actual input values of CONFIG_FILES etc. instead of their
|
||||
# values after options handling.
|
||||
ac_log="
|
||||
This file was extended by parallel $as_me 20100722, which was
|
||||
This file was extended by parallel $as_me 20100817, which was
|
||||
generated by GNU Autoconf 2.65. Invocation command line was
|
||||
|
||||
CONFIG_FILES = $CONFIG_FILES
|
||||
|
|
@ -2737,7 +2737,7 @@ _ACEOF
|
|||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||
ac_cs_version="\\
|
||||
parallel config.status 20100722
|
||||
parallel config.status 20100817
|
||||
configured by $0, generated by GNU Autoconf 2.65,
|
||||
with options \\"\$ac_cs_config\\"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
AC_INIT([parallel], [20100722], [bug-parallel@gnu.org])
|
||||
AC_INIT([parallel], [20100817], [bug-parallel@gnu.org])
|
||||
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
|
||||
AC_CONFIG_HEADERS([config.h])
|
||||
AC_CONFIG_FILES([
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
# sem -j+0
|
||||
# sem gzip foo ";" echo done
|
||||
# Examples of sem
|
||||
|
||||
Weird bug - only interactive
|
||||
|
||||
echo '### BUG: Test --fg followed by --bg'
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
|
||||
sem --wait
|
||||
|
||||
find . -execdir sem -j100 'sleep 4; echo {}' \; ; sem --wait
|
||||
|
||||
# Allow 7 to run. After then 7th is started, block untill one is dead
|
||||
parallel --semaphore --id uniqidentifier -j7 command
|
||||
parallel --semaphore -j7 command
|
||||
mdm.screen find dir -execdir mdm-run cmd {} \;
|
||||
find dir -execdir parallel --semaphore cmd {} \;
|
||||
|
||||
fex syntax for splitting fields
|
||||
http://www.semicomplete.com/projects/fex/
|
||||
|
|
|
|||
|
|
@ -79,7 +79,9 @@ Newsgroups: comp.unix.shell,comp.unix.admin
|
|||
<<<<<
|
||||
to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org
|
||||
cc:Peter Simons <simons@cryp.to>, Sandro Cazzaniga <kharec@mandriva.org>,
|
||||
Tim Cuthbertson <tim3d.junk@gmail.com>
|
||||
Tim Cuthbertson <tim3d.junk@gmail.com>, Ludovic Courtès <ludo@gnu.org>,
|
||||
Markus Ammer <mkmm@gmx-topmail.de>, Pavel Nuzhdin <pnzhdin@gmail.com>,
|
||||
((psung@alum.mit.edu))
|
||||
|
||||
Subject: GNU Parallel 20100722 released
|
||||
|
||||
|
|
@ -88,12 +90,15 @@ download at: http://ftp.gnu.org/gnu/parallel/
|
|||
|
||||
New in this release:
|
||||
|
||||
* Counting semaphore functionality: start a job in the background. If
|
||||
N jobs are already running, wait for one to complete.
|
||||
|
||||
* With --colsep a table can be used as input. Example:
|
||||
cat tab_sep_table | parallel --colsep '\t' echo col1 {1} col2 {2}
|
||||
|
||||
* --trim can remove white space around arguments.
|
||||
|
||||
* NixOS package. Thanks to Ludovic Courtès <ludo at gnu dot org>
|
||||
* --sshloginfile '..' means use ~/.parallel/sshloginfile
|
||||
|
||||
* Zero install package. Thanks to Tim Cuthbertson <tim3d dot junk at
|
||||
gmail dot com>
|
||||
|
|
@ -101,11 +106,13 @@ New in this release:
|
|||
* OpenSUSE package. Thanks to Markus Ammer <mkmm at gmx-topmail dot
|
||||
de>
|
||||
|
||||
* NixOS package. Thanks to Ludovic Courtès <ludo at gnu dot org>
|
||||
|
||||
* Web review http://oentend.blogspot.com/2010/08/gnu-parallel.html
|
||||
Thanks to Pavel Nuzhdin <pnzhdin at gmail dot com>
|
||||
|
||||
* Web review http://psung.blogspot.com/2010/08/gnu-parallel.html
|
||||
Thanks to Phil Sung
|
||||
Thanks to Phil Sung ((<psung at alum dot mit dot edu>))
|
||||
|
||||
|
||||
= About GNU Parallel =
|
||||
|
|
|
|||
72
src/parallel
72
src/parallel
|
|
@ -184,7 +184,7 @@ Multiple B<-B> can be specified to transfer more basefiles. The
|
|||
I<file> will be transferred the same way as B<--transfer>.
|
||||
|
||||
|
||||
=item B<--bg> (not implemented)
|
||||
=item B<--bg> (beta testing)
|
||||
|
||||
Run command in background thus GNU B<parallel> will not wait for
|
||||
completion of the command before exiting. This is the default if
|
||||
|
|
@ -281,7 +281,7 @@ jobs. GNU B<parallel> normally only reads the next job to run.
|
|||
Implies B<--progress>.
|
||||
|
||||
|
||||
=item B<--fg> (not implemented)
|
||||
=item B<--fg> (beta testing)
|
||||
|
||||
Run command in foreground thus GNU B<parallel> will wait for
|
||||
completion of the command before exiting.
|
||||
|
|
@ -606,7 +606,7 @@ operating system and the B<-s> option. Pipe the input from /dev/null
|
|||
to do anything.
|
||||
|
||||
|
||||
=item B<--semaphore> (not implemented)
|
||||
=item B<--semaphore> (beta testing)
|
||||
|
||||
Work as a counting semaphore. B<--semaphore> will cause GNU
|
||||
B<parallel> to start I<command> in the background. When the number of
|
||||
|
|
@ -623,9 +623,9 @@ Used with B<--fg>, B<--wait>, and B<--semaphorename>.
|
|||
The command B<sem> is an alias for B<parallel --semaphore>.
|
||||
|
||||
|
||||
=item B<--semaphorename> I<name> (not implemented)
|
||||
=item B<--semaphorename> I<name> (beta testing)
|
||||
|
||||
=item B<--id> I<name> (not implemented)
|
||||
=item B<--id> I<name>
|
||||
|
||||
The name of the semaphore to use. The semaphore can be shared between
|
||||
multiple processes.
|
||||
|
|
@ -817,7 +817,7 @@ B<--silent>. See also B<-t>.
|
|||
Print the version GNU B<parallel> and exit.
|
||||
|
||||
|
||||
=item B<--wait> (not implemented)
|
||||
=item B<--wait> (beta testing)
|
||||
|
||||
Wait for all commands to complete.
|
||||
|
||||
|
|
@ -856,6 +856,8 @@ FUBAR in all files in this dir and subdirs:
|
|||
|
||||
B<find . -type f -print0 | parallel -q0 perl -i -pe 's/FOO BAR/FUBAR/g'>
|
||||
|
||||
Note B<-q> is needed because of the space in 'FOO BAR'.
|
||||
|
||||
|
||||
=head1 EXAMPLE: Reading arguments from command line
|
||||
|
||||
|
|
@ -1658,12 +1660,16 @@ B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \
|
|||
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
|
||||
pexec -j -m blockwrite -s th_$IMG'
|
||||
|
||||
B<8> GNU B<parallel> does not support mutexes directly but uses B<mutex> to
|
||||
do that.
|
||||
B<8> Combining GNU B<parallel> and GNU B<sem>.
|
||||
|
||||
B<8> ls *jpg | parallel -j8 'mutex -m blockread cat {} | jpegtopnm |' \
|
||||
'pnmscale 0.5 | pnmtojpeg | mutex -m blockwrite cat > th_{}'
|
||||
B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
|
||||
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
|
||||
|
||||
B<8> If reading and writing is done to the same disk, this may be
|
||||
faster as only one process will be either reading or writing:
|
||||
|
||||
B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
|
||||
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
|
||||
|
||||
=head2 DIFFERENCES BETWEEN xjobs AND GNU Parallel
|
||||
|
||||
|
|
@ -1749,6 +1755,7 @@ B<seq 1 19 | parallel -j+0 buffon -o - | sort -n >>B< result>
|
|||
|
||||
B<cat files | parallel -j+0 cmd>
|
||||
|
||||
B<find dir -execdir sem -j+0 cmd {} \;>
|
||||
|
||||
=head2 DIFFERENCES BETWEEN xapply AND GNU Parallel
|
||||
|
||||
|
|
@ -1823,10 +1830,9 @@ B<cat hostlist | parallel ssh {} do_stuff>
|
|||
|
||||
=head1 BUGS
|
||||
|
||||
Filenames beginning with '-' can cause some commands to give
|
||||
unexpected results, as it will often be interpreted as an option.
|
||||
=head2 Quoting of newline
|
||||
|
||||
Because the way newline is quoted this will not work:
|
||||
Because of the way newline is quoted this will not work:
|
||||
|
||||
echo 1,2,3 | parallel -vkd, "echo 'a{}'"
|
||||
|
||||
|
|
@ -1834,6 +1840,11 @@ However, this will work:
|
|||
|
||||
echo 1,2,3 | parallel -vkd, echo a{}
|
||||
|
||||
=head2 Startup speed
|
||||
|
||||
GNU B<parallel> is slow at starting up. Half of the startup time is
|
||||
spent finding the maximal length of a command line. Setting B<-s> will
|
||||
remove this part of the startup time.
|
||||
|
||||
=head1 REPORTING BUGS
|
||||
|
||||
|
|
@ -1987,16 +1998,16 @@ use strict;
|
|||
do_not_reap();
|
||||
parse_options();
|
||||
init_run_jobs();
|
||||
my $sem;
|
||||
if($Global::semaphore) {
|
||||
run_as_semaphore();
|
||||
} else {
|
||||
start_more_jobs();
|
||||
$sem = acquire_semaphore();
|
||||
}
|
||||
start_more_jobs();
|
||||
reap_if_needed();
|
||||
drain_job_queue();
|
||||
cleanup();
|
||||
if($Global::semaphore) {
|
||||
exit $Global::exitstatus;
|
||||
$sem->release();
|
||||
}
|
||||
if($::opt_halt_on_error) {
|
||||
wait_and_exit($Global::halt_on_error_exitstatus);
|
||||
|
|
@ -2004,39 +2015,37 @@ if($::opt_halt_on_error) {
|
|||
wait_and_exit(min($Global::exitstatus,254));
|
||||
}
|
||||
|
||||
sub run_as_semaphore {
|
||||
sub acquire_semaphore {
|
||||
# Acquires semaphore. If needed: spawns to the background
|
||||
# Returns:
|
||||
# The semaphore to be released when jobs is complete
|
||||
my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
|
||||
$sem->acquire();
|
||||
debug("run");
|
||||
$Global::argfile = open_or_exit("/dev/null");
|
||||
unget_arg("");
|
||||
if($Semaphore::fg) {
|
||||
start_more_jobs();
|
||||
$sem->release();
|
||||
# skip
|
||||
} else {
|
||||
# If run in the background, the PID will change
|
||||
# therefore release and re-acquire the semaphore
|
||||
$sem->release();
|
||||
if(not fork()) {
|
||||
if(fork()) {
|
||||
exit(0);
|
||||
} else {
|
||||
# child
|
||||
# Get a semaphore for this pid
|
||||
my $child_sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
|
||||
$child_sem->acquire();
|
||||
start_more_jobs();
|
||||
reap_if_needed();
|
||||
drain_job_queue();
|
||||
cleanup();
|
||||
$child_sem->release();
|
||||
} else {
|
||||
exit(0);
|
||||
$sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
|
||||
$sem->acquire();
|
||||
}
|
||||
}
|
||||
return $sem;
|
||||
}
|
||||
|
||||
sub parse_options {
|
||||
# Returns: N/A
|
||||
# Defaults:
|
||||
$Global::version = 20100722;
|
||||
$Global::version = 20100817;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::debug = 0;
|
||||
$Global::verbose = 0;
|
||||
|
|
@ -4289,6 +4298,7 @@ sub new {
|
|||
-d $parallel_locks or mkdir $parallel_locks;
|
||||
my $lockdir = "$parallel_locks/$id";
|
||||
my $lockfile = $lockdir.".lock";
|
||||
if($count < 1) { die "Semaphore count = $count"; }
|
||||
return bless {
|
||||
'lockfile' => $lockfile,
|
||||
'lockfh' => Symbol::gensym(),
|
||||
|
|
|
|||
430
src/sem.pod
Executable file
430
src/sem.pod
Executable file
|
|
@ -0,0 +1,430 @@
|
|||
#!/usr/bin/perl -w
|
||||
|
||||
=head1 NAME
|
||||
|
||||
sem - semaphore for executing shell command lines in parallel
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
B<sem> [--fg] [--id <id>] [--timeout <secs>] [-j <num>] [--wait] command
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
GNU B<sem> is an alias for GNU B<parallel --semaphore>.
|
||||
|
||||
It works as a tool for executing shell commands in parallel. GNU
|
||||
B<sem> acts as a counting semaphore. When GNU B<sem> is called with
|
||||
command it will start the command in the background. When I<num>
|
||||
number of commands are running in the background, GNU B<sem> will wait
|
||||
for one of these to complete before starting another command.
|
||||
|
||||
Before looking at the options you may want to check out the examples
|
||||
after the list of options. That will give you an idea of what GNU
|
||||
B<sem> is capable of.
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
=over 9
|
||||
|
||||
=item I<command>
|
||||
|
||||
Command to execute. The command may be followed by arguments for the command.
|
||||
|
||||
|
||||
=item B<--count> I<N>
|
||||
|
||||
=item B<-j> I<N>
|
||||
|
||||
Run up to N commands in parallel. Default is 1 thus acting like a
|
||||
mutex.
|
||||
|
||||
|
||||
=item B<--id> I<id>
|
||||
|
||||
=item B<-i> I<id>
|
||||
|
||||
Use B<id> as the name of the semaphore. Default is the name of the
|
||||
controlling tty (output from B<tty>).
|
||||
|
||||
The default normally works as expected when used interactively, but
|
||||
when used in a script I<id> should be set. $$ is often a good value.
|
||||
|
||||
|
||||
=item B<--fg>
|
||||
|
||||
Do not put command in background.
|
||||
|
||||
|
||||
=item B<--timeout> I<secs> (not implemented)
|
||||
|
||||
=item B<-t> I<secs> (not implemented)
|
||||
|
||||
If the semaphore is not released within I<secs> seconds, take it anyway.
|
||||
|
||||
|
||||
=item B<--wait>
|
||||
|
||||
=item B<-w>
|
||||
|
||||
Wait for all commands to complete.
|
||||
|
||||
=back
|
||||
|
||||
=head1 EXAMPLE: Gzipping *.log
|
||||
|
||||
for i in `ls *.log` ; do
|
||||
echo $i
|
||||
sem gzip $i ";" echo done
|
||||
done
|
||||
sem --wait
|
||||
|
||||
|
||||
=head1 BUGS
|
||||
|
||||
Quoting and composed commands are not working.
|
||||
|
||||
|
||||
=head1 REPORTING BUGS
|
||||
|
||||
Report bugs to <bug-parallel@gnu.org>.
|
||||
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Copyright (C) 2010 Ole Tange, http://ole.tange.dk and Free Software
|
||||
Foundation, Inc.
|
||||
|
||||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (C) 2010 Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or
|
||||
at your option any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
=head2 Documentation license I
|
||||
|
||||
Permission is granted to copy, distribute and/or modify this documentation
|
||||
under the terms of the GNU Free Documentation License, Version 1.3 or
|
||||
any later version published by the Free Software Foundation; with no
|
||||
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
|
||||
Texts. A copy of the license is included in the file fdl.txt.
|
||||
|
||||
=head2 Documentation license II
|
||||
|
||||
You are free:
|
||||
|
||||
=over 9
|
||||
|
||||
=item B<to Share>
|
||||
|
||||
to copy, distribute and transmit the work
|
||||
|
||||
=item B<to Remix>
|
||||
|
||||
to adapt the work
|
||||
|
||||
=back
|
||||
|
||||
Under the following conditions:
|
||||
|
||||
=over 9
|
||||
|
||||
=item B<Attribution>
|
||||
|
||||
You must attribute the work in the manner specified by the author or
|
||||
licensor (but not in any way that suggests that they endorse you or
|
||||
your use of the work).
|
||||
|
||||
=item B<Share Alike>
|
||||
|
||||
If you alter, transform, or build upon this work, you may distribute
|
||||
the resulting work only under the same, similar or a compatible
|
||||
license.
|
||||
|
||||
=back
|
||||
|
||||
With the understanding that:
|
||||
|
||||
=over 9
|
||||
|
||||
=item B<Waiver>
|
||||
|
||||
Any of the above conditions can be waived if you get permission from
|
||||
the copyright holder.
|
||||
|
||||
=item B<Public Domain>
|
||||
|
||||
Where the work or any of its elements is in the public domain under
|
||||
applicable law, that status is in no way affected by the license.
|
||||
|
||||
=item B<Other Rights>
|
||||
|
||||
In no way are any of the following rights affected by the license:
|
||||
|
||||
=over 2
|
||||
|
||||
=item *
|
||||
|
||||
Your fair dealing or fair use rights, or other applicable
|
||||
copyright exceptions and limitations;
|
||||
|
||||
=item *
|
||||
|
||||
The author's moral rights;
|
||||
|
||||
=item *
|
||||
|
||||
Rights other persons may have either in the work itself or in
|
||||
how the work is used, such as publicity or privacy rights.
|
||||
|
||||
=back
|
||||
|
||||
=back
|
||||
|
||||
=over 9
|
||||
|
||||
=item B<Notice>
|
||||
|
||||
For any reuse or distribution, you must make clear to others the
|
||||
license terms of this work.
|
||||
|
||||
=back
|
||||
|
||||
A copy of the full license is included in the file as cc-by-sa.txt.
|
||||
|
||||
=head1 DEPENDENCIES
|
||||
|
||||
GNU B<sem> uses Perl, and the Perl modules Getopt::Long,
|
||||
Symbol, Fcntl.
|
||||
|
||||
|
||||
=head1 SEE ALSO
|
||||
|
||||
B<parallel>(1)
|
||||
|
||||
=cut
|
||||
|
||||
use strict;
|
||||
use Symbol qw(gensym);
|
||||
use Getopt::Long;
|
||||
|
||||
Getopt::Long::Configure ("bundling","require_order");
|
||||
GetOptions("debug|D" => \$::opt_D,
|
||||
"id|i=s" => \$::opt_id,
|
||||
"count|j=i" => \$::opt_count,
|
||||
"fg" => \$::opt_fg,
|
||||
"timeout|t=i" => \$::opt_timeout,
|
||||
"version" => \$::opt_version,
|
||||
"wait|w" => \$::opt_wait,
|
||||
) || die_usage();
|
||||
$Global::debug = $::opt_D;
|
||||
$Global::version = 20100814;
|
||||
$Global::progname = 'sem';
|
||||
|
||||
my $count = 1; # Default 1 = mutex
|
||||
if($::opt_count) {
|
||||
$count = $::opt_count + 1;
|
||||
}
|
||||
if($::opt_wait) {
|
||||
$count = 1;
|
||||
}
|
||||
my $id = $::opt_id;
|
||||
my $fg = $::opt_fg || $::opt_wait;
|
||||
$::opt_timeout = $::opt_timeout;
|
||||
if(defined $::opt_version) {
|
||||
version();
|
||||
}
|
||||
|
||||
if(not defined $id) {
|
||||
# $id = getppid();
|
||||
# does not work with:
|
||||
# find . -name '*linux*' -exec sem -j1000 "sleep 3; echo `tty` '{}'" \; ; sem --wait echo done
|
||||
$id = `tty`;
|
||||
}
|
||||
$id = "id-$id";
|
||||
$id=~s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex
|
||||
my $sem = Semaphore->new($id,$count);
|
||||
$sem->acquire();
|
||||
debug("run");
|
||||
if($fg) {
|
||||
system @ARGV;
|
||||
$sem->release();
|
||||
} else {
|
||||
# If run in the background, the PID will change
|
||||
# therefore release and re-acquire the semaphore
|
||||
$sem->release();
|
||||
if(not fork()) {
|
||||
# child
|
||||
# Get a semaphore for this pid
|
||||
my $child_sem = Semaphore->new($id,$count);
|
||||
$child_sem->acquire();
|
||||
system @ARGV;
|
||||
$child_sem->release();
|
||||
}
|
||||
}
|
||||
|
||||
sub version {
|
||||
# Returns: N/A
|
||||
print join("\n",
|
||||
"GNU $Global::progname $Global::version",
|
||||
"Copyright (C) 2010 Ole Tange and Free Software Foundation, Inc.",
|
||||
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
|
||||
"This is free software: you are free to change and redistribute it.",
|
||||
"GNU $Global::progname comes with no warranty.",
|
||||
"",
|
||||
"Web site: http://www.gnu.org/software/parallel\n"
|
||||
);
|
||||
}
|
||||
|
||||
sub usage {
|
||||
# Returns: N/A
|
||||
print "Usage:\n";
|
||||
print "$Global::progname [options] [command [arguments]] < list_of_arguments)\n";
|
||||
print "$Global::progname [options] [command [arguments]] ::: arguments\n";
|
||||
print "$Global::progname [options] [command [arguments]] :::: argfile(s)\n";
|
||||
print "\n";
|
||||
print "See 'man $Global::progname' for the options\n";
|
||||
}
|
||||
|
||||
sub die_usage {
|
||||
usage();
|
||||
exit(255);
|
||||
}
|
||||
|
||||
sub debug {
|
||||
# Returns: N/A
|
||||
$Global::debug or return;
|
||||
@_ = grep { defined $_ ? $_ : "" } @_;
|
||||
print map {$_,"\n" } @_;
|
||||
}
|
||||
|
||||
package Semaphore;
|
||||
|
||||
# This package provides a counting semaphore
|
||||
#
|
||||
# If a process dies without releasing the semaphore the next process
|
||||
# that needs that entry will clean up dead semaphores
|
||||
#
|
||||
# The semaphores are stored in ~/.parallel/semaphores/id-<name> Each
|
||||
# file in ~/.parallel/semaphores/id-<name>/ is the process ID of the
|
||||
# process holding the entry. If the process dies, the entry can be
|
||||
# taken by another process.
|
||||
|
||||
use Fcntl qw(:DEFAULT :flock);
|
||||
|
||||
sub new {
|
||||
my $class = shift;
|
||||
my $id = shift;
|
||||
my $count = shift;
|
||||
my $parallel_locks = $ENV{'HOME'}."/.parallel/semaphores";
|
||||
-d $parallel_locks or mkdir $parallel_locks;
|
||||
my $lockdir = "$parallel_locks/$id";
|
||||
my $lockfile = $lockdir.".lock";
|
||||
return bless {
|
||||
'lockfile' => $lockfile,
|
||||
'lockfh' => Symbol::gensym(),
|
||||
'lockdir' => $lockdir,
|
||||
'id' => $id,
|
||||
'idfile' => $lockdir."/".$id,
|
||||
'pid' => $$,
|
||||
'pidfile' => $lockdir."/".$$,
|
||||
'count' => $count
|
||||
}, ref($class) || $class;
|
||||
}
|
||||
|
||||
sub acquire {
|
||||
my $self = shift;
|
||||
while(1) {
|
||||
$self->atomic_link_if_count_less_than() and last;
|
||||
::debug("Remove dead locks");
|
||||
my $lockdir = $self->{'lockdir'};
|
||||
for my $d (<$lockdir/*>) {
|
||||
$d =~ m:$lockdir/([0-9]+):o or next;
|
||||
if(not kill 0, $1) {
|
||||
::debug("Dead: $d");
|
||||
unlink $d;
|
||||
} else {
|
||||
::debug("Alive: $d");
|
||||
}
|
||||
}
|
||||
# try again
|
||||
$self->atomic_link_if_count_less_than() and last;
|
||||
sleep 1;
|
||||
# TODO if timeout: last
|
||||
}
|
||||
::debug("got $self->{'pid'}");
|
||||
}
|
||||
|
||||
sub release {
|
||||
my ($self) = shift;
|
||||
unlink $self->{'pidfile'};
|
||||
if($self->nlinks() == 1) {
|
||||
# This is the last link, so atomic cleanup
|
||||
$self->lock();
|
||||
if($self->nlinks() == 1) {
|
||||
unlink $self->{'idfile'};
|
||||
rmdir $self->{'lockdir'};
|
||||
}
|
||||
$self->unlock();
|
||||
}
|
||||
::debug("released $self->{'pid'}");
|
||||
}
|
||||
|
||||
|
||||
sub atomic_link_if_count_less_than {
|
||||
# Link $file1 to $file2 if nlinks to $file1 < $count
|
||||
my ($self) = shift;
|
||||
my ($retval) = 0;
|
||||
$self->lock();
|
||||
if($self->nlinks() < $count) {
|
||||
-d $self->{'lockdir'} || mkdir $self->{'lockdir'};
|
||||
if(not -e $self->{'idfile'}) {
|
||||
open (A, ">", $self->{'idfile'}) or die ">$self->{'idfile'}";
|
||||
close A;
|
||||
}
|
||||
$retval = link $self->{'idfile'}, $self->{'pidfile'};
|
||||
}
|
||||
$self->unlock();
|
||||
::debug("atomic $retval");
|
||||
return $retval;
|
||||
}
|
||||
|
||||
sub nlinks {
|
||||
my $self = shift;
|
||||
if(-e $self->{'idfile'}) {
|
||||
return (stat(_))[3];
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
sub lock {
|
||||
my ($self) = shift;
|
||||
open $self->{'lockfh'}, ">", $self->{'lockfile'}
|
||||
or die "Can't open semaphore file $self->{'lockfile'}: $!";
|
||||
chmod 0666, $self->{'lockfile'}; # assuming you want it a+rw
|
||||
while(not flock $self->{'lockfh'}, LOCK_EX()|LOCK_NB()) {
|
||||
::debug("Cannot lock $self->{'lockfile'}");
|
||||
# TODO if timeout: last
|
||||
sleep 1;
|
||||
}
|
||||
::debug("locked $self->{'lockfile'}");
|
||||
}
|
||||
|
||||
sub unlock {
|
||||
my $self = shift;
|
||||
unlink $self->{'lockfile'};
|
||||
close $self->{'lockfh'};
|
||||
::debug("unlocked");
|
||||
}
|
||||
|
|
@ -6,10 +6,10 @@ parallel -u --semaphore seq 11 20 '|' pv -qL 100
|
|||
parallel --semaphore --wait
|
||||
echo done
|
||||
|
||||
echo '### Test default id = --id `tty`'
|
||||
echo '### Test default id = --id `tty` and --semaphorename'
|
||||
parallel --id `tty` -u --semaphore seq 1 10 '|' pv -qL 20
|
||||
parallel -u --semaphore seq 11 20 '|' pv -qL 100
|
||||
parallel --id `tty` --semaphore --wait
|
||||
parallel --semaphorename `tty` --semaphore --wait
|
||||
echo done
|
||||
|
||||
echo '### Test semaphore 2 jobs running simultaneously'
|
||||
|
|
@ -30,3 +30,10 @@ for i in 0.5 0.1 0.2 0.3 0.4 ; do
|
|||
sem -j+0 sleep $i ";" echo done $i
|
||||
done
|
||||
sem --wait
|
||||
|
||||
echo '### BUG: Test --fg followed by --bg'
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
|
||||
sem --wait
|
||||
|
|
@ -20,7 +20,7 @@
|
|||
19
|
||||
20
|
||||
done
|
||||
### Test default id = --id `tty`
|
||||
### Test default id = --id `tty` and --semaphorename
|
||||
1
|
||||
2
|
||||
3
|
||||
|
|
@ -61,7 +61,48 @@ done
|
|||
done 0.1
|
||||
0.3
|
||||
done 0.5
|
||||
done 0.2
|
||||
0.4
|
||||
done 0.2
|
||||
done 0.3
|
||||
done 0.4
|
||||
### BUG: Test --fg followed by --bg
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19
|
||||
20
|
||||
21
|
||||
22
|
||||
23
|
||||
24
|
||||
25
|
||||
26
|
||||
27
|
||||
28
|
||||
29
|
||||
30
|
||||
31
|
||||
32
|
||||
33
|
||||
34
|
||||
35
|
||||
36
|
||||
37
|
||||
38
|
||||
39
|
||||
40
|
||||
|
|
|
|||
Loading…
Reference in a new issue