Code cleanup. POD file for mini man page for sem

This commit is contained in:
Ole Tange 2010-08-17 08:53:46 +02:00
parent 07a2682099
commit 47cf0f0128
11 changed files with 577 additions and 77 deletions

View file

@ -1,6 +1,6 @@
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100817.
@%:@
@%:@ Report bugs to <bug-parallel@gnu.org>.
@%:@
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100722'
PACKAGE_STRING='parallel 20100722'
PACKAGE_VERSION='20100817'
PACKAGE_STRING='parallel 20100817'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
\`configure' configures parallel 20100817 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100722:";;
short | recursive ) echo "Configuration of parallel 20100817:";;
esac
cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20100722
parallel configure 20100817
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100722, which was
It was created by parallel $as_me 20100817, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20100722'
VERSION='20100817'
cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20100722, which was
This file was extended by parallel $as_me 20100817, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20100722
parallel config.status 20100817
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"

View file

@ -1,6 +1,6 @@
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100817.
@%:@
@%:@ Report bugs to <bug-parallel@gnu.org>.
@%:@
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100722'
PACKAGE_STRING='parallel 20100722'
PACKAGE_VERSION='20100817'
PACKAGE_STRING='parallel 20100817'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
\`configure' configures parallel 20100817 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100722:";;
short | recursive ) echo "Configuration of parallel 20100817:";;
esac
cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20100722
parallel configure 20100817
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100722, which was
It was created by parallel $as_me 20100817, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20100722'
VERSION='20100817'
cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20100722, which was
This file was extended by parallel $as_me 20100817, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20100722
parallel config.status 20100817
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"

View file

@ -1,4 +1,4 @@
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100722], [bug-parallel@gnu.org])
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100817], [bug-parallel@gnu.org])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS'])

20
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.65 for parallel 20100722.
# Generated by GNU Autoconf 2.65 for parallel 20100817.
#
# Report bugs to <bug-parallel@gnu.org>.
#
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100722'
PACKAGE_STRING='parallel 20100722'
PACKAGE_VERSION='20100817'
PACKAGE_STRING='parallel 20100817'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
\`configure' configures parallel 20100817 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100722:";;
short | recursive ) echo "Configuration of parallel 20100817:";;
esac
cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20100722
parallel configure 20100817
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100722, which was
It was created by parallel $as_me 20100817, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20100722'
VERSION='20100817'
cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20100722, which was
This file was extended by parallel $as_me 20100817, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20100722
parallel config.status 20100817
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"

View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20100722], [bug-parallel@gnu.org])
AC_INIT([parallel], [20100817], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([

View file

@ -1,11 +1,16 @@
# sem -j+0
# sem gzip foo ";" echo done
# Examples of sem
Weird bug - only interactive
echo '### BUG: Test --fg followed by --bg'
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
sem --wait
find . -execdir sem -j100 'sleep 4; echo {}' \; ; sem --wait
# Allow 7 to run. After then 7th is started, block untill one is dead
parallel --semaphore --id uniqidentifier -j7 command
parallel --semaphore -j7 command
mdm.screen find dir -execdir mdm-run cmd {} \;
find dir -execdir parallel --semaphore cmd {} \;
fex syntax for splitting fields
http://www.semicomplete.com/projects/fex/

View file

@ -79,7 +79,9 @@ Newsgroups: comp.unix.shell,comp.unix.admin
<<<<<
to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org
cc:Peter Simons <simons@cryp.to>, Sandro Cazzaniga <kharec@mandriva.org>,
Tim Cuthbertson <tim3d.junk@gmail.com>
Tim Cuthbertson <tim3d.junk@gmail.com>, Ludovic Courtès <ludo@gnu.org>,
Markus Ammer <mkmm@gmx-topmail.de>, Pavel Nuzhdin <pnzhdin@gmail.com>,
((psung@alum.mit.edu))
Subject: GNU Parallel 20100722 released
@ -88,12 +90,15 @@ download at: http://ftp.gnu.org/gnu/parallel/
New in this release:
* Counting semaphore functionality: start a job in the background. If
N jobs are already running, wait for one to complete.
* With --colsep a table can be used as input. Example:
cat tab_sep_table | parallel --colsep '\t' echo col1 {1} col2 {2}
* --trim can remove white space around arguments.
* NixOS package. Thanks to Ludovic Courtès <ludo at gnu dot org>
* --sshloginfile '..' means use ~/.parallel/sshloginfile
* Zero install package. Thanks to Tim Cuthbertson <tim3d dot junk at
gmail dot com>
@ -101,11 +106,13 @@ New in this release:
* OpenSUSE package. Thanks to Markus Ammer <mkmm at gmx-topmail dot
de>
* NixOS package. Thanks to Ludovic Courtès <ludo at gnu dot org>
* Web review http://oentend.blogspot.com/2010/08/gnu-parallel.html
Thanks to Pavel Nuzhdin <pnzhdin at gmail dot com>
* Web review http://psung.blogspot.com/2010/08/gnu-parallel.html
Thanks to Phil Sung
Thanks to Phil Sung ((<psung at alum dot mit dot edu>))
= About GNU Parallel =

View file

@ -184,7 +184,7 @@ Multiple B<-B> can be specified to transfer more basefiles. The
I<file> will be transferred the same way as B<--transfer>.
=item B<--bg> (not implemented)
=item B<--bg> (beta testing)
Run command in background thus GNU B<parallel> will not wait for
completion of the command before exiting. This is the default if
@ -281,7 +281,7 @@ jobs. GNU B<parallel> normally only reads the next job to run.
Implies B<--progress>.
=item B<--fg> (not implemented)
=item B<--fg> (beta testing)
Run command in foreground thus GNU B<parallel> will wait for
completion of the command before exiting.
@ -606,7 +606,7 @@ operating system and the B<-s> option. Pipe the input from /dev/null
to do anything.
=item B<--semaphore> (not implemented)
=item B<--semaphore> (beta testing)
Work as a counting semaphore. B<--semaphore> will cause GNU
B<parallel> to start I<command> in the background. When the number of
@ -623,9 +623,9 @@ Used with B<--fg>, B<--wait>, and B<--semaphorename>.
The command B<sem> is an alias for B<parallel --semaphore>.
=item B<--semaphorename> I<name> (not implemented)
=item B<--semaphorename> I<name> (beta testing)
=item B<--id> I<name> (not implemented)
=item B<--id> I<name>
The name of the semaphore to use. The semaphore can be shared between
multiple processes.
@ -817,7 +817,7 @@ B<--silent>. See also B<-t>.
Print the version GNU B<parallel> and exit.
=item B<--wait> (not implemented)
=item B<--wait> (beta testing)
Wait for all commands to complete.
@ -856,6 +856,8 @@ FUBAR in all files in this dir and subdirs:
B<find . -type f -print0 | parallel -q0 perl -i -pe 's/FOO BAR/FUBAR/g'>
Note B<-q> is needed because of the space in 'FOO BAR'.
=head1 EXAMPLE: Reading arguments from command line
@ -1658,12 +1660,16 @@ B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
pexec -j -m blockwrite -s th_$IMG'
B<8> GNU B<parallel> does not support mutexes directly but uses B<mutex> to
do that.
B<8> Combining GNU B<parallel> and GNU B<sem>.
B<8> ls *jpg | parallel -j8 'mutex -m blockread cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | mutex -m blockwrite cat > th_{}'
B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
B<8> If reading and writing is done to the same disk, this may be
faster as only one process will be either reading or writing:
B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
=head2 DIFFERENCES BETWEEN xjobs AND GNU Parallel
@ -1749,6 +1755,7 @@ B<seq 1 19 | parallel -j+0 buffon -o - | sort -n >>B< result>
B<cat files | parallel -j+0 cmd>
B<find dir -execdir sem -j+0 cmd {} \;>
=head2 DIFFERENCES BETWEEN xapply AND GNU Parallel
@ -1823,10 +1830,9 @@ B<cat hostlist | parallel ssh {} do_stuff>
=head1 BUGS
Filenames beginning with '-' can cause some commands to give
unexpected results, as it will often be interpreted as an option.
=head2 Quoting of newline
Because the way newline is quoted this will not work:
Because of the way newline is quoted this will not work:
echo 1,2,3 | parallel -vkd, "echo 'a{}'"
@ -1834,6 +1840,11 @@ However, this will work:
echo 1,2,3 | parallel -vkd, echo a{}
=head2 Startup speed
GNU B<parallel> is slow at starting up. Half of the startup time is
spent finding the maximal length of a command line. Setting B<-s> will
remove this part of the startup time.
=head1 REPORTING BUGS
@ -1987,16 +1998,16 @@ use strict;
do_not_reap();
parse_options();
init_run_jobs();
my $sem;
if($Global::semaphore) {
run_as_semaphore();
} else {
start_more_jobs();
$sem = acquire_semaphore();
}
start_more_jobs();
reap_if_needed();
drain_job_queue();
cleanup();
if($Global::semaphore) {
exit $Global::exitstatus;
$sem->release();
}
if($::opt_halt_on_error) {
wait_and_exit($Global::halt_on_error_exitstatus);
@ -2004,39 +2015,37 @@ if($::opt_halt_on_error) {
wait_and_exit(min($Global::exitstatus,254));
}
sub run_as_semaphore {
sub acquire_semaphore {
# Acquires semaphore. If needed: spawns to the background
# Returns:
# The semaphore to be released when jobs is complete
my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
$sem->acquire();
debug("run");
$Global::argfile = open_or_exit("/dev/null");
unget_arg("");
if($Semaphore::fg) {
start_more_jobs();
$sem->release();
# skip
} else {
# If run in the background, the PID will change
# therefore release and re-acquire the semaphore
$sem->release();
if(not fork()) {
if(fork()) {
exit(0);
} else {
# child
# Get a semaphore for this pid
my $child_sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
$child_sem->acquire();
start_more_jobs();
reap_if_needed();
drain_job_queue();
cleanup();
$child_sem->release();
} else {
exit(0);
$sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
$sem->acquire();
}
}
return $sem;
}
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20100722;
$Global::version = 20100817;
$Global::progname = 'parallel';
$Global::debug = 0;
$Global::verbose = 0;
@ -4289,6 +4298,7 @@ sub new {
-d $parallel_locks or mkdir $parallel_locks;
my $lockdir = "$parallel_locks/$id";
my $lockfile = $lockdir.".lock";
if($count < 1) { die "Semaphore count = $count"; }
return bless {
'lockfile' => $lockfile,
'lockfh' => Symbol::gensym(),

430
src/sem.pod Executable file
View file

@ -0,0 +1,430 @@
#!/usr/bin/perl -w
=head1 NAME
sem - semaphore for executing shell command lines in parallel
=head1 SYNOPSIS
B<sem> [--fg] [--id <id>] [--timeout <secs>] [-j <num>] [--wait] command
=head1 DESCRIPTION
GNU B<sem> is an alias for GNU B<parallel --semaphore>.
It works as a tool for executing shell commands in parallel. GNU
B<sem> acts as a counting semaphore. When GNU B<sem> is called with
command it will start the command in the background. When I<num>
number of commands are running in the background, GNU B<sem> will wait
for one of these to complete before starting another command.
Before looking at the options you may want to check out the examples
after the list of options. That will give you an idea of what GNU
B<sem> is capable of.
=head1 OPTIONS
=over 9
=item I<command>
Command to execute. The command may be followed by arguments for the command.
=item B<--count> I<N>
=item B<-j> I<N>
Run up to N commands in parallel. Default is 1 thus acting like a
mutex.
=item B<--id> I<id>
=item B<-i> I<id>
Use B<id> as the name of the semaphore. Default is the name of the
controlling tty (output from B<tty>).
The default normally works as expected when used interactively, but
when used in a script I<id> should be set. $$ is often a good value.
=item B<--fg>
Do not put command in background.
=item B<--timeout> I<secs> (not implemented)
=item B<-t> I<secs> (not implemented)
If the semaphore is not released within I<secs> seconds, take it anyway.
=item B<--wait>
=item B<-w>
Wait for all commands to complete.
=back
=head1 EXAMPLE: Gzipping *.log
for i in `ls *.log` ; do
echo $i
sem gzip $i ";" echo done
done
sem --wait
=head1 BUGS
Quoting and composed commands are not working.
=head1 REPORTING BUGS
Report bugs to <bug-parallel@gnu.org>.
=head1 AUTHOR
Copyright (C) 2010 Ole Tange, http://ole.tange.dk and Free Software
Foundation, Inc.
=head1 LICENSE
Copyright (C) 2010 Free Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
at your option any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
=head2 Documentation license I
Permission is granted to copy, distribute and/or modify this documentation
under the terms of the GNU Free Documentation License, Version 1.3 or
any later version published by the Free Software Foundation; with no
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
Texts. A copy of the license is included in the file fdl.txt.
=head2 Documentation license II
You are free:
=over 9
=item B<to Share>
to copy, distribute and transmit the work
=item B<to Remix>
to adapt the work
=back
Under the following conditions:
=over 9
=item B<Attribution>
You must attribute the work in the manner specified by the author or
licensor (but not in any way that suggests that they endorse you or
your use of the work).
=item B<Share Alike>
If you alter, transform, or build upon this work, you may distribute
the resulting work only under the same, similar or a compatible
license.
=back
With the understanding that:
=over 9
=item B<Waiver>
Any of the above conditions can be waived if you get permission from
the copyright holder.
=item B<Public Domain>
Where the work or any of its elements is in the public domain under
applicable law, that status is in no way affected by the license.
=item B<Other Rights>
In no way are any of the following rights affected by the license:
=over 2
=item *
Your fair dealing or fair use rights, or other applicable
copyright exceptions and limitations;
=item *
The author's moral rights;
=item *
Rights other persons may have either in the work itself or in
how the work is used, such as publicity or privacy rights.
=back
=back
=over 9
=item B<Notice>
For any reuse or distribution, you must make clear to others the
license terms of this work.
=back
A copy of the full license is included in the file as cc-by-sa.txt.
=head1 DEPENDENCIES
GNU B<sem> uses Perl, and the Perl modules Getopt::Long,
Symbol, Fcntl.
=head1 SEE ALSO
B<parallel>(1)
=cut
use strict;
use Symbol qw(gensym);
use Getopt::Long;
Getopt::Long::Configure ("bundling","require_order");
GetOptions("debug|D" => \$::opt_D,
"id|i=s" => \$::opt_id,
"count|j=i" => \$::opt_count,
"fg" => \$::opt_fg,
"timeout|t=i" => \$::opt_timeout,
"version" => \$::opt_version,
"wait|w" => \$::opt_wait,
) || die_usage();
$Global::debug = $::opt_D;
$Global::version = 20100814;
$Global::progname = 'sem';
my $count = 1; # Default 1 = mutex
if($::opt_count) {
$count = $::opt_count + 1;
}
if($::opt_wait) {
$count = 1;
}
my $id = $::opt_id;
my $fg = $::opt_fg || $::opt_wait;
$::opt_timeout = $::opt_timeout;
if(defined $::opt_version) {
version();
}
if(not defined $id) {
# $id = getppid();
# does not work with:
# find . -name '*linux*' -exec sem -j1000 "sleep 3; echo `tty` '{}'" \; ; sem --wait echo done
$id = `tty`;
}
$id = "id-$id";
$id=~s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex
my $sem = Semaphore->new($id,$count);
$sem->acquire();
debug("run");
if($fg) {
system @ARGV;
$sem->release();
} else {
# If run in the background, the PID will change
# therefore release and re-acquire the semaphore
$sem->release();
if(not fork()) {
# child
# Get a semaphore for this pid
my $child_sem = Semaphore->new($id,$count);
$child_sem->acquire();
system @ARGV;
$child_sem->release();
}
}
sub version {
# Returns: N/A
print join("\n",
"GNU $Global::progname $Global::version",
"Copyright (C) 2010 Ole Tange and Free Software Foundation, Inc.",
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
"This is free software: you are free to change and redistribute it.",
"GNU $Global::progname comes with no warranty.",
"",
"Web site: http://www.gnu.org/software/parallel\n"
);
}
sub usage {
# Returns: N/A
print "Usage:\n";
print "$Global::progname [options] [command [arguments]] < list_of_arguments)\n";
print "$Global::progname [options] [command [arguments]] ::: arguments\n";
print "$Global::progname [options] [command [arguments]] :::: argfile(s)\n";
print "\n";
print "See 'man $Global::progname' for the options\n";
}
sub die_usage {
usage();
exit(255);
}
sub debug {
# Returns: N/A
$Global::debug or return;
@_ = grep { defined $_ ? $_ : "" } @_;
print map {$_,"\n" } @_;
}
package Semaphore;
# This package provides a counting semaphore
#
# If a process dies without releasing the semaphore the next process
# that needs that entry will clean up dead semaphores
#
# The semaphores are stored in ~/.parallel/semaphores/id-<name> Each
# file in ~/.parallel/semaphores/id-<name>/ is the process ID of the
# process holding the entry. If the process dies, the entry can be
# taken by another process.
use Fcntl qw(:DEFAULT :flock);
sub new {
my $class = shift;
my $id = shift;
my $count = shift;
my $parallel_locks = $ENV{'HOME'}."/.parallel/semaphores";
-d $parallel_locks or mkdir $parallel_locks;
my $lockdir = "$parallel_locks/$id";
my $lockfile = $lockdir.".lock";
return bless {
'lockfile' => $lockfile,
'lockfh' => Symbol::gensym(),
'lockdir' => $lockdir,
'id' => $id,
'idfile' => $lockdir."/".$id,
'pid' => $$,
'pidfile' => $lockdir."/".$$,
'count' => $count
}, ref($class) || $class;
}
sub acquire {
my $self = shift;
while(1) {
$self->atomic_link_if_count_less_than() and last;
::debug("Remove dead locks");
my $lockdir = $self->{'lockdir'};
for my $d (<$lockdir/*>) {
$d =~ m:$lockdir/([0-9]+):o or next;
if(not kill 0, $1) {
::debug("Dead: $d");
unlink $d;
} else {
::debug("Alive: $d");
}
}
# try again
$self->atomic_link_if_count_less_than() and last;
sleep 1;
# TODO if timeout: last
}
::debug("got $self->{'pid'}");
}
sub release {
my ($self) = shift;
unlink $self->{'pidfile'};
if($self->nlinks() == 1) {
# This is the last link, so atomic cleanup
$self->lock();
if($self->nlinks() == 1) {
unlink $self->{'idfile'};
rmdir $self->{'lockdir'};
}
$self->unlock();
}
::debug("released $self->{'pid'}");
}
sub atomic_link_if_count_less_than {
# Link $file1 to $file2 if nlinks to $file1 < $count
my ($self) = shift;
my ($retval) = 0;
$self->lock();
if($self->nlinks() < $count) {
-d $self->{'lockdir'} || mkdir $self->{'lockdir'};
if(not -e $self->{'idfile'}) {
open (A, ">", $self->{'idfile'}) or die ">$self->{'idfile'}";
close A;
}
$retval = link $self->{'idfile'}, $self->{'pidfile'};
}
$self->unlock();
::debug("atomic $retval");
return $retval;
}
sub nlinks {
my $self = shift;
if(-e $self->{'idfile'}) {
return (stat(_))[3];
} else {
return 0;
}
}
sub lock {
my ($self) = shift;
open $self->{'lockfh'}, ">", $self->{'lockfile'}
or die "Can't open semaphore file $self->{'lockfile'}: $!";
chmod 0666, $self->{'lockfile'}; # assuming you want it a+rw
while(not flock $self->{'lockfh'}, LOCK_EX()|LOCK_NB()) {
::debug("Cannot lock $self->{'lockfile'}");
# TODO if timeout: last
sleep 1;
}
::debug("locked $self->{'lockfile'}");
}
sub unlock {
my $self = shift;
unlink $self->{'lockfile'};
close $self->{'lockfh'};
::debug("unlocked");
}

View file

@ -6,10 +6,10 @@ parallel -u --semaphore seq 11 20 '|' pv -qL 100
parallel --semaphore --wait
echo done
echo '### Test default id = --id `tty`'
echo '### Test default id = --id `tty` and --semaphorename'
parallel --id `tty` -u --semaphore seq 1 10 '|' pv -qL 20
parallel -u --semaphore seq 11 20 '|' pv -qL 100
parallel --id `tty` --semaphore --wait
parallel --semaphorename `tty` --semaphore --wait
echo done
echo '### Test semaphore 2 jobs running simultaneously'
@ -30,3 +30,10 @@ for i in 0.5 0.1 0.2 0.3 0.4 ; do
sem -j+0 sleep $i ";" echo done $i
done
sem --wait
echo '### BUG: Test --fg followed by --bg'
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
sem --wait

View file

@ -20,7 +20,7 @@
19
20
done
### Test default id = --id `tty`
### Test default id = --id `tty` and --semaphorename
1
2
3
@ -61,7 +61,48 @@ done
done 0.1
0.3
done 0.5
done 0.2
0.4
done 0.2
done 0.3
done 0.4
### BUG: Test --fg followed by --bg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40