From 47cf0f0128f2e900a1d5fafe63bdb0fa32e3b7c8 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 17 Aug 2010 08:53:46 +0200 Subject: [PATCH] Code cleanup. POD file for mini man page for sem --- autom4te.cache/output.0 | 20 +- autom4te.cache/output.1 | 20 +- autom4te.cache/traces.1 | 2 +- configure | 20 +- configure.ac | 2 +- doc/FUTURE_IDEAS | 19 +- doc/release_new_version | 13 +- src/parallel | 72 +++--- src/sem.pod | 430 +++++++++++++++++++++++++++++++++ unittest/tests-to-run/sem01.sh | 11 +- unittest/wanted-results/sem01 | 45 +++- 11 files changed, 577 insertions(+), 77 deletions(-) create mode 100755 src/sem.pod diff --git a/autom4te.cache/output.0 b/autom4te.cache/output.0 index 30667204..3ba1f54a 100644 --- a/autom4te.cache/output.0 +++ b/autom4te.cache/output.0 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100817. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100722' -PACKAGE_STRING='parallel 20100722' +PACKAGE_VERSION='20100817' +PACKAGE_STRING='parallel 20100817' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100722 to adapt to many kinds of systems. +\`configure' configures parallel 20100817 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100722:";; + short | recursive ) echo "Configuration of parallel 20100817:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100722 +parallel configure 20100817 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100722, which was +It was created by parallel $as_me 20100817, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100722' + VERSION='20100817' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100722, which was +This file was extended by parallel $as_me 20100817, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100722 +parallel config.status 20100817 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/output.1 b/autom4te.cache/output.1 index 30667204..3ba1f54a 100644 --- a/autom4te.cache/output.1 +++ b/autom4te.cache/output.1 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100817. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100722' -PACKAGE_STRING='parallel 20100722' +PACKAGE_VERSION='20100817' +PACKAGE_STRING='parallel 20100817' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100722 to adapt to many kinds of systems. +\`configure' configures parallel 20100817 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100722:";; + short | recursive ) echo "Configuration of parallel 20100817:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100722 +parallel configure 20100817 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100722, which was +It was created by parallel $as_me 20100817, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100722' + VERSION='20100817' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100722, which was +This file was extended by parallel $as_me 20100817, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100722 +parallel config.status 20100817 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/traces.1 b/autom4te.cache/traces.1 index 71e2f0ff..b8da6432 100644 --- a/autom4te.cache/traces.1 +++ b/autom4te.cache/traces.1 @@ -1,4 +1,4 @@ -m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100722], [bug-parallel@gnu.org]) +m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100817], [bug-parallel@gnu.org]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS']) diff --git a/configure b/configure index 075ac74b..492bb339 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.65 for parallel 20100722. +# Generated by GNU Autoconf 2.65 for parallel 20100817. # # Report bugs to . # @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100722' -PACKAGE_STRING='parallel 20100722' +PACKAGE_VERSION='20100817' +PACKAGE_STRING='parallel 20100817' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100722 to adapt to many kinds of systems. +\`configure' configures parallel 20100817 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100722:";; + short | recursive ) echo "Configuration of parallel 20100817:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100722 +parallel configure 20100817 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100722, which was +It was created by parallel $as_me 20100817, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100722' + VERSION='20100817' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100722, which was +This file was extended by parallel $as_me 20100817, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100722 +parallel config.status 20100817 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e3f0bf70..40a4c127 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20100722], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20100817], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index b8b1e53f..ffd67df9 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,11 +1,16 @@ -# sem -j+0 -# sem gzip foo ";" echo done +# Examples of sem + +Weird bug - only interactive + +echo '### BUG: Test --fg followed by --bg' +parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20 +parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20 +parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20 +parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20 +sem --wait + +find . -execdir sem -j100 'sleep 4; echo {}' \; ; sem --wait -# Allow 7 to run. After then 7th is started, block untill one is dead -parallel --semaphore --id uniqidentifier -j7 command -parallel --semaphore -j7 command -mdm.screen find dir -execdir mdm-run cmd {} \; -find dir -execdir parallel --semaphore cmd {} \; fex syntax for splitting fields http://www.semicomplete.com/projects/fex/ diff --git a/doc/release_new_version b/doc/release_new_version index c6487cff..05c0c643 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -79,7 +79,9 @@ Newsgroups: comp.unix.shell,comp.unix.admin <<<<< to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org cc:Peter Simons , Sandro Cazzaniga , - Tim Cuthbertson + Tim Cuthbertson , Ludovic Courtès , + Markus Ammer , Pavel Nuzhdin , + ((psung@alum.mit.edu)) Subject: GNU Parallel 20100722 released @@ -88,12 +90,15 @@ download at: http://ftp.gnu.org/gnu/parallel/ New in this release: +* Counting semaphore functionality: start a job in the background. If + N jobs are already running, wait for one to complete. + * With --colsep a table can be used as input. Example: cat tab_sep_table | parallel --colsep '\t' echo col1 {1} col2 {2} * --trim can remove white space around arguments. -* NixOS package. Thanks to Ludovic Courtès +* --sshloginfile '..' means use ~/.parallel/sshloginfile * Zero install package. Thanks to Tim Cuthbertson @@ -101,11 +106,13 @@ New in this release: * OpenSUSE package. Thanks to Markus Ammer +* NixOS package. Thanks to Ludovic Courtès + * Web review http://oentend.blogspot.com/2010/08/gnu-parallel.html Thanks to Pavel Nuzhdin * Web review http://psung.blogspot.com/2010/08/gnu-parallel.html - Thanks to Phil Sung + Thanks to Phil Sung (()) = About GNU Parallel = diff --git a/src/parallel b/src/parallel index eed705db..241d3486 100755 --- a/src/parallel +++ b/src/parallel @@ -184,7 +184,7 @@ Multiple B<-B> can be specified to transfer more basefiles. The I will be transferred the same way as B<--transfer>. -=item B<--bg> (not implemented) +=item B<--bg> (beta testing) Run command in background thus GNU B will not wait for completion of the command before exiting. This is the default if @@ -281,7 +281,7 @@ jobs. GNU B normally only reads the next job to run. Implies B<--progress>. -=item B<--fg> (not implemented) +=item B<--fg> (beta testing) Run command in foreground thus GNU B will wait for completion of the command before exiting. @@ -606,7 +606,7 @@ operating system and the B<-s> option. Pipe the input from /dev/null to do anything. -=item B<--semaphore> (not implemented) +=item B<--semaphore> (beta testing) Work as a counting semaphore. B<--semaphore> will cause GNU B to start I in the background. When the number of @@ -623,9 +623,9 @@ Used with B<--fg>, B<--wait>, and B<--semaphorename>. The command B is an alias for B. -=item B<--semaphorename> I (not implemented) +=item B<--semaphorename> I (beta testing) -=item B<--id> I (not implemented) +=item B<--id> I The name of the semaphore to use. The semaphore can be shared between multiple processes. @@ -817,7 +817,7 @@ B<--silent>. See also B<-t>. Print the version GNU B and exit. -=item B<--wait> (not implemented) +=item B<--wait> (beta testing) Wait for all commands to complete. @@ -856,6 +856,8 @@ FUBAR in all files in this dir and subdirs: B +Note B<-q> is needed because of the space in 'FOO BAR'. + =head1 EXAMPLE: Reading arguments from command line @@ -1658,12 +1660,16 @@ B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \ jpegtopnm | pnmscale 0.5 | pnmtojpeg | \ pexec -j -m blockwrite -s th_$IMG' -B<8> GNU B does not support mutexes directly but uses B to -do that. +B<8> Combining GNU B and GNU B. -B<8> ls *jpg | parallel -j8 'mutex -m blockread cat {} | jpegtopnm |' \ - 'pnmscale 0.5 | pnmtojpeg | mutex -m blockwrite cat > th_{}' +B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \ + 'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}' +B<8> If reading and writing is done to the same disk, this may be +faster as only one process will be either reading or writing: + +B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \ + 'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}' =head2 DIFFERENCES BETWEEN xjobs AND GNU Parallel @@ -1749,6 +1755,7 @@ B>B< result> B +B =head2 DIFFERENCES BETWEEN xapply AND GNU Parallel @@ -1823,10 +1830,9 @@ B =head1 BUGS -Filenames beginning with '-' can cause some commands to give -unexpected results, as it will often be interpreted as an option. +=head2 Quoting of newline -Because the way newline is quoted this will not work: +Because of the way newline is quoted this will not work: echo 1,2,3 | parallel -vkd, "echo 'a{}'" @@ -1834,6 +1840,11 @@ However, this will work: echo 1,2,3 | parallel -vkd, echo a{} +=head2 Startup speed + +GNU B is slow at starting up. Half of the startup time is +spent finding the maximal length of a command line. Setting B<-s> will +remove this part of the startup time. =head1 REPORTING BUGS @@ -1987,16 +1998,16 @@ use strict; do_not_reap(); parse_options(); init_run_jobs(); +my $sem; if($Global::semaphore) { - run_as_semaphore(); -} else { - start_more_jobs(); + $sem = acquire_semaphore(); } +start_more_jobs(); reap_if_needed(); drain_job_queue(); cleanup(); if($Global::semaphore) { - exit $Global::exitstatus; + $sem->release(); } if($::opt_halt_on_error) { wait_and_exit($Global::halt_on_error_exitstatus); @@ -2004,39 +2015,37 @@ if($::opt_halt_on_error) { wait_and_exit(min($Global::exitstatus,254)); } -sub run_as_semaphore { +sub acquire_semaphore { + # Acquires semaphore. If needed: spawns to the background + # Returns: + # The semaphore to be released when jobs is complete my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'}); $sem->acquire(); debug("run"); $Global::argfile = open_or_exit("/dev/null"); unget_arg(""); if($Semaphore::fg) { - start_more_jobs(); - $sem->release(); + # skip } else { # If run in the background, the PID will change # therefore release and re-acquire the semaphore $sem->release(); - if(not fork()) { + if(fork()) { + exit(0); + } else { # child # Get a semaphore for this pid - my $child_sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'}); - $child_sem->acquire(); - start_more_jobs(); - reap_if_needed(); - drain_job_queue(); - cleanup(); - $child_sem->release(); - } else { - exit(0); + $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'}); + $sem->acquire(); } } + return $sem; } sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20100722; + $Global::version = 20100817; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -4289,6 +4298,7 @@ sub new { -d $parallel_locks or mkdir $parallel_locks; my $lockdir = "$parallel_locks/$id"; my $lockfile = $lockdir.".lock"; + if($count < 1) { die "Semaphore count = $count"; } return bless { 'lockfile' => $lockfile, 'lockfh' => Symbol::gensym(), diff --git a/src/sem.pod b/src/sem.pod new file mode 100755 index 00000000..bbec5ca1 --- /dev/null +++ b/src/sem.pod @@ -0,0 +1,430 @@ +#!/usr/bin/perl -w + +=head1 NAME + +sem - semaphore for executing shell command lines in parallel + +=head1 SYNOPSIS + +B [--fg] [--id ] [--timeout ] [-j ] [--wait] command + +=head1 DESCRIPTION + +GNU B is an alias for GNU B. + +It works as a tool for executing shell commands in parallel. GNU +B acts as a counting semaphore. When GNU B is called with +command it will start the command in the background. When I +number of commands are running in the background, GNU B will wait +for one of these to complete before starting another command. + +Before looking at the options you may want to check out the examples +after the list of options. That will give you an idea of what GNU +B is capable of. + +=head1 OPTIONS + +=over 9 + +=item I + +Command to execute. The command may be followed by arguments for the command. + + +=item B<--count> I + +=item B<-j> I + +Run up to N commands in parallel. Default is 1 thus acting like a +mutex. + + +=item B<--id> I + +=item B<-i> I + +Use B as the name of the semaphore. Default is the name of the +controlling tty (output from B). + +The default normally works as expected when used interactively, but +when used in a script I should be set. $$ is often a good value. + + +=item B<--fg> + +Do not put command in background. + + +=item B<--timeout> I (not implemented) + +=item B<-t> I (not implemented) + +If the semaphore is not released within I seconds, take it anyway. + + +=item B<--wait> + +=item B<-w> + +Wait for all commands to complete. + +=back + +=head1 EXAMPLE: Gzipping *.log + + for i in `ls *.log` ; do + echo $i + sem gzip $i ";" echo done + done + sem --wait + + +=head1 BUGS + +Quoting and composed commands are not working. + + +=head1 REPORTING BUGS + +Report bugs to . + + +=head1 AUTHOR + +Copyright (C) 2010 Ole Tange, http://ole.tange.dk and Free Software +Foundation, Inc. + + +=head1 LICENSE + +Copyright (C) 2010 Free Software Foundation, Inc. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +at your option any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +=head2 Documentation license I + +Permission is granted to copy, distribute and/or modify this documentation +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, with no Front-Cover Texts, and with no Back-Cover +Texts. A copy of the license is included in the file fdl.txt. + +=head2 Documentation license II + +You are free: + +=over 9 + +=item B + +to copy, distribute and transmit the work + +=item B + +to adapt the work + +=back + +Under the following conditions: + +=over 9 + +=item B + +You must attribute the work in the manner specified by the author or +licensor (but not in any way that suggests that they endorse you or +your use of the work). + +=item B + +If you alter, transform, or build upon this work, you may distribute +the resulting work only under the same, similar or a compatible +license. + +=back + +With the understanding that: + +=over 9 + +=item B + +Any of the above conditions can be waived if you get permission from +the copyright holder. + +=item B + +Where the work or any of its elements is in the public domain under +applicable law, that status is in no way affected by the license. + +=item B + +In no way are any of the following rights affected by the license: + +=over 2 + +=item * + +Your fair dealing or fair use rights, or other applicable +copyright exceptions and limitations; + +=item * + +The author's moral rights; + +=item * + +Rights other persons may have either in the work itself or in +how the work is used, such as publicity or privacy rights. + +=back + +=back + +=over 9 + +=item B + +For any reuse or distribution, you must make clear to others the +license terms of this work. + +=back + +A copy of the full license is included in the file as cc-by-sa.txt. + +=head1 DEPENDENCIES + +GNU B uses Perl, and the Perl modules Getopt::Long, +Symbol, Fcntl. + + +=head1 SEE ALSO + +B(1) + +=cut + +use strict; +use Symbol qw(gensym); +use Getopt::Long; + +Getopt::Long::Configure ("bundling","require_order"); +GetOptions("debug|D" => \$::opt_D, + "id|i=s" => \$::opt_id, + "count|j=i" => \$::opt_count, + "fg" => \$::opt_fg, + "timeout|t=i" => \$::opt_timeout, + "version" => \$::opt_version, + "wait|w" => \$::opt_wait, + ) || die_usage(); +$Global::debug = $::opt_D; +$Global::version = 20100814; +$Global::progname = 'sem'; + +my $count = 1; # Default 1 = mutex +if($::opt_count) { + $count = $::opt_count + 1; +} +if($::opt_wait) { + $count = 1; +} +my $id = $::opt_id; +my $fg = $::opt_fg || $::opt_wait; +$::opt_timeout = $::opt_timeout; +if(defined $::opt_version) { + version(); +} + +if(not defined $id) { + # $id = getppid(); + # does not work with: + # find . -name '*linux*' -exec sem -j1000 "sleep 3; echo `tty` '{}'" \; ; sem --wait echo done + $id = `tty`; +} +$id = "id-$id"; +$id=~s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex +my $sem = Semaphore->new($id,$count); +$sem->acquire(); +debug("run"); +if($fg) { + system @ARGV; + $sem->release(); +} else { + # If run in the background, the PID will change + # therefore release and re-acquire the semaphore + $sem->release(); + if(not fork()) { + # child + # Get a semaphore for this pid + my $child_sem = Semaphore->new($id,$count); + $child_sem->acquire(); + system @ARGV; + $child_sem->release(); + } +} + +sub version { + # Returns: N/A + print join("\n", + "GNU $Global::progname $Global::version", + "Copyright (C) 2010 Ole Tange and Free Software Foundation, Inc.", + "License GPLv3+: GNU GPL version 3 or later ", + "This is free software: you are free to change and redistribute it.", + "GNU $Global::progname comes with no warranty.", + "", + "Web site: http://www.gnu.org/software/parallel\n" + ); +} + +sub usage { + # Returns: N/A + print "Usage:\n"; + print "$Global::progname [options] [command [arguments]] < list_of_arguments)\n"; + print "$Global::progname [options] [command [arguments]] ::: arguments\n"; + print "$Global::progname [options] [command [arguments]] :::: argfile(s)\n"; + print "\n"; + print "See 'man $Global::progname' for the options\n"; +} + +sub die_usage { + usage(); + exit(255); +} + +sub debug { + # Returns: N/A + $Global::debug or return; + @_ = grep { defined $_ ? $_ : "" } @_; + print map {$_,"\n" } @_; +} + +package Semaphore; + +# This package provides a counting semaphore +# +# If a process dies without releasing the semaphore the next process +# that needs that entry will clean up dead semaphores +# +# The semaphores are stored in ~/.parallel/semaphores/id- Each +# file in ~/.parallel/semaphores/id-/ is the process ID of the +# process holding the entry. If the process dies, the entry can be +# taken by another process. + +use Fcntl qw(:DEFAULT :flock); + +sub new { + my $class = shift; + my $id = shift; + my $count = shift; + my $parallel_locks = $ENV{'HOME'}."/.parallel/semaphores"; + -d $parallel_locks or mkdir $parallel_locks; + my $lockdir = "$parallel_locks/$id"; + my $lockfile = $lockdir.".lock"; + return bless { + 'lockfile' => $lockfile, + 'lockfh' => Symbol::gensym(), + 'lockdir' => $lockdir, + 'id' => $id, + 'idfile' => $lockdir."/".$id, + 'pid' => $$, + 'pidfile' => $lockdir."/".$$, + 'count' => $count + }, ref($class) || $class; +} + +sub acquire { + my $self = shift; + while(1) { + $self->atomic_link_if_count_less_than() and last; + ::debug("Remove dead locks"); + my $lockdir = $self->{'lockdir'}; + for my $d (<$lockdir/*>) { + $d =~ m:$lockdir/([0-9]+):o or next; + if(not kill 0, $1) { + ::debug("Dead: $d"); + unlink $d; + } else { + ::debug("Alive: $d"); + } + } + # try again + $self->atomic_link_if_count_less_than() and last; + sleep 1; + # TODO if timeout: last + } + ::debug("got $self->{'pid'}"); +} + +sub release { + my ($self) = shift; + unlink $self->{'pidfile'}; + if($self->nlinks() == 1) { + # This is the last link, so atomic cleanup + $self->lock(); + if($self->nlinks() == 1) { + unlink $self->{'idfile'}; + rmdir $self->{'lockdir'}; + } + $self->unlock(); + } + ::debug("released $self->{'pid'}"); +} + + +sub atomic_link_if_count_less_than { + # Link $file1 to $file2 if nlinks to $file1 < $count + my ($self) = shift; + my ($retval) = 0; + $self->lock(); + if($self->nlinks() < $count) { + -d $self->{'lockdir'} || mkdir $self->{'lockdir'}; + if(not -e $self->{'idfile'}) { + open (A, ">", $self->{'idfile'}) or die ">$self->{'idfile'}"; + close A; + } + $retval = link $self->{'idfile'}, $self->{'pidfile'}; + } + $self->unlock(); + ::debug("atomic $retval"); + return $retval; +} + +sub nlinks { + my $self = shift; + if(-e $self->{'idfile'}) { + return (stat(_))[3]; + } else { + return 0; + } +} + +sub lock { + my ($self) = shift; + open $self->{'lockfh'}, ">", $self->{'lockfile'} + or die "Can't open semaphore file $self->{'lockfile'}: $!"; + chmod 0666, $self->{'lockfile'}; # assuming you want it a+rw + while(not flock $self->{'lockfh'}, LOCK_EX()|LOCK_NB()) { + ::debug("Cannot lock $self->{'lockfile'}"); + # TODO if timeout: last + sleep 1; + } + ::debug("locked $self->{'lockfile'}"); +} + +sub unlock { + my $self = shift; + unlink $self->{'lockfile'}; + close $self->{'lockfh'}; + ::debug("unlocked"); +} diff --git a/unittest/tests-to-run/sem01.sh b/unittest/tests-to-run/sem01.sh index c5ded6cb..de175934 100755 --- a/unittest/tests-to-run/sem01.sh +++ b/unittest/tests-to-run/sem01.sh @@ -6,10 +6,10 @@ parallel -u --semaphore seq 11 20 '|' pv -qL 100 parallel --semaphore --wait echo done -echo '### Test default id = --id `tty`' +echo '### Test default id = --id `tty` and --semaphorename' parallel --id `tty` -u --semaphore seq 1 10 '|' pv -qL 20 parallel -u --semaphore seq 11 20 '|' pv -qL 100 -parallel --id `tty` --semaphore --wait +parallel --semaphorename `tty` --semaphore --wait echo done echo '### Test semaphore 2 jobs running simultaneously' @@ -30,3 +30,10 @@ for i in 0.5 0.1 0.2 0.3 0.4 ; do sem -j+0 sleep $i ";" echo done $i done sem --wait + +echo '### BUG: Test --fg followed by --bg' +parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20 +parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20 +parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20 +parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20 +sem --wait \ No newline at end of file diff --git a/unittest/wanted-results/sem01 b/unittest/wanted-results/sem01 index 774090ae..3fa79f69 100644 --- a/unittest/wanted-results/sem01 +++ b/unittest/wanted-results/sem01 @@ -20,7 +20,7 @@ 19 20 done -### Test default id = --id `tty` +### Test default id = --id `tty` and --semaphorename 1 2 3 @@ -61,7 +61,48 @@ done done 0.1 0.3 done 0.5 -done 0.2 0.4 +done 0.2 done 0.3 done 0.4 +### BUG: Test --fg followed by --bg +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40