Merge branch 'master' of git://git.savannah.gnu.org/parallel

This commit is contained in:
Ole Tange 2012-09-03 15:02:29 +02:00
commit 32eed243b8
38 changed files with 667 additions and 472 deletions

View file

@ -1,4 +1,4 @@
# Makefile.in generated by automake 1.11.3 from Makefile.am.
# Makefile.in generated by automake 1.11.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
@ -15,6 +15,23 @@
@SET_MAKE@
VPATH = @srcdir@
am__make_dryrun = \
{ \
am__dry=no; \
case $$MAKEFLAGS in \
*\\[\ \ ]*) \
echo 'am--echo: ; @echo "AM" OK' | $(MAKE) -f - 2>/dev/null \
| grep '^AM OK$$' >/dev/null || am__dry=yes;; \
*) \
for am__flg in $$MAKEFLAGS; do \
case $$am__flg in \
*=*|--*) ;; \
*n*) am__dry=yes; break;; \
esac; \
done;; \
esac; \
test $$am__dry = yes; \
}
pkgdatadir = $(datadir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
@ -54,6 +71,11 @@ RECURSIVE_TARGETS = all-recursive check-recursive dvi-recursive \
install-pdf-recursive install-ps-recursive install-recursive \
installcheck-recursive installdirs-recursive pdf-recursive \
ps-recursive uninstall-recursive
am__can_run_installinfo = \
case $$AM_UPDATE_INFO_DIR in \
n|no|NO) false;; \
*) (install-info --version) >/dev/null 2>&1;; \
esac
RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \
distclean-recursive maintainer-clean-recursive
AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \
@ -398,13 +420,10 @@ distdir: $(DISTFILES)
done
@list='$(DIST_SUBDIRS)'; for subdir in $$list; do \
if test "$$subdir" = .; then :; else \
test -d "$(distdir)/$$subdir" \
|| $(MKDIR_P) "$(distdir)/$$subdir" \
|| exit 1; \
fi; \
done
@list='$(DIST_SUBDIRS)'; for subdir in $$list; do \
if test "$$subdir" = .; then :; else \
$(am__make_dryrun) \
|| test -d "$(distdir)/$$subdir" \
|| $(MKDIR_P) "$(distdir)/$$subdir" \
|| exit 1; \
dir1=$$subdir; dir2="$(distdir)/$$subdir"; \
$(am__relativize); \
new_distdir=$$reldir; \
@ -490,7 +509,7 @@ distcheck: dist
*.zip*) \
unzip $(distdir).zip ;;\
esac
chmod -R a-w $(distdir); chmod a+w $(distdir)
chmod -R a-w $(distdir); chmod u+w $(distdir)
mkdir $(distdir)/_build
mkdir $(distdir)/_inst
chmod a-w $(distdir)

11
NEWS
View file

@ -1,3 +1,14 @@
20120822
* Due to a bugfix the main flow has changed. Making this release beta
quality.
* Parallel Proxy Scraper & Checker Tool
https://www.youtube.com/watch?v=iXCeR_XsP6o
* Bug fixes and man page updates.
20120722
* GNU Parallel was presented at Chiang Mai Bar Camp

15
README
View file

@ -24,9 +24,9 @@ you would get had you run the commands sequentially. This makes it
possible to use output from GNU Parallel as input for other programs.
= Installation =
= Full installation =
Installation of GNU Parallel is as simple as:
Full installation of GNU Parallel is as simple as:
./configure && make && make install
@ -38,10 +38,19 @@ If you are not root you can add ~/bin to your path and install in
Or if your system lacks 'make' you can simply copy src/parallel
src/sem src/niceload src/sql to a dir in your path.
If you just need parallel in a pinch:
= Minimal installation =
If you just need parallel and do not have 'make' installed (maybe the
system is old or Microsoft Windows):
wget http://git.savannah.gnu.org/cgit/parallel.git/plain/src/parallel
chmod 755 parallel
cp parallel sem
mv parallel sem dir-in-your-$PATH/bin/
= Test the installation =
After this you should be able to do:

10
aclocal.m4 vendored
View file

@ -1,4 +1,4 @@
# generated automatically by aclocal 1.11.3 -*- Autoconf -*-
# generated automatically by aclocal 1.11.6 -*- Autoconf -*-
# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
# 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
@ -14,8 +14,8 @@
m4_ifndef([AC_AUTOCONF_VERSION],
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.68],,
[m4_warning([this file was generated for autoconf 2.68.
m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.69],,
[m4_warning([this file was generated for autoconf 2.69.
You have another version of autoconf. It may work, but is not guaranteed to.
If you have problems, you may need to regenerate the build system entirely.
To do so, use the procedure documented by the package, typically `autoreconf'.])])
@ -38,7 +38,7 @@ AC_DEFUN([AM_AUTOMAKE_VERSION],
[am__api_version='1.11'
dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to
dnl require some minimum version. Point them to the right macro.
m4_if([$1], [1.11.3], [],
m4_if([$1], [1.11.6], [],
[AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl
])
@ -54,7 +54,7 @@ m4_define([_AM_AUTOCONF_VERSION], [])
# Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced.
# This function is AC_REQUIREd by AM_INIT_AUTOMAKE.
AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
[AM_AUTOMAKE_VERSION([1.11.3])dnl
[AM_AUTOMAKE_VERSION([1.11.6])dnl
m4_ifndef([AC_AUTOCONF_VERSION],
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])

196
configure vendored
View file

@ -1,13 +1,11 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.68 for parallel 20120722.
# Generated by GNU Autoconf 2.69 for parallel 20120822.
#
# Report bugs to <bug-parallel@gnu.org>.
#
#
# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software
# Foundation, Inc.
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
#
#
# This configure script is free software; the Free Software Foundation
@ -136,6 +134,31 @@ export LANGUAGE
# CDPATH.
(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
# Use a proper internal environment variable to ensure we don't fall
# into an infinite loop, continuously re-executing ourselves.
if test x"${_as_can_reexec}" != xno && test "x$CONFIG_SHELL" != x; then
_as_can_reexec=no; export _as_can_reexec;
# We cannot yet assume a decent shell, so we have to provide a
# neutralization value for shells without unset; and this also
# works around shells that cannot unset nonexistent variables.
# Preserve -v and -x to the replacement shell.
BASH_ENV=/dev/null
ENV=/dev/null
(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV
case $- in # ((((
*v*x* | *x*v* ) as_opts=-vx ;;
*v* ) as_opts=-v ;;
*x* ) as_opts=-x ;;
* ) as_opts= ;;
esac
exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"}
# Admittedly, this is quite paranoid, since all the known shells bail
# out after a failed `exec'.
$as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2
as_fn_exit 255
fi
# We don't want this to propagate to other subprocesses.
{ _as_can_reexec=; unset _as_can_reexec;}
if test "x$CONFIG_SHELL" = x; then
as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then :
emulate sh
@ -169,7 +192,8 @@ if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then :
else
exitcode=1; echo positional parameters were not saved.
fi
test x\$exitcode = x0 || exit 1"
test x\$exitcode = x0 || exit 1
test -x / || exit 1"
as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO
as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO
eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" &&
@ -213,21 +237,25 @@ IFS=$as_save_IFS
if test "x$CONFIG_SHELL" != x; then :
# We cannot yet assume a decent shell, so we have to provide a
# neutralization value for shells without unset; and this also
# works around shells that cannot unset nonexistent variables.
# Preserve -v and -x to the replacement shell.
BASH_ENV=/dev/null
ENV=/dev/null
(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV
export CONFIG_SHELL
case $- in # ((((
*v*x* | *x*v* ) as_opts=-vx ;;
*v* ) as_opts=-v ;;
*x* ) as_opts=-x ;;
* ) as_opts= ;;
esac
exec "$CONFIG_SHELL" $as_opts "$as_myself" ${1+"$@"}
export CONFIG_SHELL
# We cannot yet assume a decent shell, so we have to provide a
# neutralization value for shells without unset; and this also
# works around shells that cannot unset nonexistent variables.
# Preserve -v and -x to the replacement shell.
BASH_ENV=/dev/null
ENV=/dev/null
(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV
case $- in # ((((
*v*x* | *x*v* ) as_opts=-vx ;;
*v* ) as_opts=-v ;;
*x* ) as_opts=-x ;;
* ) as_opts= ;;
esac
exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"}
# Admittedly, this is quite paranoid, since all the known shells bail
# out after a failed `exec'.
$as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2
exit 255
fi
if test x$as_have_required = xno; then :
@ -330,6 +358,14 @@ $as_echo X"$as_dir" |
} # as_fn_mkdir_p
# as_fn_executable_p FILE
# -----------------------
# Test if FILE is an executable regular file.
as_fn_executable_p ()
{
test -f "$1" && test -x "$1"
} # as_fn_executable_p
# as_fn_append VAR VALUE
# ----------------------
# Append the text in VALUE to the end of the definition contained in VAR. Take
@ -451,6 +487,10 @@ as_cr_alnum=$as_cr_Letters$as_cr_digits
chmod +x "$as_me.lineno" ||
{ $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; }
# If we had to re-execute with $CONFIG_SHELL, we're ensured to have
# already done that, so ensure we don't try to do so again and fall
# in an infinite loop. This has already happened in practice.
_as_can_reexec=no; export _as_can_reexec
# Don't try to exec as it changes $[0], causing all sort of problems
# (the dirname of $[0] is not the place where we might find the
# original and so on. Autoconf is especially sensitive to this).
@ -485,16 +525,16 @@ if (echo >conf$$.file) 2>/dev/null; then
# ... but there are two gotchas:
# 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
# 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
# In both cases, we have to default to `cp -p'.
# In both cases, we have to default to `cp -pR'.
ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
as_ln_s='cp -p'
as_ln_s='cp -pR'
elif ln conf$$.file conf$$ 2>/dev/null; then
as_ln_s=ln
else
as_ln_s='cp -p'
as_ln_s='cp -pR'
fi
else
as_ln_s='cp -p'
as_ln_s='cp -pR'
fi
rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
rmdir conf$$.dir 2>/dev/null
@ -506,28 +546,8 @@ else
as_mkdir_p=false
fi
if test -x / >/dev/null 2>&1; then
as_test_x='test -x'
else
if ls -dL / >/dev/null 2>&1; then
as_ls_L_option=L
else
as_ls_L_option=
fi
as_test_x='
eval sh -c '\''
if test -d "$1"; then
test -d "$1/.";
else
case $1 in #(
-*)set "./$1";;
esac;
case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #((
???[sx]*):;;*)false;;esac;fi
'\'' sh
'
fi
as_executable_p=$as_test_x
as_test_x='test -x'
as_executable_p=as_fn_executable_p
# Sed expression to map a string onto a valid CPP name.
as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
@ -559,8 +579,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20120722'
PACKAGE_STRING='parallel 20120722'
PACKAGE_VERSION='20120822'
PACKAGE_STRING='parallel 20120822'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1089,8 +1109,6 @@ target=$target_alias
if test "x$host_alias" != x; then
if test "x$build_alias" = x; then
cross_compiling=maybe
$as_echo "$as_me: WARNING: if you wanted to set the --build type, don't use --host.
If a cross compiler is detected then cross compile mode will be used" >&2
elif test "x$build_alias" != "x$host_alias"; then
cross_compiling=yes
fi
@ -1176,7 +1194,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20120722 to adapt to many kinds of systems.
\`configure' configures parallel 20120822 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1242,7 +1260,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20120722:";;
short | recursive ) echo "Configuration of parallel 20120822:";;
esac
cat <<\_ACEOF
@ -1309,10 +1327,10 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20120722
generated by GNU Autoconf 2.68
parallel configure 20120822
generated by GNU Autoconf 2.69
Copyright (C) 2010 Free Software Foundation, Inc.
Copyright (C) 2012 Free Software Foundation, Inc.
This configure script is free software; the Free Software Foundation
gives unlimited permission to copy, distribute and modify it.
_ACEOF
@ -1326,8 +1344,8 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20120722, which was
generated by GNU Autoconf 2.68. Invocation command line was
It was created by parallel $as_me 20120822, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -1742,7 +1760,7 @@ case $as_dir/ in #((
# by default.
for ac_prog in ginstall scoinst install; do
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then
if as_fn_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
if test $ac_prog = install &&
grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
# AIX install. It has an incompatible calling convention.
@ -1911,7 +1929,7 @@ do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_prog_STRIP="${ac_tool_prefix}strip"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
@ -1951,7 +1969,7 @@ do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_prog_ac_ct_STRIP="strip"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
@ -2002,7 +2020,7 @@ do
test -z "$as_dir" && as_dir=.
for ac_prog in mkdir gmkdir; do
for ac_exec_ext in '' $ac_executable_extensions; do
{ test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; } || continue
as_fn_executable_p "$as_dir/$ac_prog$ac_exec_ext" || continue
case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
'mkdir (GNU coreutils) '* | \
'mkdir (coreutils) '* | \
@ -2055,7 +2073,7 @@ do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_prog_AWK="$ac_prog"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
@ -2141,7 +2159,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20120722'
VERSION='20120822'
cat >>confdefs.h <<_ACEOF
@ -2593,16 +2611,16 @@ if (echo >conf$$.file) 2>/dev/null; then
# ... but there are two gotchas:
# 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
# 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
# In both cases, we have to default to `cp -p'.
# In both cases, we have to default to `cp -pR'.
ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
as_ln_s='cp -p'
as_ln_s='cp -pR'
elif ln conf$$.file conf$$ 2>/dev/null; then
as_ln_s=ln
else
as_ln_s='cp -p'
as_ln_s='cp -pR'
fi
else
as_ln_s='cp -p'
as_ln_s='cp -pR'
fi
rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
rmdir conf$$.dir 2>/dev/null
@ -2662,28 +2680,16 @@ else
as_mkdir_p=false
fi
if test -x / >/dev/null 2>&1; then
as_test_x='test -x'
else
if ls -dL / >/dev/null 2>&1; then
as_ls_L_option=L
else
as_ls_L_option=
fi
as_test_x='
eval sh -c '\''
if test -d "$1"; then
test -d "$1/.";
else
case $1 in #(
-*)set "./$1";;
esac;
case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #((
???[sx]*):;;*)false;;esac;fi
'\'' sh
'
fi
as_executable_p=$as_test_x
# as_fn_executable_p FILE
# -----------------------
# Test if FILE is an executable regular file.
as_fn_executable_p ()
{
test -f "$1" && test -x "$1"
} # as_fn_executable_p
as_test_x='test -x'
as_executable_p=as_fn_executable_p
# Sed expression to map a string onto a valid CPP name.
as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
@ -2704,8 +2710,8 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20120722, which was
generated by GNU Autoconf 2.68. Invocation command line was
This file was extended by parallel $as_me 20120822, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
CONFIG_HEADERS = $CONFIG_HEADERS
@ -2766,11 +2772,11 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20120722
configured by $0, generated by GNU Autoconf 2.68,
parallel config.status 20120822
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
Copyright (C) 2010 Free Software Foundation, Inc.
Copyright (C) 2012 Free Software Foundation, Inc.
This config.status script is free software; the Free Software Foundation
gives unlimited permission to copy, distribute and modify it."
@ -2861,7 +2867,7 @@ fi
_ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
if \$ac_cs_recheck; then
set X '$SHELL' '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
shift
\$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6
CONFIG_SHELL='$SHELL'

View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20120722], [bug-parallel@gnu.org])
AC_INIT([parallel], [20120822], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([

View file

@ -171,24 +171,18 @@ cc:Sandro Cazzaniga <kharec@mandriva.org>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20120722 ('Nai Soi') released
Subject: GNU Parallel 20120822 ('Boson') released
GNU Parallel 20127022 ('Nai Soi') has been released. It is
GNU Parallel 20120822 ('Boson') has been released. It is
available for download at: http://ftp.gnu.org/gnu/parallel/
This is a bugfix release with no new features. Probably a good release
for stable long-term use.
New in this release:
* GNU Parallel was presented at Chiang Mai Bar Camp
http://barcampchiangmai.org/
See write up here:
http://www.cnx-software.com/2012/07/02/use-gnu-parallel-to-speed-up-script-execution-on-multiple-cores-andor-machines/
* Due to a bugfix the main flow has changed. Making this release beta
quality.
* I don't always max out all my CPUs, but when I do, I do it with GNU
Parallel.
http://memegenerator.net/instance/22638454
* Parallel Proxy Scraper & Checker Tool
https://www.youtube.com/watch?v=iXCeR_XsP6o
* Bug fixes and man page updates.

View file

@ -121,11 +121,19 @@ xargs or cat | sh.</description>
<implementation id="sha1new=006f918cda3a7cbe620f6e2b66123379c1948089" released="2012-06-23" version="20120622">
<manifest-digest sha256="b1182ecf93c1b93e0cf3f22784c27c01f7aa64801f038b652624c3e210e03dea"/>
<archive extract="parallel-20120622" href="http://ftp.gnu.org/gnu/parallel/parallel-20120622.tar.bz2" size="228881"/>
</implementation>
<implementation id="sha1new=36a6f13e597193d7d163ca5c6a5097eceecc6dab" released="2012-08-23" version="20120722">
<manifest-digest sha256="d10af8e443c12b8540a6b8ecc41f0ca480d851f45b1b93bac3001d1a5031d76c"/>
<archive extract="parallel-20120722" href="http://ftp.gnu.org/gnu/parallel/parallel-20120722.tar.bz2" size="230141"/>
</implementation>
<implementation id="sha1new=f5da0850d901d05b9086bf5599d72ba30f7d9685" released="2012-08-23" version="20120822">
<manifest-digest sha256="98efd866ca5cf6737922d1caf3802488ce572fadd1b643161a3a0836d8ad1790"/>
<archive extract="parallel-20120822" href="http://ftp.gnu.org/gnu/parallel/parallel-20120822.tar.bz2" size="231076"/>
</implementation>
</group>
</interface>
<!-- Base64 Signature
iEYEABECAAYFAk/lnIgACgkQ/lhgK1iJTtKfrgCeIjYITindJa50NjPqxCes8e7kF1MAn0qMTpwx
cvnKdRDwC1px3+HQg435
iEYEABECAAYFAlA1WzcACgkQ/lhgK1iJTtKhkACgxcdVkWLq2uqYma1g+qHKO9vfNJwAni1y3U2u
AKT0/BQ6dfyxD1y2zqru
-->

View file

@ -1,6 +1,6 @@
Summary: Shell tool for executing jobs in parallel
Name: parallel
Version: 20120722
Version: 20120822
Release: 1
License: GPL
Group: Productivity/File utilities

View file

@ -1,4 +1,4 @@
# Makefile.in generated by automake 1.11.3 from Makefile.am.
# Makefile.in generated by automake 1.11.6 from Makefile.am.
# @configure_input@
# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
@ -17,6 +17,23 @@
VPATH = @srcdir@
am__make_dryrun = \
{ \
am__dry=no; \
case $$MAKEFLAGS in \
*\\[\ \ ]*) \
echo 'am--echo: ; @echo "AM" OK' | $(MAKE) -f - 2>/dev/null \
| grep '^AM OK$$' >/dev/null || am__dry=yes;; \
*) \
for am__flg in $$MAKEFLAGS; do \
case $$am__flg in \
*=*|--*) ;; \
*n*) am__dry=yes; break;; \
esac; \
done;; \
esac; \
test $$am__dry = yes; \
}
pkgdatadir = $(datadir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
@ -75,6 +92,11 @@ am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)" \
SCRIPTS = $(bin_SCRIPTS)
SOURCES =
DIST_SOURCES =
am__can_run_installinfo = \
case $$AM_UPDATE_INFO_DIR in \
n|no|NO) false;; \
*) (install-info --version) >/dev/null 2>&1;; \
esac
man1dir = $(mandir)/man1
NROFF = nroff
MANS = $(man_MANS)
@ -197,8 +219,11 @@ $(ACLOCAL_M4): $(am__aclocal_m4_deps)
$(am__aclocal_m4_deps):
install-binSCRIPTS: $(bin_SCRIPTS)
@$(NORMAL_INSTALL)
test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
@list='$(bin_SCRIPTS)'; test -n "$(bindir)" || list=; \
if test -n "$$list"; then \
echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \
$(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \
fi; \
for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \
@ -229,11 +254,18 @@ uninstall-binSCRIPTS:
dir='$(DESTDIR)$(bindir)'; $(am__uninstall_files_from_dir)
install-man1: $(man_MANS)
@$(NORMAL_INSTALL)
test -z "$(man1dir)" || $(MKDIR_P) "$(DESTDIR)$(man1dir)"
@list=''; test -n "$(man1dir)" || exit 0; \
{ for i in $$list; do echo "$$i"; done; \
l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
sed -n '/\.1[a-z]*$$/p'; \
@list1=''; \
list2='$(man_MANS)'; \
test -n "$(man1dir)" \
&& test -n "`echo $$list1$$list2`" \
|| exit 0; \
echo " $(MKDIR_P) '$(DESTDIR)$(man1dir)'"; \
$(MKDIR_P) "$(DESTDIR)$(man1dir)" || exit 1; \
{ for i in $$list1; do echo "$$i"; done; \
if test -n "$$list2"; then \
for i in $$list2; do echo "$$i"; done \
| sed -n '/\.1[a-z]*$$/p'; \
fi; \
} | while read p; do \
if test -f $$p; then d=; else d="$(srcdir)/"; fi; \
echo "$$d$$p"; echo "$$p"; \
@ -265,8 +297,11 @@ uninstall-man1:
dir='$(DESTDIR)$(man1dir)'; $(am__uninstall_files_from_dir)
install-docDATA: $(doc_DATA)
@$(NORMAL_INSTALL)
test -z "$(docdir)" || $(MKDIR_P) "$(DESTDIR)$(docdir)"
@list='$(doc_DATA)'; test -n "$(docdir)" || list=; \
if test -n "$$list"; then \
echo " $(MKDIR_P) '$(DESTDIR)$(docdir)'"; \
$(MKDIR_P) "$(DESTDIR)$(docdir)" || exit 1; \
fi; \
for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
echo "$$d$$p"; \

View file

@ -24,7 +24,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20120722;
$Global::version = 20120822;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($::opt_version) {

View file

@ -34,8 +34,8 @@ use strict;
$::oodebug=0;
$SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X
if(not $ENV{SHELL}) {
# $ENV{SHELL} is sometimes not set on Mac OS X
print STDERR ("parallel: Warning: \$SHELL not set. Using /bin/sh\n");
# $ENV{SHELL} is sometimes not set on Mac OS X and Windows
::warning("\$SHELL not set. Using /bin/sh.\n");
$ENV{SHELL} = "/bin/sh";
}
%Global::original_sig = %SIG;
@ -110,9 +110,8 @@ if($::opt_filter_hosts) {
} elsif($col[6] eq "127") {
# signal == 127: parallel not installed remote
# Set ncpus and ncores = 1
print $Global::original_stderr
("parallel: Warning: Could not figure out ",
"number of cpus on $col[1]. Using 1\n");
::warning("Could not figure out ",
"number of cpus on $col[1]. Using 1.\n");
$ncores{$col[1]} = 1;
$ncpus{$col[1]} = 1;
$maxlen{$col[1]} = Limits::Command::max_length();
@ -349,7 +348,7 @@ sub spreadstdin {
}
}
}
# If there is anything left in the buffer write it
substr($buf,0,0) = "";
write_record_to_pipe(\$header,\$buf,$recstart,$recend,length $buf);
@ -408,7 +407,7 @@ sub write_record_to_pipe {
$job->write(\$record);
my $fh = $job->stdin();
close $fh;
exit;
exit(0);
}
my $fh = $job->stdin();
close $fh;
@ -510,6 +509,7 @@ sub options_hash {
"arg-sep|argsep=s" => \$::opt_arg_sep,
"arg-file-sep|argfilesep=s" => \$::opt_arg_file_sep,
"trim=s" => \$::opt_trim,
"plain" => \$::opt_plain,
"profile|J=s" => \@::opt_profile,
"pipe|spreadstdin" => \$::opt_pipe,
"recstart=s" => \$::opt_recstart,
@ -584,7 +584,7 @@ sub get_options_from_array {
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20120722;
$Global::version = 20120822;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -614,13 +614,13 @@ sub parse_options {
@ARGV=read_options();
if(defined $::opt_retired) {
print STDERR "$Global::progname: -g has been retired. Use --group.\n";
print STDERR "$Global::progname: -B has been retired. Use --bf.\n";
print STDERR "$Global::progname: -T has been retired. Use --tty.\n";
print STDERR "$Global::progname: -U has been retired. Use --er.\n";
print STDERR "$Global::progname: -W has been retired. Use --wd.\n";
print STDERR "$Global::progname: -Y has been retired. Use --shebang.\n";
print STDERR "$Global::progname: -H has been retired. Use --halt.\n";
::error("-g has been retired. Use --group.\n");
::error("-B has been retired. Use --bf.\n");
::error("-T has been retired. Use --tty.\n");
::error("-U has been retired. Use --er.\n");
::error("-W has been retired. Use --wd.\n");
::error("-Y has been retired. Use --shebang.\n");
::error("-H has been retired. Use --halt.\n");
::wait_and_exit(255);
}
if(@::opt_v) { $Global::verbose = $#::opt_v+1; } # Convert -v -v to v=2
@ -690,7 +690,7 @@ sub parse_options {
wait_and_exit(0);
}
}
if($::opt_tollef and not $::opt_gnu) {
if($::opt_tollef and not $::opt_gnu and not $::opt_plain) {
# Behave like tollef parallel (from moreutils)
$::opt_u = 1;
$Global::grouped = 0;
@ -791,8 +791,8 @@ sub parse_options {
$::opt_P = 1;
}
if($Global::interactive and $::opt_bg) {
print STDERR "$Global::progname: Jobs running in the ".
"background cannot be interactive.\n";
::error("Jobs running in the ".
"background cannot be interactive.\n");
::wait_and_exit(255);
}
}
@ -806,8 +806,7 @@ sub parse_options {
# As we do not know the max line length on the remote machine
# long commands generated by xargs may fail
# If opt_N is set, it is probably safe
print STDERR ("$Global::progname: Warning: using -X or -m ",
"with --sshlogin may fail\n");
::warning("Using -X or -m with --sshlogin may fail.\n");
}
if(not defined $::opt_P) {
@ -819,7 +818,7 @@ sub parse_options {
sub open_joblog {
my $append = 0;
if($::opt_resume and not $::opt_joblog) {
print STDERR ("$Global::progname: --resume requires --joblog\n");
::error("--resume requires --joblog.\n");
::wait_and_exit(255);
}
if($::opt_joblog) {
@ -832,7 +831,7 @@ sub open_joblog {
# This is 30% faster than set_job_already_run($1);
vec($Global::job_already_run,$1,1) = 1;
} else {
print STDERR ("$Global::progname: Format of '$::opt_joblog' is wrong\n");
::error("Format of '$::opt_joblog' is wrong.\n");
::wait_and_exit(255);
}
}
@ -842,15 +841,13 @@ sub open_joblog {
if($append) {
# Append to joblog
if(not open($Global::joblog,">>$::opt_joblog")) {
print STDERR ("$Global::progname: Cannot append to ",
"--joblog $::opt_joblog\n");
::error("Cannot append to --joblog $::opt_joblog.\n");
::wait_and_exit(255);
}
} else {
# Overwrite the joblog
if(not open($Global::joblog,">$::opt_joblog")) {
print STDERR ("$Global::progname: Cannot write to ",
"--joblog $::opt_joblog\n");
::error("Cannot write to --joblog $::opt_joblog.\n");
::wait_and_exit(255);
} else {
print $Global::joblog
@ -879,44 +876,49 @@ sub read_options {
Getopt::Long::Configure("bundling","pass_through");
# Check if there is a --profile to set @::opt_profile
GetOptions("profile|J=s" => \@::opt_profile) || die_usage();
# Add options from .parallel/config and other profiles
GetOptions("profile|J=s" => \@::opt_profile,
"plain" => \$::opt_plain) || die_usage();
my @ARGV_profile = ();
my @ARGV_env = ();
my @config_profiles = (
"/etc/parallel/config",
$ENV{'HOME'}."/.parallel/config",
$ENV{'HOME'}."/.parallelrc");
my @profiles = @config_profiles;
if(@::opt_profile) {
# --profile overrides default profiles
@profiles = ();
for my $profile (@::opt_profile) {
push @profiles, $ENV{'HOME'}."/.parallel/".$profile;
if(not $::opt_plain) {
# Add options from .parallel/config and other profiles
my @config_profiles = (
"/etc/parallel/config",
$ENV{'HOME'}."/.parallel/config",
$ENV{'HOME'}."/.parallelrc");
my @profiles = @config_profiles;
if(@::opt_profile) {
# --profile overrides default profiles
@profiles = ();
for my $profile (@::opt_profile) {
push @profiles, $ENV{'HOME'}."/.parallel/".$profile;
}
}
for my $profile (@profiles) {
if(-r $profile) {
open (IN, "<", $profile) || ::die_bug("read-profile: $profile");
while(<IN>) {
/^\s*\#/ and next;
chomp;
push @ARGV_profile, shell_unquote(split/(?<![\\])\s/, $_);
}
close IN;
} else {
if(grep /^$profile$/, @config_profiles) {
# config file is not required to exist
} else {
::error("$profile not readable.\n");
wait_and_exit(255);
}
}
}
# Add options from shell variable $PARALLEL
if($ENV{'PARALLEL'}) {
@ARGV_env = shell_unquote(split/(?<![\\])\s/, $ENV{'PARALLEL'});
}
}
for my $profile (@profiles) {
if(-r $profile) {
open (IN, "<", $profile) || ::die_bug("read-profile: $profile");
while(<IN>) {
/^\s*\#/ and next;
chomp;
push @ARGV_profile, shell_unquote(split/(?<![\\])\s/, $_);
}
close IN;
} else {
if(grep /^$profile$/, @config_profiles) {
# config file is not required to exist
} else {
print STDERR "$Global::progname: $profile not readable\n";
wait_and_exit(255);
}
}
}
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV_profile) || die_usage();
# Add options from shell variable $PARALLEL
$ENV{'PARALLEL'} and @ARGV_env = shell_unquote(split/(?<![\\])\s/, $ENV{'PARALLEL'});
get_options_from_array(\@ARGV_env) || die_usage();
get_options_from_array(\@ARGV) || die_usage();
@ -1096,9 +1098,7 @@ sub open_or_exit {
}
my $fh = gensym;
if(not open($fh,"<",$file)) {
print STDERR "$Global::progname: ".
"Cannot open input file `$file': ".
"No such file or directory\n";
::error("Cannot open input file `$file': No such file or directory.\n");
wait_and_exit(255);
}
return $fh;
@ -1220,8 +1220,19 @@ sub start_another_job {
debug("Started as seq ",$job->seq()," pid:",$job->pid(),"\n");
return 1;
} else {
# If interactive says: Dont run the job, then skip it and run the next
return start_another_job($sshlogin);
# Not enough processes to run the job.
# Put it back on the queue.
$Global::JobQueue->unget($job);
# Count down the number of jobs to run for this SSHLogin.
my $max = $sshlogin->max_jobs_running();
if($max > 1) { $max--; }
$sshlogin->set_max_jobs_running($max);
# Sleep up to 300 ms to give other processes time to die
::usleep(rand()*300);
::warning("No more processes: ",
"Decreasing number of running jobs to $max. ",
"Raising ulimit -u may help.\n");
return 0;
}
}
} else {
@ -1547,8 +1558,8 @@ sub read_sshloginfile {
$close = 0;
} else {
if(not open(IN, $file)) {
print $Global::original_stderr "Cannot open $file\n";
exit(255);
::error("Cannot open $file.\n");
::wait_and_exit(255);
}
}
while(<IN>) {
@ -1586,20 +1597,15 @@ sub parse_sshlogin {
if(not remote_hosts()) {
# There are no remote hosts
if(@::opt_trc) {
print $Global::original_stderr
"parallel: Warning: --trc ignored as there are no remote --sshlogin\n";
::warning("--trc ignored as there are no remote --sshlogin.\n");
} elsif (defined $::opt_transfer) {
print $Global::original_stderr
"parallel: Warning: --transfer ignored as there are no remote --sshlogin\n";
::warning("--transfer ignored as there are no remote --sshlogin.\n");
} elsif (@::opt_return) {
print $Global::original_stderr
"parallel: Warning: --return ignored as there are no remote --sshlogin\n";
::warning("--return ignored as there are no remote --sshlogin.\n");
} elsif (defined $::opt_cleanup) {
print $Global::original_stderr
"parallel: Warning: --cleanup ignored as there are no remote --sshlogin\n";
::warning("--cleanup ignored as there are no remote --sshlogin.\n");
} elsif (@::opt_basefile) {
print $Global::original_stderr
"parallel: Warning: --basefile ignored as there are no remote --sshlogin\n";
::warning("--basefile ignored as there are no remote --sshlogin.\n");
}
}
}
@ -1823,6 +1829,22 @@ sub usage {
"");
}
sub warning {
my @w = @_;
my $fh = $Global::original_stderr || *STDERR;
my $prog = $Global::progname || "parallel";
print $fh $prog, ": Warning: ", @w;
}
sub error {
my @w = @_;
my $fh = $Global::original_stderr || *STDERR;
my $prog = $Global::progname || "parallel";
print $fh $prog, ": Error: ", @w;
}
sub die_bug {
my $bugid = shift;
print STDERR
@ -2362,7 +2384,7 @@ sub compute_max_loadavg {
$load = $self->compute_max_loadavg($opt_load_file);
} else {
print $Global::original_stderr "Cannot open $loadspec\n";
exit(255);
::wait_and_exit(255);
}
} else {
print $Global::original_stderr "Parsing of --load failed\n";
@ -2407,11 +2429,6 @@ sub compute_number_of_processes {
::debug("Wanted procs: $wanted_processes\n");
my $system_limit =
$self->processes_available_by_system_limit($wanted_processes);
if($system_limit < 1) {
print STDERR "$Global::progname: Cannot spawn more jobs. ",
"Raising ulimit -u may help.\n";
exit(255);
}
::debug("Limited to procs: $system_limit\n");
return $system_limit;
}
@ -2537,20 +2554,6 @@ sub processes_available_by_system_limit {
$slow_spawining_warning_printed = 1;
}
}
if($system_limit < $wanted_processes and not $more_filehandles) {
print $Global::original_stderr
("parallel: Warning: Only enough filehandles to run ",
$system_limit, " jobs in parallel. ",
"Raising ulimit -n may help.\n");
}
if($system_limit < $wanted_processes and $max_system_proc_reached) {
print $Global::original_stderr
("parallel: Warning: Only enough available processes to run ",
$system_limit, " jobs in parallel.\n");
}
if($Global::JobQueue->empty()) {
$system_limit ||= 1;
}
# Cleanup: Close the files
for (values %fh) { close $_ }
# Cleanup: Kill the children
@ -2559,10 +2562,27 @@ sub processes_available_by_system_limit {
waitpid($pid,0);
delete $Global::unkilled_children{$pid};
}
#wait();
# Cleanup: Unget the command_lines or the @args
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget(@args);
$Global::JobQueue->unget(@jobs);
if($system_limit < $wanted_processes) {
# The system_limit is less than the wanted_processes
if($system_limit < 1 and not $Global::JobQueue->empty()) {
::warning("Cannot spawn any jobs. Raising ulimit -u may help.\n");
::wait_and_exit(255);
}
if(not $more_filehandles) {
::warning("Only enough filehandles to run ", $system_limit,
" jobs in parallel. Raising ulimit -n may help.\n");
}
if($max_system_proc_reached) {
::warning("Only enough available processes to run ", $system_limit,
" jobs in parallel. Raising ulimit -u may help.\n");
}
}
if($Global::JobQueue->empty()) {
$system_limit ||= 1;
}
if($self->string() ne ":" and
$system_limit > $Global::default_simultaneous_sshlogins) {
$system_limit =
@ -2584,13 +2604,12 @@ sub simultaneous_sshlogin_limit {
$self->simultaneous_sshlogin($wanted_processes));
if($ssh_limit < $wanted_processes) {
my $serverlogin = $self->serverlogin();
print $Global::original_stderr
("parallel: Warning: ssh to $serverlogin only allows ",
"for $ssh_limit simultaneous logins.\n",
"You may raise this by changing ",
"/etc/ssh/sshd_config:MaxStartup on $serverlogin\n",
"Using only ",$ssh_limit-1," connections ",
"to avoid race conditions\n");
::warning("ssh to $serverlogin only allows ",
"for $ssh_limit simultaneous logins.\n",
"You may raise this by changing ",
"/etc/ssh/sshd_config:MaxStartup on $serverlogin.\n",
"Using only ",$ssh_limit-1," connections ",
"to avoid race conditions.\n");
}
# Race condition can cause problem if using all sshs.
if($ssh_limit > 1) { $ssh_limit -= 1; }
@ -2658,11 +2677,11 @@ sub user_requested_processes {
close IN;
$processes = $self->user_requested_processes($opt_P_file);
} else {
print $Global::original_stderr "Cannot open $opt_P\n";
exit(255);
::error("Cannot open $opt_P.\n");
::wait_and_exit(255);
}
} else {
print $Global::original_stderr "Parsing of --jobs/-j/--max-procs/-P failed\n";
::error("Parsing of --jobs/-j/--max-procs/-P failed.\n");
::die_usage();
}
if($processes < 1) {
@ -2694,9 +2713,8 @@ sub ncpus {
if($ncpu =~ /^\s*[0-9]+\s*$/s) {
$self->{'ncpus'} = $ncpu;
} else {
print $Global::original_stderr
("parallel: Warning: Could not figure out ",
"number of cpus on $serverlogin ($ncpu). Using 1\n");
::warning("Could not figure out ",
"number of cpus on $serverlogin ($ncpu). Using 1.\n");
$self->{'ncpus'} = 1;
}
}
@ -2731,7 +2749,7 @@ sub no_of_cpus {
chomp $no_of_cpus;
return $no_of_cpus;
} else {
warn("parallel: Cannot figure out number of cpus. Using 1");
::warning("Cannot figure out number of cpus. Using 1.\n");
return 1;
}
}
@ -2763,7 +2781,7 @@ sub no_of_cores {
chomp $no_of_cores;
return $no_of_cores;
} else {
warn("parallel: Cannot figure out number of CPU cores. Using 1");
::warning("Cannot figure out number of CPU cores. Using 1.\n");
return 1;
}
}
@ -3445,9 +3463,7 @@ sub sshtransfer {
$pre .= "$mkremote_workdir; rsync $rsync_opt "
. ::shell_quote_scalar($file)." $serverlogin:$rsync_destdir;";
} else {
print $Global::original_stderr
"parallel: Warning: "
. $file . " is not readable and will not be transferred\n";
::warning($file, " is not readable and will not be transferred.\n");
}
}
return $pre;
@ -3663,24 +3679,30 @@ sub start {
if($::opt_dryrun) {
$command = "true";
}
$Global::total_running++;
$Global::total_started++;
$ENV{'PARALLEL_SEQ'} = $job->seq();
$ENV{'PARALLEL_PID'} = $$;
::debug("$Global::total_running processes. Starting ("
. $job->seq() . "): $command\n");
if($::opt_pipe) {
my ($in);
$pid = ::open3($in, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-pipe");
# The eval is needed to catch exception from open3
eval {
$pid = ::open3($in, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-pipe");
1;
};
$job->set_stdin($in);
} elsif(@::opt_a and not $Global::stdin_in_opt_a and $job->seq() == 1
and $job->sshlogin()->string() eq ":") {
# Give STDIN to the first job if using -a (but only if running
# locally - otherwise CTRL-C does not work for other jobs Bug#36585)
*IN = *STDIN;
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-a");
# The eval is needed to catch exception from open3
eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-a");
1;
};
# Re-open to avoid complaining
open STDIN, "<&", $Global::original_stdin
or ::die_bug("dup-\$Global::original_stdin: $!");
@ -3688,22 +3710,38 @@ sub start {
open(DEVTTY, "/dev/tty")) {
# Give /dev/tty to the command if no one else is using it
*IN = *DEVTTY;
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-/dev/tty");
$Global::tty_taken = $pid;
close DEVTTY;
# The eval is needed to catch exception from open3
eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-/dev/tty");
$Global::tty_taken = $pid;
close DEVTTY;
1;
};
} else {
$pid = ::open3(::gensym, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-gensym");
eval {
$pid = ::open3(::gensym, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) ||
::die_bug("open3-gensym");
1;
};
}
$job->set_pid($pid);
$job->set_starttime();
if($::opt_timeout) {
# Timeout must be set before inserting into queue
$job->set_timeout($::opt_timeout);
$Global::timeoutq->insert($job);
if($pid) {
# A job was started
$Global::total_running++;
$Global::total_started++;
$job->set_pid($pid);
$job->set_starttime();
if($::opt_timeout) {
# Timeout must be set before inserting into queue
$job->set_timeout($::opt_timeout);
$Global::timeoutq->insert($job);
}
return $job;
} else {
# No more processes
::debug("Cannot spawn more jobs.\n");
return undef;
}
return $job;
}
sub is_already_in_joblog {
@ -4196,9 +4234,7 @@ sub replaced {
# Is this really a command in $PATH starting with '-'?
my $cmd = $1;
if(not grep { -e $_."/".$cmd } split(":",$ENV{'PATH'})) {
print STDERR "parallel: Error:"
. " Command ($cmd) starts with '-'."
. " Is this a wrong option?\n";
::error("Command ($cmd) starts with '-'. Is this a wrong option?.\n");
::wait_and_exit(255);
}
}
@ -4465,8 +4501,7 @@ sub get {
if($::opt_pipe) {
if($cmd_line->replaced() eq "") {
# Empty command - pipe requires a command
print STDERR "$Global::progname: --pipe must have a ".
"command to pipe into (e.g. 'cat')\n";
::error("--pipe must have a command to pipe into (e.g. 'cat').\n");
::wait_and_exit(255);
}
} else {
@ -4546,8 +4581,8 @@ sub max_length {
if($::opt_s <= $Limits::Command::line_max_len) {
$Limits::Command::line_max_len = $::opt_s;
} else {
print STDERR "$Global::progname: value for -s option ",
"should be < $Limits::Command::line_max_len\n";
::warning("Value for -s option ",
"should be < $Limits::Command::line_max_len.\n");
}
} else {
$Limits::Command::line_max_len = real_max_length();
@ -4733,9 +4768,9 @@ sub new {
my $fhs = shift;
for my $fh (@$fhs) {
if(-t $fh) {
print STDERR "$Global::progname: Input is read from the terminal. ".
"Only experts do this on purpose. ".
"Press CTRL-D to exit.\n";
::warning("Input is read from the terminal. ".
"Only experts do this on purpose. ".
"Press CTRL-D to exit.\n");
}
}
return bless {
@ -5020,8 +5055,7 @@ sub trim_of {
} elsif($Global::trim eq "rl" or $Global::trim eq "lr") {
for $arg (@strings) { $arg =~ s/^\s+//; $arg =~ s/\s+$//; }
} else {
print STDERR "$Global::progname: --trim must be one of: r l ".
"rl lr\n";
::error("--trim must be one of: r l rl lr.\n");
::wait_and_exit(255);
}
return wantarray ? @strings : "@strings";
@ -5215,9 +5249,8 @@ sub lock {
$Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;";
while(not flock $self->{'lockfh'}, LOCK_EX()|LOCK_NB()) {
if ($! =~ m/Function not implemented/) {
print $Global::original_stderr
("parallel: Warning: flock: $!");
print "parallel: Will wait for a random while\n";
::warning("flock: $!");
::warning("Will wait for a random while\n");
::usleep(rand(5000));
last;
}
@ -5250,8 +5283,8 @@ sub mkdir_or_die {
mkdir $ddir;
}
if(not -w $dir) {
print STDERR "$Global::progname: Cannot write to $dir: $!\n";
exit(255);
::error("Cannot write to $dir: $!\n");
::wait_and_exit(255);
}
}

View file

@ -462,7 +462,7 @@ See also: B<--bg>, B<man sem>
Implies B<--semaphore>.
=item B<--filter-hosts> (beta testing)
=item B<--filter-hosts>
Remove down hosts. For each remote host: check that login through ssh
works. If not: do not use this host.
@ -549,7 +549,7 @@ specified, and for B<-I>{} otherwise. This option is deprecated;
use B<-I> instead.
=item B<--joblog> I<logfile> (beta testing)
=item B<--joblog> I<logfile>
Logfile for executed jobs. Save a list of the executed jobs to
I<logfile> in the following TAB separated format: sequence number,
@ -645,7 +645,7 @@ to see the difference:
parallel -j4 sleep {}\; echo {} ::: 2 1 4 3
parallel -j4 -k sleep {}\; echo {} ::: 2 1 4 3
=item B<-L> I<max-lines> (beta testing)
=item B<-L> I<max-lines>
When used with B<--pipe>: Read records of I<max-lines>.
@ -675,7 +675,7 @@ B<-l 0> is an alias for B<-l 1>.
Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
=item B<--load> I<max-load> (beta testing)
=item B<--load> I<max-load>
Do not start new jobs on a given computer unless the load is less than
I<max-load>. I<max-load> uses the same syntax as B<--jobs>, so I<100%>
@ -770,9 +770,9 @@ Instead of printing the output to stdout (standard output) the output
of each job is saved in a file and the filename is then printed.
=item B<--pipe> (beta testing)
=item B<--pipe>
=item B<--spreadstdin> (beta testing)
=item B<--spreadstdin>
Spread input to jobs on stdin (standard input). Read a block of data
from stdin (standard input) and give one block of data as input to one
@ -797,6 +797,13 @@ defaults to '\n'. To have no record separator use B<--recend "">.
B<--files> is often used with B<--pipe>.
=item B<--plain> (alpha testing)
Ignore any B<--profile>, $PARALLEL, ~/.parallel/config, and
B<--tollef> to get full control on the command line (used by GNU
B<parallel> internally when called with B<--sshlogin>).
=item B<--progress>
Show progress of computations. List the computers involved in the task
@ -1232,14 +1239,14 @@ different dir for the files. Setting B<--tmpdir> is equivalent to
setting $TMPDIR.
=item B<--timeout> I<sec> (beta testing)
=item B<--timeout> I<sec>
Time out for command. If the command runs for longer than I<sec>
seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms
later, followed by SIGKILL 200 ms later.
=item B<--tollef> (beta testing)
=item B<--tollef>
Make GNU B<parallel> behave more like Tollef's parallel command. It
activates B<-u>, B<-q>, and B<--arg-sep -->. It also causes B<-l> to
@ -1445,9 +1452,9 @@ Compare these two:
See also B<--header>.
=item B<--shebang> (beta testing)
=item B<--shebang>
=item B<--hashbang> (beta testing)
=item B<--hashbang>
GNU B<Parallel> can be called as a shebang (#!) command as the first line of a script. Like this:

View file

@ -489,8 +489,8 @@ See also: @strong{--bg}, @strong{man sem}
Implies @strong{--semaphore}.
@item @strong{--filter-hosts} (beta testing)
@anchor{@strong{--filter-hosts} (beta testing)}
@item @strong{--filter-hosts}
@anchor{@strong{--filter-hosts}}
Remove down hosts. For each remote host: check that login through ssh
works. If not: do not use this host.
@ -581,8 +581,8 @@ This option is a synonym for @strong{-I}@emph{replace-str} if @emph{replace-str}
specified, and for @strong{-I}@{@} otherwise. This option is deprecated;
use @strong{-I} instead.
@item @strong{--joblog} @emph{logfile} (beta testing)
@anchor{@strong{--joblog} @emph{logfile} (beta testing)}
@item @strong{--joblog} @emph{logfile}
@anchor{@strong{--joblog} @emph{logfile}}
Logfile for executed jobs. Save a list of the executed jobs to
@emph{logfile} in the following TAB separated format: sequence number,
@ -696,8 +696,8 @@ to see the difference:
parallel -j4 -k sleep {}\; echo {} ::: 2 1 4 3
@end verbatim
@item @strong{-L} @emph{max-lines} (beta testing)
@anchor{@strong{-L} @emph{max-lines} (beta testing)}
@item @strong{-L} @emph{max-lines}
@anchor{@strong{-L} @emph{max-lines}}
When used with @strong{--pipe}: Read records of @emph{max-lines}.
@ -727,8 +727,8 @@ standard specifies @strong{-L} instead.
Implies @strong{-X} unless @strong{-m}, @strong{--xargs}, or @strong{--pipe} is set.
@item @strong{--load} @emph{max-load} (beta testing)
@anchor{@strong{--load} @emph{max-load} (beta testing)}
@item @strong{--load} @emph{max-load}
@anchor{@strong{--load} @emph{max-load}}
Do not start new jobs on a given computer unless the load is less than
@emph{max-load}. @emph{max-load} uses the same syntax as @strong{--jobs}, so @emph{100%}
@ -825,11 +825,11 @@ all the output from one server will be grouped together.
Instead of printing the output to stdout (standard output) the output
of each job is saved in a file and the filename is then printed.
@item @strong{--pipe} (beta testing)
@anchor{@strong{--pipe} (beta testing)}
@item @strong{--pipe}
@anchor{@strong{--pipe}}
@item @strong{--spreadstdin} (beta testing)
@anchor{@strong{--spreadstdin} (beta testing)}
@item @strong{--spreadstdin}
@anchor{@strong{--spreadstdin}}
Spread input to jobs on stdin (standard input). Read a block of data
from stdin (standard input) and give one block of data as input to one
@ -853,6 +853,13 @@ defaults to '\n'. To have no record separator use @strong{--recend ""}.
@strong{--files} is often used with @strong{--pipe}.
@item @strong{--plain} (alpha testing)
@anchor{@strong{--plain} (alpha testing)}
Ignore any @strong{--profile}, $PARALLEL, ~/.parallel/config, and
@strong{--tollef} to get full control on the command line (used by GNU
@strong{parallel} internally when called with @strong{--sshlogin}).
@item @strong{--progress}
@anchor{@strong{--progress}}
@ -1311,15 +1318,15 @@ into temporary files in /tmp. By setting @strong{--tmpdir} you can use a
different dir for the files. Setting @strong{--tmpdir} is equivalent to
setting $TMPDIR.
@item @strong{--timeout} @emph{sec} (beta testing)
@anchor{@strong{--timeout} @emph{sec} (beta testing)}
@item @strong{--timeout} @emph{sec}
@anchor{@strong{--timeout} @emph{sec}}
Time out for command. If the command runs for longer than @emph{sec}
seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms
later, followed by SIGKILL 200 ms later.
@item @strong{--tollef} (beta testing)
@anchor{@strong{--tollef} (beta testing)}
@item @strong{--tollef}
@anchor{@strong{--tollef}}
Make GNU @strong{parallel} behave more like Tollef's parallel command. It
activates @strong{-u}, @strong{-q}, and @strong{--arg-sep --}. It also causes @strong{-l} to
@ -1540,11 +1547,11 @@ Compare these two:
See also @strong{--header}.
@item @strong{--shebang} (beta testing)
@anchor{@strong{--shebang} (beta testing)}
@item @strong{--shebang}
@anchor{@strong{--shebang}}
@item @strong{--hashbang} (beta testing)
@anchor{@strong{--hashbang} (beta testing)}
@item @strong{--hashbang}
@anchor{@strong{--hashbang}}
GNU @strong{Parallel} can be called as a shebang (#!) command as the first line of a script. Like this:

View file

@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20120722;
$Global::version = 20120822;
$Global::progname = 'sql';
# This must be done first as this may exec myself

View file

@ -1,26 +1,29 @@
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereq
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
time sh Start.sh
date
local: ../src/parallel tests-to-run/*local* wanted-results/*local* startdb prereq
testdb: ../src/parallel tests-to-run/*sql* wanted-results/*sql* prereqdb
time sh Start.sh sql
date
testlocal: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal
time sh Start.sh local
date
prereq: installparallel
echo | mop || (echo mop is required for testsuite; /bin/false)
seq 1 2 | mop || (echo seq is required for testsuite; /bin/false)
prereqlocal: installparallel
seq 1 2 >/dev/null || (echo seq is required for testsuite; /bin/false)
stdout echo || (echo stdout is required for testsuite; /bin/false)
convert | mop || (echo convert is required for testsuite; /bin/false)
ren 2>&1 | mop || (echo ren is required for testsuite; /bin/false)
echo | buffer | mop || (echo buffer is required for testsuite; /bin/false)
echo 1+2 | bc | mop || (echo bc is required for testsuite; /bin/false)
stdout gawk | mop || (echo gawk is required for testsuite; /bin/false)
convert >/dev/null || (echo convert is required for testsuite; /bin/false)
echo 1+2 | bc >/dev/null || (echo bc is required for testsuite; /bin/false)
echo | gawk '{print "gawk is installed"}' || (echo gawk is required for testsuite; /bin/false)
expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for testsuite; /bin/false)
echo | pv -qL 10 || (echo pv is required for testsuite; /bin/false)
echo | script -c echo -q /dev/null || (echo script is required for testsuite; /bin/false)
niceload true || (echo niceload is required for testsuite; /bin/false)
which burnP6 || (echo burnP6 is required for testsuite; /bin/false)
which timeout || (echo timeout is required for testsuite; /bin/false)
prereqremote: installparallel
parallel ssh parallel@parallel-server{} parallel --minversion 20100101 ::: 1 2 3 || (echo parallel on remote required for testsuite; /bin/true)
installparallel: ../src/parallel

View file

@ -1,50 +1,50 @@
#!/bin/bash
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1
echo "### Test -I"
seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'
echo "### Test -I";
seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'
echo "### Test -X -I"
seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::'
echo "### Test -X -I";
seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::'
echo "### Test -m -I"
seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::'
echo "### Test -m -I";
seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::'
echo "### Test max line length -m -I"
seq 1 60000 | parallel -I :: -m -j1 echo a::b::c |
mop -q "|sort |md5sum" :par1;
export CHAR=$(cat ~/.mop/:par1 | wc -c);
export LINES=$(cat ~/.mop/:par1 | wc -l);
echo -n "Chars per line ($CHAR/$LINES): ";
echo "$CHAR/$LINES" | bc
echo "### Test max line length -m -I";
seq 1 60000 | parallel -I :: -m -j1 echo a::b::c |
tee >(sort |md5sum) >/tmp/114-a$$;
export CHAR=$(cat /tmp/114-a$$ | wc -c);
export LINES=$(cat /tmp/114-a$$ | wc -l);
echo "Chars per line ($CHAR/$LINES): "$(echo "$CHAR/$LINES" | bc);
rm /tmp/114-a$$
echo "### Test max line length -X -I"
seq 1 60000 | parallel -I :: -X -j1 echo a::b::c |
mop -q "|sort |md5sum" :par;
export CHAR=$(cat ~/.mop/:par | wc -c);
export LINES=$(cat ~/.mop/:par | wc -l);
echo -n "Chars per line ($CHAR/$LINES): ";
echo "$CHAR/$LINES" | bc
echo "### Test max line length -X -I";
seq 1 60000 | parallel -I :: -X -j1 echo a::b::c |
tee >(sort |md5sum) >/tmp/114-b$$;
export CHAR=$(cat /tmp/114-b$$ | wc -c);
export LINES=$(cat /tmp/114-b$$ | wc -l);
echo "Chars per line ($CHAR/$LINES): "$(echo "$CHAR/$LINES" | bc);
rm /tmp/114-b$$
echo "### bug #36659: --sshlogin strips leading slash from ssh command"
parallel --sshlogin '/usr/bin/ssh localhost' echo ::: OK
echo "### bug #36659: --sshlogin strips leading slash from ssh command";
parallel --sshlogin '/usr/bin/ssh localhost' echo ::: OK
echo "### bug #36660: --workdir mkdir does not use --sshlogin custom ssh"
echo "### bug #36660: --workdir mkdir does not use --sshlogin custom ssh";
cd /tmp; echo OK > parallel_test.txt;
ssh () { echo Failed; };
export -f ssh;
parallel --workdir /tmp/foo/bar --transfer --sshlogin '/usr/bin/ssh localhost' cat ::: parallel_test.txt;
echo "bug #36657: --load does not work with custom ssh"
echo "bug #36657: --load does not work with custom ssh";
cd /tmp; echo OK > parallel_test.txt;
ssh () { echo Failed; };
export -f ssh;
parallel --load=1000% -S "/usr/bin/ssh localhost" echo ::: OK
echo "bug #34958: --pipe with record size measured in lines"
echo "bug #34958: --pipe with record size measured in lines";
seq 10 | parallel --pipe -L 4 cat\;echo FOO
echo "bug #34958: --pipe with record size measured in lines"
echo "bug #34958: --pipe with record size measured in lines";
seq 10 | parallel --pipe -l 4 cat\;echo FOO
EOF

View file

@ -4,8 +4,7 @@ rsync -Ha --delete input-files/testdir/ tmp/
cd tmp
echo echo test of cat pipe sh | parallel -j 50 2>&1
find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {}_thumb.jpg
find . -name '*_thumb.jpg' | ren 's:/([^/]+)_thumb.jpg$:/thumb_$1:'
find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}
ls | parallel ls | sort
ls | parallel echo ls | sort

View file

@ -37,10 +37,19 @@ echo '### Test -m'
(echo foo;echo bar;echo joe.gif) | parallel -j1 -kX echo 1{}2{.}3 A{.}B{.}C
seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -km echo a{}b{.}c{.}
seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -kX echo a{}b{.}c{.}
echo '### Test -m with 60000 args'
seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -km echo a{}b{.}c{.} | mop -d 4 "|md5sum" "| wc"
echo '### Test -X with 60000 args'
seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.} | mop -d 4 "|md5sum" "| wc"
echo '### Test -m with 60000 args';
seq 1 60000 | perl -pe 's/$/.gif\n/' |
parallel -j1 -km echo a{}b{.}c{.} |
tee >(wc) >(md5sum) >/dev/null;
wait
echo '### Test -X with 60000 args';
seq 1 60000 | perl -pe 's/$/.gif\n/' |
parallel -j1 -kX echo a{}b{.}c{.} |
tee >(wc) >(md5sum) >/dev/null;
wait
echo '### Test -X with 60000 args and 5 expansions'
seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
seq 1 60000 | perl -pe 's/$/.gif\n/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l

View file

@ -1,24 +1,44 @@
#!/bin/bash
echo '### This causes problems if we kill child processes'
seq 1 40 | parallel -j 0 seq 1 10 | sort |md5sum
seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -L1
echo '### Test of xargs -m command lines > 130k';
seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc) >(sort |md5sum) >/tmp/a$$;
wait;
CHAR=$(cat /tmp/a$$ | wc -c);
LINES=$(cat /tmp/a$$ | wc -l);
echo "Chars per line:" $(echo "$CHAR/$LINES" | bc);
rm /tmp/a$$
echo '### Test of xargs -m and -X'
seq 1 60000 | parallel -j1 -m echo | mop -d 4 "|sort |md5sum" "| wc"
(echo foo;echo bar) | parallel -j1 -m echo 1{}2{}3 A{}B{}C
(echo foo;echo bar) | parallel -j1 -X echo 1{}2{}3 A{}B{}C
seq 1 60000 | parallel -m -j1 echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc"
seq 1 60000 | parallel -m -j1 echo a{}b{}c | \
mop -q "|sort |md5sum" :par
echo -n "Chars per line: "
CHAR=$(cat ~/.mop/:par | wc -c)
LINES=$(cat ~/.mop/:par | wc -l)
echo "$CHAR/$LINES" | bc
echo '### Test of xargs -X command lines > 130k';
seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc) >(sort |md5sum) >/tmp/b$$;
wait;
CHAR=$(cat /tmp/b$$ | wc -c);
LINES=$(cat /tmp/b$$ | wc -l);
echo "Chars per line:" $(echo "$CHAR/$LINES" | bc);
rm /tmp/b$$
echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop'
echo a | parallel -qX echo "'"{}"' "
echo a | parallel -qX echo "'{}'"
echo '### Test of xargs -m command lines > 130k';
seq 1 60000 | parallel -k -j1 -m echo | md5sum
echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL'
seq 1 2 | SHELL=tcsh stdout parallel -k --nice 8 setenv a b\;echo \$SHELL
echo '### This causes problems if we kill child processes';
seq 1 40 | parallel -j 0 seq 1 10 | sort |md5sum
echo '### This causes problems if we kill child processes (II)';
seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort | md5sum
echo '### Test -m';
(echo foo;echo bar) | parallel -j1 -m echo 1{}2{}3 A{}B{}C
echo '### Test -X';
(echo foo;echo bar) | parallel -j1 -X echo 1{}2{}3 A{}B{}C
echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop';
echo a | parallel -qX echo "'"{}"' "
echo '### Bug before 2009-08-26 causing regexp compile error or infinite loop (II)';
echo a | parallel -qX echo "'{}'"
echo '### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL';
seq 1 2 | SHELL=tcsh stdout parallel -k --nice 8 setenv a b\;echo \$SHELL
EOF

View file

@ -1,14 +1,11 @@
#!/bin/bash
PAR=parallel
rm -rf tmp 2>/dev/null
cp -a input-files/testdir2 tmp
cd tmp
echo '### Test filenames containing UTF-8'
find . -name '*.jpg' | $PAR -j +0 convert -geometry 120 {} {}_thumb.jpg
find . -name '*_thumb.jpg' | ren 's:/([^/]+)_thumb.jpg$:/thumb_$1:'
find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}
find |grep -v CVS | sort

View file

@ -97,13 +97,13 @@ nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000
nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 |
$PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum
echo '### Test of -j filename';
echo '### Test of -j filename - non-existent file';
nice stdout parallel -j no_such_file echo ::: 1
echo '### Test of -j filename';
echo 3 >/tmp/jobs_to_run1;
parallel -j /tmp/jobs_to_run1 -v sleep 0.{} ::: 9 8 7 6 4;
# Should give 7 8 9 4 6
parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4;
# Should give 6 8 10 5 4
echo '### Test ::::'
echo '### Change --arg-file-sep'
@ -216,4 +216,4 @@ EOF
echo '### Test of -j filename with file content changing';
echo 1 >/tmp/jobs_to_run2;
(sleep 3; echo 10 >/tmp/jobs_to_run2) &
parallel -j /tmp/jobs_to_run2 -v sleep {} ::: 3.3 1.1 1.3 1.4 1.2 1 1 1 1 1 1 1 1 1 1 1
parallel -j /tmp/jobs_to_run2 -v sleep {} ::: 3.3 1.21 1.43 1.54 1.32 1 1 1 1 1 1 1 1 1 1 1

View file

@ -89,7 +89,7 @@ seq 1 10 >/tmp/$$-1; parallel -k -a /tmp/$$-1 echo
seq 1 10 >/tmp/$$-2; parallel -k --arg-file /tmp/$$-2 echo
echo '### Test killing children with --timeout and exit value (failed if timed out)'
pstree | grep sleep | grep -v anacron | wc; parallel --timeout 3 'true {} ; for i in `seq 100 120`; do bash -c "(sleep $i)" & sleep $i & done; wait; echo No good' ::: 1000000000 1000000001 ; echo $?; pstree | grep sleep | grep -v anacron | wc
pstree | grep sleep | grep -v anacron | grep -v screensave | wc; parallel --timeout 3 'true {} ; for i in `seq 100 120`; do bash -c "(sleep $i)" & sleep $i & done; wait; echo No good' ::: 1000000000 1000000001 ; echo $?; pstree | grep sleep | grep -v anacron | grep -v screensave | wc
EOF
#echo '### Test bugfix if no command given'

View file

@ -1,6 +1,5 @@
#!/bin/bash
PAR=parallel
SERVER1=parallel-server1
SERVER2=parallel-server2
SSHLOGIN1=parallel@$SERVER1
@ -35,7 +34,7 @@ echo "### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containi
stdout parallel -j1 -k -Xv --cleanup --return tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file -S $SSHLOGIN2 \
mkdir -p tmp/{/}_{/.}_{2/.}_{2/}_{2.} \;touch tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file \
::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 'number 7' 'number <8|8>'
find tmp
find tmp | sort
rm -rf tmp
echo "### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"

View file

@ -17,7 +17,7 @@ stdout parallel --progress "sleep 1; echo {}" < /dev/null
echo '### bug #34422: parallel -X --eta crashes with div by zero'
seq 2 | stdout parallel -X --eta echo
echo '### --timeout on remote machines'
echo '### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed'
parallel -j0 --timeout 6 --onall -S localhost,parallel@parallel-server1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $?
echo '### --pipe without command'

View file

@ -36,6 +36,9 @@ cat <<EOF >~/.parallel/test_profile
EOF
parallel -v -J test_profile ::: <(echo a) <(echo b)
echo '### Test ugly quoting from profile file --plain'
parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b)
PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 -v ::: foo
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo
@ -45,15 +48,24 @@ cat <<EOF >~/.parallel/test_profile
EOF
parallel -v -J test_profile -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 '<(echo {})' ::: foo
echo '### Test quoting of $ in command from profile file --plain'
parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 'cat <(echo {})' ::: foo
echo '### Test quoting of $ in command from $PARALLEL'
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo
echo '### Test quoting of $ in command from $PARALLEL --plain'
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from profile file'
cat <<EOF >~/.parallel/test_profile
-k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe '\$a=1; print \$a'
EOF
parallel -v -J test_profile '<(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from profile file --plain'
parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo
echo '### Test quoting of space in arguments (-S) from $PARALLEL'
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe "\\$a=1; print \\$a" ' parallel -v '<(echo {})' ::: foo
@ -73,3 +85,6 @@ echo -S .. > ~/.parallel/test_S..
echo parallel@parallel-server1 > ~/.parallel/sshloginfile
echo parallel@parallel-server2 >> ~/.parallel/sshloginfile
parallel -Jtest_tag -Jtest_S.. --nonall echo a | sort
echo '### Test merging of profiles - sort needed because -k only works on the single machine --plain'
parallel --plain -Jtest_tag -Jtest_S.. --nonall echo a | sort

View file

@ -3,7 +3,7 @@
### Test of quoting of > bug if line continuation
> > >
### Test of --trim illegal
parallel: --trim must be one of: r l rl lr
parallel: Error: --trim must be one of: r l rl lr.
### Test of eof string on :::
foo
### Test of ignore-empty string on :::

View file

@ -17,4 +17,4 @@ ajob2b job2
### Test bug #35820: sem breaks if $HOME is not writable
Workaround: use another writable dir
OK
parallel: Cannot write to /usr/this/should/fail/.parallel: No such file or directory
parallel: Error: Cannot write to /usr/this/should/fail/.parallel: No such file or directory

View file

@ -1,17 +1,24 @@
### This causes problems if we kill child processes
33bf8b2986551515cdaff5e860618098 -
d7fb96d6a56d4347bc24930a395c431a -
### Test of xargs -m and -X
b35d8e49be8d94899b719c40d3f1f4bb -
3 60000 348894
1foo bar2foo bar3 Afoo barBfoo barC
1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC
31d9274be5fdc2de59487cb05ba57776 -
### Test of xargs -m command lines > 130k
6 119994 697800
31d9274be5fdc2de59487cb05ba57776 -
Chars per line: 116300
### Test of xargs -X command lines > 130k
7 60000 817788
22074f9acada52462defb18ba912d744 -
Chars per line: 116826
### Test of xargs -m command lines > 130k
b35d8e49be8d94899b719c40d3f1f4bb -
### This causes problems if we kill child processes
33bf8b2986551515cdaff5e860618098 -
### This causes problems if we kill child processes (II)
d7fb96d6a56d4347bc24930a395c431a -
### Test -m
1foo bar2foo bar3 Afoo barBfoo barC
### Test -X
1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC
### Bug before 2009-08-26 causing regexp compile error or infinite loop
'a'
### Bug before 2009-08-26 causing regexp compile error or infinite loop (II)
'a'
### nice and tcsh and Bug #33995: Jobs executed with sh instead of $SHELL
tcsh 1

View file

@ -31,7 +31,7 @@ echo '### Test of --retries on unreachable host'
### Test of --retries on unreachable host
seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo
ssh: connect to host 4.3.2.1 port 22: Connection timed out
parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1
parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1.
echo 1
1
echo 2

View file

@ -57,7 +57,7 @@ b
350eda13a37912d755c9d733d149bdaf -
350eda13a37912d755c9d733d149bdaf -
350eda13a37912d755c9d733d149bdaf -
### Test of -j filename
### Test of -j filename - non-existent file
Usage:
parallel [options] [command [arguments]] < list_of_arguments
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
@ -86,13 +86,13 @@ When using GNU Parallel for a publication please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
Parsing of --jobs/-j/--max-procs/-P failed
parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed.
### Test of -j filename
sleep 0.7
sleep 0.8
sleep 0.9
sleep 0.4
sleep 0.6
sleep 6
sleep 8
sleep 10
sleep 5
sleep 4
### Test ::::
### Change --arg-file-sep
1 5
@ -151,7 +151,7 @@ sleep 0.6
3 4 5
6 7
### Test :::: on nonexistent
parallel: Cannot open input file `nonexistent': No such file or directory
parallel: Error: Cannot open input file `nonexistent': No such file or directory.
### Test :::: two files
1 5
2 6
@ -233,8 +233,8 @@ b a
echo b a
b a
### Multiple -a: nonexistent
parallel: Cannot open input file `nonexist': No such file or directory
parallel: Cannot open input file `nonexist': No such file or directory
parallel: Error: Cannot open input file `nonexist': No such file or directory.
parallel: Error: Cannot open input file `nonexist': No such file or directory.
### Test {#.}
echo c-ext b-withext a-noext
c-ext b-withext a-noext
@ -318,10 +318,10 @@ sleep 1
sleep 1
sleep 1
sleep 1
sleep 1.1
sleep 1.2
sleep 1.3
sleep 1.4
sleep 1.21
sleep 1.32
sleep 1.43
sleep 1.54
sleep 1
sleep 1
sleep 1

View file

@ -117,4 +117,4 @@ done 4
39
40
### Test bug #33621: --bg -p should give an error message
parallel: Jobs running in the background cannot be interactive.
parallel: Error: Jobs running in the background cannot be interactive.

View file

@ -39,11 +39,11 @@ def 3
2 17
2 17
### Test --db-size --dbsize
2 6 53
2 6 53
2 6 54
2 6 54
### Test --table-size --tablesize
30
30
39
39
### Test --debug
dburl mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest
databasedriver mysql user sqlunittest password CB5A1FFFA5A host localhost port 3306 database sqlunittest query

View file

@ -40,7 +40,7 @@
19
20
### Check warning if --transfer but file not found
parallel: Warning: /tmp/noexistant/file is not readable and will not be transferred
parallel: Warning: /tmp/noexistant/file is not readable and will not be transferred.
/tmp/noexistant/file
### Transfer for file starting with :
remote-:.:
@ -54,13 +54,13 @@ content-file: name.foo
remote-file : name.foo.file : name
content-file : name.foo
### Check warning if --transfer but not --sshlogin
parallel: Warning: --transfer ignored as there are no remote --sshlogin
parallel: Warning: --transfer ignored as there are no remote --sshlogin.
### Check warning if --return but not --sshlogin
parallel: Warning: --return ignored as there are no remote --sshlogin
parallel: Warning: --return ignored as there are no remote --sshlogin.
### Check warning if --cleanup but not --sshlogin
parallel: Warning: --cleanup ignored as there are no remote --sshlogin
parallel: Warning: --cleanup ignored as there are no remote --sshlogin.
### Test --sshlogin -S --sshloginfile
1

View file

@ -1,5 +1,5 @@
### Test --basefile with no --sshlogin
parallel: Warning: --basefile ignored as there are no remote --sshlogin
parallel: Warning: --basefile ignored as there are no remote --sshlogin.
### Test --basefile + --cleanup + permissions
script1 run 1
@ -45,32 +45,32 @@ OK
8
9
### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
parallel: Warning: using -X or -m with --sshlogin may fail
parallel: Warning: Using -X or -m with --sshlogin may fail.
mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 tmp/number\ 7_number\ 7_number2_number2.c_a/number2 tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file tmp/number\ 7_number\ 7_number2_number2.c_a/number2/file tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2/file
tmp
tmp/number6_number6_number2_number2.c_a
tmp/number6_number6_number2_number2.c_a/number2
tmp/number6_number6_number2_number2.c_a/number2/file
tmp/number5_number5_number2_number2.c_a
tmp/number5_number5_number2_number2.c_a/number2
tmp/number5_number5_number2_number2.c_a/number2/file
tmp/number4_number4_number2_number2.c_a
tmp/number4_number4_number2_number2.c_a/number2
tmp/number4_number4_number2_number2.c_a/number2/file
tmp/number3.c_number3_number2_number2.c_a
tmp/number3.c_number3_number2_number2.c_a/number2
tmp/number3.c_number3_number2_number2.c_a/number2/file
tmp/number2.c_number2_number2_number2.c_a
tmp/number2.c_number2_number2_number2.c_a/number2
tmp/number2.c_number2_number2_number2.c_a/number2/file
tmp/number1.c_number1_number2_number2.c_a
tmp/number1.c_number1_number2_number2.c_a/number2
tmp/number1.c_number1_number2_number2.c_a/number2/file
tmp/number <8|8>_number <8|8>_number2_number2.c_a
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2/file
tmp/number 7_number 7_number2_number2.c_a
tmp/number 7_number 7_number2_number2.c_a/number2
tmp/number 7_number 7_number2_number2.c_a/number2/file
tmp/number <8|8>_number <8|8>_number2_number2.c_a
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2/file
tmp/number1.c_number1_number2_number2.c_a
tmp/number1.c_number1_number2_number2.c_a/number2
tmp/number1.c_number1_number2_number2.c_a/number2/file
tmp/number2.c_number2_number2_number2.c_a
tmp/number2.c_number2_number2_number2.c_a/number2
tmp/number2.c_number2_number2_number2.c_a/number2/file
tmp/number3.c_number3_number2_number2.c_a
tmp/number3.c_number3_number2_number2.c_a/number2
tmp/number3.c_number3_number2_number2.c_a/number2/file
tmp/number4_number4_number2_number2.c_a
tmp/number4_number4_number2_number2.c_a/number2
tmp/number4_number4_number2_number2.c_a/number2/file
tmp/number5_number5_number2_number2.c_a
tmp/number5_number5_number2_number2.c_a/number2
tmp/number5_number5_number2_number2.c_a/number2/file
tmp/number6_number6_number2_number2.c_a
tmp/number6_number6_number2_number2.c_a/number2
tmp/number6_number6_number2_number2.c_a/number2/file
### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
### But we will wait for a real world scenario

View file

@ -20,10 +20,11 @@ Computers / CPU cores / Max jobs to run
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
local:1/0/100%/0.0s
### --timeout on remote machines
### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed
slept 1
slept 1
jobs failed: 2
### --pipe without command
parallel: --pipe must have a command to pipe into (e.g. 'cat')
parallel: Error: --pipe must have a command to pipe into (e.g. 'cat').
### bug #36260: {n} expansion in --colsep files fails for empty fields if all following fields are also empty
AB

View file

@ -35,6 +35,11 @@ perl -pe '$a=1; print $a' /dev/fd/63
1a
perl -pe '$a=1; print $a' /dev/fd/62
1b
### Test ugly quoting from profile file --plain
echo /dev/fd/63
/dev/fd/63
echo /dev/fd/62
/dev/fd/62
echo foo
foo
ssh parallel@parallel-server3 ssh parallel-server2 'eval `echo $SHELL | grep -E "/(t)?csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' perl\ -pe\ \"\\\$a=1\;\ print\ \\\$a\"\ \<\(echo\ foo\);
@ -42,12 +47,21 @@ ssh parallel@parallel-server3 ssh parallel-server2 'eval `echo $SHELL | grep -E
### Test quoting of $ in command from profile file
perl -pe '$a=1; print $a' <(echo foo)
1foo
### Test quoting of $ in command from profile file --plain
cat <(echo foo)
foo
### Test quoting of $ in command from $PARALLEL
perl -pe "\$a=1; print \$a" <(echo foo)
1foo
### Test quoting of $ in command from $PARALLEL --plain
cat <(echo foo)
foo
### Test quoting of space in arguments (-S) from profile file
perl -pe '$a=1; print $a' <(echo foo)
1foo
### Test quoting of space in arguments (-S) from profile file --plain
cat <(echo foo)
foo
### Test quoting of space in arguments (-S) from $PARALLEL
perl -pe "\$a=1; print \$a" <(echo foo)
1foo
@ -60,3 +74,5 @@ perl -pe "\$a=1; print \$a" <(echo foo)
### Test merging of profiles - sort needed because -k only works on the single machine
parallel@parallel-server1 a
parallel@parallel-server2 a
### Test merging of profiles - sort needed because -k only works on the single machine --plain
a

View file

@ -20,27 +20,27 @@ OK
7.7
8.8
### Test retired
parallel: -g has been retired. Use --group.
parallel: -B has been retired. Use --bf.
parallel: -T has been retired. Use --tty.
parallel: -U has been retired. Use --er.
parallel: -W has been retired. Use --wd.
parallel: -Y has been retired. Use --shebang.
parallel: -H has been retired. Use --halt.
parallel: -g has been retired. Use --group.
parallel: -B has been retired. Use --bf.
parallel: -T has been retired. Use --tty.
parallel: -U has been retired. Use --er.
parallel: -W has been retired. Use --wd.
parallel: -Y has been retired. Use --shebang.
parallel: -H has been retired. Use --halt.
parallel: -g has been retired. Use --group.
parallel: -B has been retired. Use --bf.
parallel: -T has been retired. Use --tty.
parallel: -U has been retired. Use --er.
parallel: -W has been retired. Use --wd.
parallel: -Y has been retired. Use --shebang.
parallel: -H has been retired. Use --halt.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
parallel: Error: -g has been retired. Use --group.
parallel: Error: -B has been retired. Use --bf.
parallel: Error: -T has been retired. Use --tty.
parallel: Error: -U has been retired. Use --er.
parallel: Error: -W has been retired. Use --wd.
parallel: Error: -Y has been retired. Use --shebang.
parallel: Error: -H has been retired. Use --halt.
### Test --joblog followed by --resume --joblog
5 49
### Test --resume --joblog followed by --resume --joblog