mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
parallel: Added difference between paexec
This commit is contained in:
Ole Tange
parent
59cb631264
commit
e06f6d52ce
20
configure
vendored
20
configure
vendored
|
|
@ -1,6 +1,6 @@
|
|||
#! /bin/sh
|
||||
# Guess values for system-dependent variables and create Makefiles.
|
||||
# Generated by GNU Autoconf 2.67 for parallel 20100906.
|
||||
# Generated by GNU Autoconf 2.67 for parallel 20100922.
|
||||
#
|
||||
# Report bugs to <bug-parallel@gnu.org>.
|
||||
#
|
||||
|
|
@ -551,8 +551,8 @@ MAKEFLAGS=
|
|||
# Identity of this package.
|
||||
PACKAGE_NAME='parallel'
|
||||
PACKAGE_TARNAME='parallel'
|
||||
PACKAGE_VERSION='20100906'
|
||||
PACKAGE_STRING='parallel 20100906'
|
||||
PACKAGE_VERSION='20100922'
|
||||
PACKAGE_STRING='parallel 20100922'
|
||||
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
||||
PACKAGE_URL=''
|
||||
|
||||
|
|
@ -1168,7 +1168,7 @@ if test "$ac_init_help" = "long"; then
|
|||
# Omit some internal or obsolete options to make the list less imposing.
|
||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||
cat <<_ACEOF
|
||||
\`configure' configures parallel 20100906 to adapt to many kinds of systems.
|
||||
\`configure' configures parallel 20100922 to adapt to many kinds of systems.
|
||||
|
||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||
|
||||
|
|
@ -1234,7 +1234,7 @@ fi
|
|||
|
||||
if test -n "$ac_init_help"; then
|
||||
case $ac_init_help in
|
||||
short | recursive ) echo "Configuration of parallel 20100906:";;
|
||||
short | recursive ) echo "Configuration of parallel 20100922:";;
|
||||
esac
|
||||
cat <<\_ACEOF
|
||||
|
||||
|
|
@ -1301,7 +1301,7 @@ fi
|
|||
test -n "$ac_init_help" && exit $ac_status
|
||||
if $ac_init_version; then
|
||||
cat <<\_ACEOF
|
||||
parallel configure 20100906
|
||||
parallel configure 20100922
|
||||
generated by GNU Autoconf 2.67
|
||||
|
||||
Copyright (C) 2010 Free Software Foundation, Inc.
|
||||
|
|
@ -1318,7 +1318,7 @@ cat >config.log <<_ACEOF
|
|||
This file contains any messages produced by compilers while
|
||||
running configure, to aid debugging if configure makes a mistake.
|
||||
|
||||
It was created by parallel $as_me 20100906, which was
|
||||
It was created by parallel $as_me 20100922, which was
|
||||
generated by GNU Autoconf 2.67. Invocation command line was
|
||||
|
||||
$ $0 $@
|
||||
|
|
@ -2133,7 +2133,7 @@ fi
|
|||
|
||||
# Define the identity of the package.
|
||||
PACKAGE='parallel'
|
||||
VERSION='20100906'
|
||||
VERSION='20100922'
|
||||
|
||||
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
|
|
@ -2684,7 +2684,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
|||
# report actual input values of CONFIG_FILES etc. instead of their
|
||||
# values after options handling.
|
||||
ac_log="
|
||||
This file was extended by parallel $as_me 20100906, which was
|
||||
This file was extended by parallel $as_me 20100922, which was
|
||||
generated by GNU Autoconf 2.67. Invocation command line was
|
||||
|
||||
CONFIG_FILES = $CONFIG_FILES
|
||||
|
|
@ -2746,7 +2746,7 @@ _ACEOF
|
|||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||
ac_cs_version="\\
|
||||
parallel config.status 20100906
|
||||
parallel config.status 20100922
|
||||
configured by $0, generated by GNU Autoconf 2.67,
|
||||
with options \\"\$ac_cs_config\\"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
AC_INIT([parallel], [20100906], [bug-parallel@gnu.org])
|
||||
AC_INIT([parallel], [20100922], [bug-parallel@gnu.org])
|
||||
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
|
||||
AC_CONFIG_HEADERS([config.h])
|
||||
AC_CONFIG_FILES([
|
||||
|
|
|
|||
|
|
@ -1,25 +1,10 @@
|
|||
== Retry ==
|
||||
|
||||
Bugfix: --keeporder with -j100% mixed up order of first job
|
||||
parallel: Unittest for --retries
|
||||
|
||||
== FSCONS ==
|
||||
|
||||
Annoncer foredrag til FSCONS
|
||||
|
||||
I will be at FSCONS 2010-11-07 talking about GNU Parallel.
|
||||
http://www.fscons.org/fs/gnu-parallel
|
||||
|
||||
== SQL ==
|
||||
|
||||
Example with %0a as newline
|
||||
sql :pg_foo?'\dt %0a SELECT * FROM users'
|
||||
sql :my_postgres?'\dt %0a SELECT * FROM users'
|
||||
|
||||
cat ~/.sql/aliases | parallel --colsep '\s' sql {1} '"select 0.14159+3;" | grep -q 3.14159 || (echo dead: {1}; exit 1)'
|
||||
|
||||
--list-tables
|
||||
select OBJECT_NAME from user_objects where object_type = 'TABLE';
|
||||
|
||||
== FEX ==
|
||||
|
||||
fex syntax for splitting fields
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ cp parallel-$YYYYMMDD.tar.bz2 /tmp
|
|||
cd /tmp
|
||||
tar xjvf parallel-$YYYYMMDD.tar.bz2
|
||||
cd parallel-$YYYYMMDD
|
||||
./configure && make && sudo make install
|
||||
./configure && make -j && sudo make -j install
|
||||
|
||||
== Upload ==
|
||||
|
||||
|
|
@ -70,6 +70,7 @@ make -j && sudo make -j install
|
|||
== Update OpenSUSE build system ==
|
||||
|
||||
https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
|
||||
cd ~/privat/parallel/packager/obs ; make
|
||||
|
||||
== Update Savannah ==
|
||||
|
||||
|
|
|
|||
7
packager/obs/Makefile
Normal file
7
packager/obs/Makefile
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
all:
|
||||
# cd ../debian/ && make
|
||||
# cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/
|
||||
cp `ls ../../parallel-*.tar.bz2|tail -n1` home\:tange/parallel/
|
||||
cd home\:tange/parallel/ && osc up && osc add *.dsc *.tar.gz && osc ci
|
||||
|
||||
|
||||
1
packager/obs/README
Normal file
1
packager/obs/README
Normal file
|
|
@ -0,0 +1 @@
|
|||
https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
|
||||
105
src/parallel
105
src/parallel
|
|
@ -1542,8 +1542,8 @@ Outputs
|
|||
O2. Send stderr to stderr
|
||||
O3. Send stdout to stdout
|
||||
O4. Order of output can be same as order of input
|
||||
O5. Stdout only from the command
|
||||
O6. Stderr only from the command
|
||||
O5. Stdout only contains stdout from the command
|
||||
O6. Stderr only contains stdout from the command
|
||||
|
||||
Inputs
|
||||
I1. Arguments can be read from stdin
|
||||
|
|
@ -1571,6 +1571,7 @@ Remote execution
|
|||
R6. No config files needed
|
||||
R7. Do not run more than SSHD's MaxStartup can handle
|
||||
R8. Configurable SSH command
|
||||
R9. Retry if connection breaks occationally
|
||||
|
||||
Semaphore
|
||||
S1. Possibility to work as a mutex
|
||||
|
|
@ -1582,37 +1583,47 @@ Legend
|
|||
ID = yes
|
||||
|
||||
As every new version of the programs are not tested the table may be
|
||||
outdated. Please file a bug-report if you find errors.
|
||||
outdated. Please file a bug-report if you find errors (See REPORTING
|
||||
BUGS).
|
||||
|
||||
parallel: E1 E2 E3 E4 E5 O1 O2 O3 O4 O5 O6 I1 I2 I3 I4 I5 I6 I7 M1 M2
|
||||
M3 M4 M5 M6 R1 R2 R3 R4 R5 R6 R7 R8 S1 S2
|
||||
M3 M4 M5 M6 R1 R2 R3 R4 R5 R6 R7 R8 R9 S1 S2
|
||||
|
||||
xargs: E1 - - - - - O2 O3 - O5 O6 I1 I2 - - - - - - M2 M3 -
|
||||
- - - - - - - x - - -
|
||||
- - - - - - - x - - - -
|
||||
|
||||
find -exec: - - - x - x O2 O3 O4 O5 O6 - - - - - - - - M2 M3 -
|
||||
- - - - - - - - - x x
|
||||
- - - - - - - - - - x x
|
||||
|
||||
make -j: E1 - - - E5 O1 O2 O3 - x O6 - - - - - - - - - - - - - - - - - -
|
||||
- - -
|
||||
- - - -
|
||||
|
||||
ppss: E1 E2 ?E3 E4 - O1 - - x - - I1 I2 - - - - I7 M1 - M3 - - M6 R1 R2
|
||||
R3 R4 - - ?R7 - -
|
||||
R3 R4 - - ?R7 ? ? - -
|
||||
|
||||
pexec: E1 - - E4 - O1 O2 O3 - O5 O6 I1 I2 - I4 I5 - - M1 - M3 - - M6 R1
|
||||
- - - - R6 - S1 -
|
||||
- - - - R6 - - - S1 -
|
||||
|
||||
xjobs: TODO
|
||||
xjobs: TODO - Please file a bug-report if you know what features xjobs
|
||||
supports (See REPORTING BUGS).
|
||||
|
||||
prll: TODO
|
||||
prll: TODO - Please file a bug-report if you know what features prll
|
||||
supports (See REPORTING BUGS).
|
||||
|
||||
dxargs: TODO
|
||||
dxargs: TODO - Please file a bug-report if you know what features dxargs
|
||||
supports (See REPORTING BUGS).
|
||||
|
||||
mdm/middelman: TODO
|
||||
mdm/middelman: TODO - Please file a bug-report if you know what
|
||||
features mdm/middelman supports (See REPORTING BUGS).
|
||||
|
||||
xapply: TODO
|
||||
xapply: TODO - Please file a bug-report if you know what features xapply
|
||||
supports (See REPORTING BUGS).
|
||||
|
||||
ClusterSSH: TODO
|
||||
paexec: TODO - Please file a bug-report if you know what features paexec
|
||||
supports (See REPORTING BUGS).
|
||||
|
||||
ClusterSSH: TODO - Please file a bug-report if you know what features ClusterSSH
|
||||
supports (See REPORTING BUGS).
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN xargs AND GNU Parallel
|
||||
|
|
@ -1876,7 +1887,7 @@ echo unzip 1.zip >> /var/run/my_named_pipe;
|
|||
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN prll AND GNU parallel
|
||||
=head2 DIFFERENCES BETWEEN prll AND GNU Parallel
|
||||
|
||||
B<prll> is also a tool for running jobs in parallel. It does not
|
||||
support running jobs on remote computers.
|
||||
|
|
@ -1979,6 +1990,51 @@ B<11> xapply -f '[ -f %1 ] && echo %1' List | ...
|
|||
B<11> parallel '[ -f {} ] && echo {}' < List | ...
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel
|
||||
|
||||
B<paexec> can run jobs in parallel on both the local and remote computers.
|
||||
|
||||
B<paexec> requires commands to print a blank line as the last
|
||||
output. This means you will have to write a wrapper for most programs.
|
||||
|
||||
B<paexec> has a job dependency facility so a job can depend on another
|
||||
job to be executed successfully. Sort of a poor-man's B<make>.
|
||||
|
||||
Here are the examples from B<paexec>'s example catalog with the equivalent
|
||||
using GNU B<parallel>:
|
||||
|
||||
=over 1
|
||||
|
||||
=item 1_div_X_run:
|
||||
|
||||
../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
|
||||
parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
|
||||
|
||||
=item all_substr_run:
|
||||
|
||||
../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
|
||||
parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
|
||||
|
||||
=item cc_wrapper_run:
|
||||
|
||||
../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
|
||||
-n 'host1 host2' \
|
||||
-t '/usr/bin/ssh -x' <<EOF [...]
|
||||
parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
|
||||
-S host1,host2 <<EOF [...]
|
||||
# This is not exactly the same, but avoids the wrapper
|
||||
parallel gcc -O2 -c -o {.}.o {} \
|
||||
-S host1,host2 <<EOF [...]
|
||||
|
||||
=item toupper_run:
|
||||
|
||||
../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
|
||||
parallel echo {} '|' ./toupper_cmd <<EOF [...]
|
||||
# Without the wrapper:
|
||||
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
|
||||
|
||||
=back
|
||||
|
||||
=head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel
|
||||
|
||||
ClusterSSH solves a different problem than GNU B<parallel>.
|
||||
|
|
@ -1993,7 +2049,7 @@ computing. If more than one machine is listed in B<-S> GNU B<parallel> may
|
|||
only use one of these (e.g. if there are 8 jobs to be run and one
|
||||
machine has 8 cores).
|
||||
|
||||
GNU B<parallel> can be used as a poor-mans version of ClusterSSH:
|
||||
GNU B<parallel> can be used as a poor-man's version of ClusterSSH:
|
||||
|
||||
B<cat hostlist | parallel ssh {} do_stuff>
|
||||
|
||||
|
|
@ -2012,9 +2068,16 @@ echo 1,2,3 | parallel -vkd, echo a{}
|
|||
|
||||
=head2 Startup speed
|
||||
|
||||
GNU B<parallel> is slow at starting up. Half of the startup time is
|
||||
spent finding the maximal length of a command line. Setting B<-s> will
|
||||
remove this part of the startup time.
|
||||
GNU B<parallel> is slow at starting up. Half of the startup time on
|
||||
the local computer is spent finding the maximal length of a command
|
||||
line. Setting B<-s> will remove this part of the startup time.
|
||||
|
||||
When using multiple computers GNU B<parallel> opens B<ssh> connections
|
||||
to them to figure out how many connections can be used reliably
|
||||
simultaneously (Namely SSHD's MaxStartup). This test is done for each
|
||||
host in serial, so if your --sshloginfile contains many hosts it may
|
||||
be slow.
|
||||
|
||||
|
||||
=head1 REPORTING BUGS
|
||||
|
||||
|
|
@ -2215,7 +2278,7 @@ sub acquire_semaphore {
|
|||
sub parse_options {
|
||||
# Returns: N/A
|
||||
# Defaults:
|
||||
$Global::version = 20100906;
|
||||
$Global::version = 20100922;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::debug = 0;
|
||||
$Global::verbose = 0;
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -452,7 +452,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20100914;
|
||||
$Global::version = 20100922;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
20
unittest/tests-to-run/test32.sh
Normal file
20
unittest/tests-to-run/test32.sh
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo '### Test of --retries'
|
||||
seq 1 10 | parallel --retries 2 -v -S 4.3.2.1,: echo
|
||||
|
||||
echo '### Test of --retries - it should run 13 jobs in total'
|
||||
seq 0 12 | parallel --progress -kj100% --retries 1 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
|
||||
perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \
|
||||
perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"'
|
||||
|
||||
echo '### Test of --retries - it should run 25 jobs in total'
|
||||
seq 0 12 | parallel --progress -kj100% --retries 2 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
|
||||
perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \
|
||||
perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"'
|
||||
|
||||
echo '### Test of --retries - it should run 49 jobs in total'
|
||||
seq 0 12 | parallel --progress -kj100% --retries 4 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
|
||||
perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \
|
||||
perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"'
|
||||
|
||||
Loading…
Reference in a new issue