--colsep implemented. Unittest for --colsep missing, but old unittest passes.

This commit is contained in:
Ole Tange 2010-07-23 00:39:51 +02:00
parent e8a30d9fac
commit 5fea6dfd98
10 changed files with 123 additions and 61 deletions

2
NEWS
View file

@ -1,4 +1,4 @@
2010XXXX
20100722
* Arguments can now be given on the command line. Example:
parallel bzip2 ::: *.txt

View file

@ -1,6 +1,6 @@
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100620.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722.
@%:@
@%:@ Report bugs to <bug-parallel@gnu.org>.
@%:@
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100620'
PACKAGE_STRING='parallel 20100620'
PACKAGE_VERSION='20100722'
PACKAGE_STRING='parallel 20100722'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20100620 to adapt to many kinds of systems.
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100620:";;
short | recursive ) echo "Configuration of parallel 20100722:";;
esac
cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20100620
parallel configure 20100722
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100620, which was
It was created by parallel $as_me 20100722, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20100620'
VERSION='20100722'
cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20100620, which was
This file was extended by parallel $as_me 20100722, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20100620
parallel config.status 20100722
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"

View file

@ -1,6 +1,6 @@
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100620.
@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722.
@%:@
@%:@ Report bugs to <bug-parallel@gnu.org>.
@%:@
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100620'
PACKAGE_STRING='parallel 20100620'
PACKAGE_VERSION='20100722'
PACKAGE_STRING='parallel 20100722'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20100620 to adapt to many kinds of systems.
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100620:";;
short | recursive ) echo "Configuration of parallel 20100722:";;
esac
cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20100620
parallel configure 20100722
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100620, which was
It was created by parallel $as_me 20100722, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20100620'
VERSION='20100722'
cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20100620, which was
This file was extended by parallel $as_me 20100722, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20100620
parallel config.status 20100722
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"

View file

@ -1,4 +1,4 @@
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100620], [bug-parallel@gnu.org])
m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100722], [bug-parallel@gnu.org])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS'])

20
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.65 for parallel 20100620.
# Generated by GNU Autoconf 2.65 for parallel 20100722.
#
# Report bugs to <bug-parallel@gnu.org>.
#
@ -551,8 +551,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20100620'
PACKAGE_STRING='parallel 20100620'
PACKAGE_VERSION='20100722'
PACKAGE_STRING='parallel 20100722'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20100620 to adapt to many kinds of systems.
\`configure' configures parallel 20100722 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1233,7 +1233,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20100620:";;
short | recursive ) echo "Configuration of parallel 20100722:";;
esac
cat <<\_ACEOF
@ -1300,7 +1300,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20100620
parallel configure 20100722
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20100620, which was
It was created by parallel $as_me 20100722, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@ -2125,7 +2125,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20100620'
VERSION='20100722'
cat >>confdefs.h <<_ACEOF
@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20100620, which was
This file was extended by parallel $as_me 20100722, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2737,7 +2737,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20100620
parallel config.status 20100722
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"

View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20100620], [bug-parallel@gnu.org])
AC_INIT([parallel], [20100722], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([

View file

@ -1,18 +1,13 @@
Code refactor: move :::: and ::: processing.
Unittest: eof string on :::
echo 'a%c%b' | parallel --colsep % echo {1} {3} {2}
(echo 'a%c%b'; echo a%c%b%d) | parallel --colsep % echo {1} {3} {2} {4}
(echo a%c%b; echo d%f%e) | parallel --colsep % echo {1} {3} {2}
parallel --colsep % echo {1} {3} {2} ::: a%c%b d%f%e
Example:
parallel --colsep % echo {1} {3} {2} ::: a%c%b
Given a list of URLs, list all URLs that fail to download. Print the
line number and the URL.
cat urlfile | parallel "wget {} 2>>/dev/null || grep -n {} urlfile"
EXAMPLE:
Copy foo.es.ext to foo.ext:
ls *.es.* | perl -pe 'print; s/\.es//' | parallel -N2 cp {1} {2}
parallel --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d
Import sql

View file

@ -53,6 +53,7 @@ echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org
== Update Savannah ==
https://savannah.gnu.org/news/submit.php?group=parallel
https://savannah.gnu.org/news/approve.php?group=parallel
doc/pod2savannah_publicinfo src/parallel | klipper-stdin
https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel
@ -63,6 +64,7 @@ http://www.gnu.org/software/parallel/
http://www.gnu.org/software/parallel/man.html
pod2html src/parallel > ../parallel-web/parallel/man.html
cd ../parallel-web/parallel
cvs ci
== Update Freshmeat ==
@ -76,10 +78,11 @@ Newsgroups: comp.unix.shell,comp.unix.admin
<<<<<
to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org
cc:Peter Simons <simons@cryp.to>, Sandro Cazzaniga <kharec@mandriva.org>
Subject: GNU Parallel 20100620 released
Subject: GNU Parallel 20100722 released
GNU Parallel 20100620 has been released. It is available for
GNU Parallel 20100722 has been released. It is available for
download at: http://ftp.gnu.org/gnu/parallel/
New in this release:
@ -89,6 +92,7 @@ New in this release:
* xapply like functionality. Example:
parallel diff {1} {2} :::: filelist1 filelist2
parallel diff {1} {2} ">"{1.}-{2.}.diff :::: filelist1 filelist2
* Arch Linux package. Thanks to Peter Simons <simons at cryp dot to>
@ -119,6 +123,7 @@ possible to use output from GNU Parallel as input for other programs.
You can find more about GNU Parallel at:
http://www.gnu.org/software/parallel/
Watch the intro video on http://www.youtube.com/watch?v=OpaiGYxkSuQ
Watch the intro video on http://www.youtube.com/watch?v=OpaiGYxkSuQ or
at http://tinyogg.com/watch/TORaR/ and http://tinyogg.com/watch/hfxKj/
>>>>>

View file

@ -847,6 +847,11 @@ Even small shell scripts can be run by GNU B<parallel>:
B<find . | parallel 'a={}; name=${a##*/}; upper=$(echo "$name" | tr "[:lower:]" "[:upper:]"); echo "$name - $upper"'>
Given a list of URLs, list all URLs that fail to download. Print the
line number and the URL.
B<cat urlfile | parallel "wget {} 2>>B</dev/null || grep -n {} urlfile">
=head1 EXAMPLE: Removing file extension when processing files
@ -1099,6 +1104,15 @@ Convert *.mp3 to *.ogg running one process per CPU core on local computer and se
parallel --trc {.}.ogg -j+0 -S server2,: \
'mpg321 -w - {} | oggenc -q0 - -o {.}.ogg' ::: *.mp3
=head1 EXAMPLE: Use multiple inputs in one command
Copy files like foo.es.ext to foo.ext:
B<ls *.es.* | perl -pe 'print; s/\.es//' | parallel -N2 cp {1} {2}>
The perl command spits out 2 lines for each input. GNU B<parallel>
takes 2 inputs (using B<-N2>) and replaces {1} and {2} with the inputs.
=head1 EXAMPLE: Working as cat | sh. Ressource inexpensive jobs and evaluation
@ -1827,7 +1841,7 @@ if($::opt_halt_on_error) {
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20100709;
$Global::version = 20100722;
$Global::progname = 'parallel';
$Global::debug = 0;
$Global::verbose = 0;
@ -1905,6 +1919,7 @@ sub parse_options {
"eof|e:s" => \$::opt_E,
"max-args|n=i" => \$::opt_n,
"max-replace-args|N=i" => \$::opt_N,
"colsep|col-sep=s" => \$::opt_colsep,
"help|h" => \$::opt_help,
"L=i" => \$::opt_L,
"max-lines|l:i" => \$::opt_l,
@ -1969,7 +1984,7 @@ sub parse_options {
$Global::input_is_filename ||= (@ARGV);
if(@::opt_a) {
# must be done after
# must be done after
# convert_argfiles_from_command_line_to_multiple_opt_a
if($#::opt_a == 0) {
# One -a => xargs compatibility
@ -1980,8 +1995,8 @@ sub parse_options {
}
}
if(($::opt_l || $::opt_L || $::opt_n || $::opt_N || $::opt_s) and not
($::opt_m or $::opt_X)) {
if(($::opt_l || $::opt_L || $::opt_n || $::opt_N || $::opt_s ||
$::opt_colsep) and not ($::opt_m or $::opt_X)) {
# The options --max-line, -l, -L, --max-args, -n, --max-chars, -s
# do not make sense without -X or -m
# so default to -X
@ -2041,18 +2056,13 @@ sub read_args_from_command_line {
# Returns:
# @ARGV without ::: and following args
if(not @::opt_a) { push @::opt_a, "/dev/null"; }
# Input: @ARGV = command option ::: arg arg arg
my @new_argv = ();
while(@ARGV) {
my $arg = shift @ARGV;
if($arg eq $Global::arg_sep) {
$Global::input_is_filename = (@new_argv);
if($Global::input_is_filename) {
unget_arg(shell_quote(@ARGV));
} else {
unget_arg(@ARGV);
}
push @Global::unget_lines, @ARGV;
$Global::total_jobs += @ARGV;
@ARGV=();
last;
@ -2921,6 +2931,8 @@ sub max {
# $Global::total_started = total jobs started
# $Global::total_jobs = total jobs to be started at all
# $Global::total_completed = total jobs completed
# @Global::unget_arg = arguments quoted as needed ready to use
# @Global::unget_lines = raw argument lines - needs quoting and splitting
sub init_run_jobs {
# Remember the original STDOUT and STDERR
@ -3006,14 +3018,21 @@ sub more_arguments {
# Returns:
# whether there are more arguments to be processed or not
my $fh = shift || $Global::argfile;
return (@Global::unget_arg or not eof $fh);
return (@Global::unget_arg or @Global::unget_lines or not eof $fh);
}
sub get_next_arg_from_fh {
# Returns:
# next argument from file handle
# next argument from file handle - quoted if needed
# undef if end of file
my $fh = shift;
if(not $Private::unget{$fh}) {
@{$Private::unget{$fh}} = ();
}
my $unget_ref = $Private::unget{$fh};
if(@$unget_ref) {
return shift @$unget_ref;
}
if(not more_arguments($fh)) {
return undef;
}
@ -3037,6 +3056,13 @@ sub get_next_arg_from_fh {
$arg .= get_next_arg_from_fh($fh);
}
}
if($::opt_colsep) {
# split this into columns
push @$unget_ref, split /$::opt_colsep/o, $arg;
$::opt_N = $#$unget_ref+1;
$Global::max_number_of_args = $::opt_N;
$arg = shift @$unget_ref;
}
if($Global::input_is_filename) {
$arg = shell_quote($arg);
}
@ -3050,6 +3076,41 @@ sub get_next_arg {
my $arg;
if(@Global::unget_arg) {
$arg = shift @Global::unget_arg;
} elsif(@Global::unget_lines) {
$arg = shift @Global::unget_lines;
if($Global::end_of_file_string and
$arg eq $Global::end_of_file_string) {
# Ignore the rest of input file
@Global::unget_lines = ();
return undef;
}
if($Global::ignore_empty) {
if($arg =~ /^\s*$/) {
return get_next_arg();
}
}
if($Global::max_lines and more_arguments()) {
if($arg =~ /\s$/) {
# Trailing space => continued on next line
$arg .= get_next_arg();
}
}
if($::opt_colsep) {
# split this into columns
my @columns = split /$::opt_colsep/o, $arg;
$::opt_N = $#columns+1;
$Global::max_number_of_args = $::opt_N;
if($Global::input_is_filename) {
unget_arg(shell_quote(@columns));
} else {
unget_arg(@columns);
}
$arg = get_next_arg();
} else {
if($Global::input_is_filename) {
$arg = shell_quote($arg);
}
}
} else {
$arg = get_next_arg_from_fh($Global::argfile);
if(defined $arg) {

View file

@ -8,7 +8,8 @@ unittest: ../src/parallel tests-to-run/* wanted-results/*
echo 1+2 | bc | mop || (echo bc is required for unittest; /bin/false)
stdout gawk | mop || (echo gawk is required for unittest; /bin/false)
expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for unittest; /bin/false)
sh Start.sh
time sh Start.sh
date
clean:
rm -rf input-files/random_dirs_no_newline