diff --git a/NEWS b/NEWS index fe710da1..80bffe7f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -2010XXXX +20100722 * Arguments can now be given on the command line. Example: parallel bzip2 ::: *.txt diff --git a/autom4te.cache/output.0 b/autom4te.cache/output.0 index 4b74731a..30667204 100644 --- a/autom4te.cache/output.0 +++ b/autom4te.cache/output.0 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100620. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100620' -PACKAGE_STRING='parallel 20100620' +PACKAGE_VERSION='20100722' +PACKAGE_STRING='parallel 20100722' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100620 to adapt to many kinds of systems. +\`configure' configures parallel 20100722 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100620:";; + short | recursive ) echo "Configuration of parallel 20100722:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100620 +parallel configure 20100722 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100620, which was +It was created by parallel $as_me 20100722, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100620' + VERSION='20100722' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100620, which was +This file was extended by parallel $as_me 20100722, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100620 +parallel config.status 20100722 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/output.1 b/autom4te.cache/output.1 index 4b74731a..30667204 100644 --- a/autom4te.cache/output.1 +++ b/autom4te.cache/output.1 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100620. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100722. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100620' -PACKAGE_STRING='parallel 20100620' +PACKAGE_VERSION='20100722' +PACKAGE_STRING='parallel 20100722' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100620 to adapt to many kinds of systems. +\`configure' configures parallel 20100722 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100620:";; + short | recursive ) echo "Configuration of parallel 20100722:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100620 +parallel configure 20100722 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100620, which was +It was created by parallel $as_me 20100722, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100620' + VERSION='20100722' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100620, which was +This file was extended by parallel $as_me 20100722, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100620 +parallel config.status 20100722 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/traces.1 b/autom4te.cache/traces.1 index cf20f12f..71e2f0ff 100644 --- a/autom4te.cache/traces.1 +++ b/autom4te.cache/traces.1 @@ -1,4 +1,4 @@ -m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100620], [bug-parallel@gnu.org]) +m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100722], [bug-parallel@gnu.org]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS']) diff --git a/configure b/configure index 82a142b4..075ac74b 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.65 for parallel 20100620. +# Generated by GNU Autoconf 2.65 for parallel 20100722. # # Report bugs to . # @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100620' -PACKAGE_STRING='parallel 20100620' +PACKAGE_VERSION='20100722' +PACKAGE_STRING='parallel 20100722' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100620 to adapt to many kinds of systems. +\`configure' configures parallel 20100722 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100620:";; + short | recursive ) echo "Configuration of parallel 20100722:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100620 +parallel configure 20100722 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100620, which was +It was created by parallel $as_me 20100722, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100620' + VERSION='20100722' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100620, which was +This file was extended by parallel $as_me 20100722, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100620 +parallel config.status 20100722 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 869e7564..e3f0bf70 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20100620], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20100722], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 7fed4ab2..66ed7645 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,18 +1,13 @@ -Code refactor: move :::: and ::: processing. +Unittest: eof string on ::: +echo 'a%c%b' | parallel --colsep % echo {1} {3} {2} +(echo 'a%c%b'; echo a%c%b%d) | parallel --colsep % echo {1} {3} {2} {4} +(echo a%c%b; echo d%f%e) | parallel --colsep % echo {1} {3} {2} +parallel --colsep % echo {1} {3} {2} ::: a%c%b d%f%e -Example: +parallel --colsep % echo {1} {3} {2} ::: a%c%b -Given a list of URLs, list all URLs that fail to download. Print the -line number and the URL. - -cat urlfile | parallel "wget {} 2>>/dev/null || grep -n {} urlfile" - -EXAMPLE: - -Copy foo.es.ext to foo.ext: - -ls *.es.* | perl -pe 'print; s/\.es//' | parallel -N2 cp {1} {2} +parallel --colsep % echo {1} {3} {2} {4} ::: a%c%b a%c%b%d Import sql diff --git a/doc/release_new_version b/doc/release_new_version index 050beb20..de8e0635 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -53,6 +53,7 @@ echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org == Update Savannah == https://savannah.gnu.org/news/submit.php?group=parallel +https://savannah.gnu.org/news/approve.php?group=parallel doc/pod2savannah_publicinfo src/parallel | klipper-stdin https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel @@ -63,6 +64,7 @@ http://www.gnu.org/software/parallel/ http://www.gnu.org/software/parallel/man.html pod2html src/parallel > ../parallel-web/parallel/man.html +cd ../parallel-web/parallel cvs ci == Update Freshmeat == @@ -76,10 +78,11 @@ Newsgroups: comp.unix.shell,comp.unix.admin <<<<< to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org +cc:Peter Simons , Sandro Cazzaniga -Subject: GNU Parallel 20100620 released +Subject: GNU Parallel 20100722 released -GNU Parallel 20100620 has been released. It is available for +GNU Parallel 20100722 has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: @@ -89,6 +92,7 @@ New in this release: * xapply like functionality. Example: parallel diff {1} {2} :::: filelist1 filelist2 + parallel diff {1} {2} ">"{1.}-{2.}.diff :::: filelist1 filelist2 * Arch Linux package. Thanks to Peter Simons @@ -119,6 +123,7 @@ possible to use output from GNU Parallel as input for other programs. You can find more about GNU Parallel at: http://www.gnu.org/software/parallel/ -Watch the intro video on http://www.youtube.com/watch?v=OpaiGYxkSuQ +Watch the intro video on http://www.youtube.com/watch?v=OpaiGYxkSuQ or +at http://tinyogg.com/watch/TORaR/ and http://tinyogg.com/watch/hfxKj/ >>>>> diff --git a/src/parallel b/src/parallel index d974e670..41dc70b2 100755 --- a/src/parallel +++ b/src/parallel @@ -847,6 +847,11 @@ Even small shell scripts can be run by GNU B: B +Given a list of URLs, list all URLs that fail to download. Print the +line number and the URL. + +B>B + =head1 EXAMPLE: Removing file extension when processing files @@ -1099,6 +1104,15 @@ Convert *.mp3 to *.ogg running one process per CPU core on local computer and se parallel --trc {.}.ogg -j+0 -S server2,: \ 'mpg321 -w - {} | oggenc -q0 - -o {.}.ogg' ::: *.mp3 +=head1 EXAMPLE: Use multiple inputs in one command + +Copy files like foo.es.ext to foo.ext: + +B + +The perl command spits out 2 lines for each input. GNU B +takes 2 inputs (using B<-N2>) and replaces {1} and {2} with the inputs. + =head1 EXAMPLE: Working as cat | sh. Ressource inexpensive jobs and evaluation @@ -1827,7 +1841,7 @@ if($::opt_halt_on_error) { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20100709; + $Global::version = 20100722; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -1905,6 +1919,7 @@ sub parse_options { "eof|e:s" => \$::opt_E, "max-args|n=i" => \$::opt_n, "max-replace-args|N=i" => \$::opt_N, + "colsep|col-sep=s" => \$::opt_colsep, "help|h" => \$::opt_help, "L=i" => \$::opt_L, "max-lines|l:i" => \$::opt_l, @@ -1969,7 +1984,7 @@ sub parse_options { $Global::input_is_filename ||= (@ARGV); if(@::opt_a) { - # must be done after + # must be done after # convert_argfiles_from_command_line_to_multiple_opt_a if($#::opt_a == 0) { # One -a => xargs compatibility @@ -1980,8 +1995,8 @@ sub parse_options { } } - if(($::opt_l || $::opt_L || $::opt_n || $::opt_N || $::opt_s) and not - ($::opt_m or $::opt_X)) { + if(($::opt_l || $::opt_L || $::opt_n || $::opt_N || $::opt_s || + $::opt_colsep) and not ($::opt_m or $::opt_X)) { # The options --max-line, -l, -L, --max-args, -n, --max-chars, -s # do not make sense without -X or -m # so default to -X @@ -2041,18 +2056,13 @@ sub read_args_from_command_line { # Returns: # @ARGV without ::: and following args if(not @::opt_a) { push @::opt_a, "/dev/null"; } - # Input: @ARGV = command option ::: arg arg arg my @new_argv = (); while(@ARGV) { my $arg = shift @ARGV; if($arg eq $Global::arg_sep) { $Global::input_is_filename = (@new_argv); - if($Global::input_is_filename) { - unget_arg(shell_quote(@ARGV)); - } else { - unget_arg(@ARGV); - } + push @Global::unget_lines, @ARGV; $Global::total_jobs += @ARGV; @ARGV=(); last; @@ -2921,6 +2931,8 @@ sub max { # $Global::total_started = total jobs started # $Global::total_jobs = total jobs to be started at all # $Global::total_completed = total jobs completed +# @Global::unget_arg = arguments quoted as needed ready to use +# @Global::unget_lines = raw argument lines - needs quoting and splitting sub init_run_jobs { # Remember the original STDOUT and STDERR @@ -3006,14 +3018,21 @@ sub more_arguments { # Returns: # whether there are more arguments to be processed or not my $fh = shift || $Global::argfile; - return (@Global::unget_arg or not eof $fh); + return (@Global::unget_arg or @Global::unget_lines or not eof $fh); } sub get_next_arg_from_fh { # Returns: - # next argument from file handle + # next argument from file handle - quoted if needed # undef if end of file my $fh = shift; + if(not $Private::unget{$fh}) { + @{$Private::unget{$fh}} = (); + } + my $unget_ref = $Private::unget{$fh}; + if(@$unget_ref) { + return shift @$unget_ref; + } if(not more_arguments($fh)) { return undef; } @@ -3037,6 +3056,13 @@ sub get_next_arg_from_fh { $arg .= get_next_arg_from_fh($fh); } } + if($::opt_colsep) { + # split this into columns + push @$unget_ref, split /$::opt_colsep/o, $arg; + $::opt_N = $#$unget_ref+1; + $Global::max_number_of_args = $::opt_N; + $arg = shift @$unget_ref; + } if($Global::input_is_filename) { $arg = shell_quote($arg); } @@ -3050,6 +3076,41 @@ sub get_next_arg { my $arg; if(@Global::unget_arg) { $arg = shift @Global::unget_arg; + } elsif(@Global::unget_lines) { + $arg = shift @Global::unget_lines; + if($Global::end_of_file_string and + $arg eq $Global::end_of_file_string) { + # Ignore the rest of input file + @Global::unget_lines = (); + return undef; + } + if($Global::ignore_empty) { + if($arg =~ /^\s*$/) { + return get_next_arg(); + } + } + if($Global::max_lines and more_arguments()) { + if($arg =~ /\s$/) { + # Trailing space => continued on next line + $arg .= get_next_arg(); + } + } + if($::opt_colsep) { + # split this into columns + my @columns = split /$::opt_colsep/o, $arg; + $::opt_N = $#columns+1; + $Global::max_number_of_args = $::opt_N; + if($Global::input_is_filename) { + unget_arg(shell_quote(@columns)); + } else { + unget_arg(@columns); + } + $arg = get_next_arg(); + } else { + if($Global::input_is_filename) { + $arg = shell_quote($arg); + } + } } else { $arg = get_next_arg_from_fh($Global::argfile); if(defined $arg) { diff --git a/unittest/Makefile b/unittest/Makefile index 6dbeef3f..e039ef3d 100644 --- a/unittest/Makefile +++ b/unittest/Makefile @@ -8,7 +8,8 @@ unittest: ../src/parallel tests-to-run/* wanted-results/* echo 1+2 | bc | mop || (echo bc is required for unittest; /bin/false) stdout gawk | mop || (echo gawk is required for unittest; /bin/false) expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for unittest; /bin/false) - sh Start.sh + time sh Start.sh + date clean: rm -rf input-files/random_dirs_no_newline