From a16d120711ef7e6a31a672519c335a31f82d5771 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 21 Dec 2010 18:08:16 +0100 Subject: [PATCH] Implemented --tty as the old default of assigning a tty to the first job causes problems. Cleanup if --workdir <> . did not work. Fixed. --- README | 2 +- configure | 20 ++-- configure.ac | 2 +- doc/FUTURE_IDEAS | 6 +- doc/release_new_version | 32 ++++-- src/Makefile.am | 24 ++--- src/Makefile.in | 24 ++--- src/parallel | 86 +++++++++------- src/parallel.pod | 81 ++++++++------- src/sem.pod | 11 ++ src/sql | 2 +- testsuite/tests-to-run/test15.sh | 4 +- testsuite/tests-to-run/test25.sh | 4 +- testsuite/tests-to-run/test47.sh | 170 +++++++++++++++++++++++++++++++ testsuite/wanted-results/test19 | 4 +- testsuite/wanted-results/test47 | 127 +++++++++++++++++++++++ 16 files changed, 479 insertions(+), 120 deletions(-) create mode 100644 testsuite/tests-to-run/test47.sh create mode 100644 testsuite/wanted-results/test47 diff --git a/README b/README index a8005f6f..849f1e84 100644 --- a/README +++ b/README @@ -6,7 +6,7 @@ Please send problems and feedback to bug-parallel@gnu.org. = Presentation of GNU Parallel = GNU Parallel is a shell tool for executing jobs in parallel using one -or more machines. A job is typically a single command or a small +or more computers. A job is typically a single command or a small script that has to be run for each of the lines in the input. The typical input is a list of files, a list of hosts, a list of users, or a list of tables. diff --git a/configure b/configure index d2d12dbe..46f838e4 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.67 for parallel 20101206. +# Generated by GNU Autoconf 2.67 for parallel 20101220. # # Report bugs to . # @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20101206' -PACKAGE_STRING='parallel 20101206' +PACKAGE_VERSION='20101220' +PACKAGE_STRING='parallel 20101220' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1168,7 +1168,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20101206 to adapt to many kinds of systems. +\`configure' configures parallel 20101220 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1234,7 +1234,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20101206:";; + short | recursive ) echo "Configuration of parallel 20101220:";; esac cat <<\_ACEOF @@ -1301,7 +1301,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20101206 +parallel configure 20101220 generated by GNU Autoconf 2.67 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1318,7 +1318,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20101206, which was +It was created by parallel $as_me 20101220, which was generated by GNU Autoconf 2.67. Invocation command line was $ $0 $@ @@ -2133,7 +2133,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20101206' + VERSION='20101220' cat >>confdefs.h <<_ACEOF @@ -2684,7 +2684,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20101206, which was +This file was extended by parallel $as_me 20101220, which was generated by GNU Autoconf 2.67. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2746,7 +2746,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20101206 +parallel config.status 20101220 configured by $0, generated by GNU Autoconf 2.67, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index ec86589b..454797e2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20101206], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20101220], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 23dda270..8f238c0a 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,3 +1,6 @@ +cleanup of transferred files in workdir fixed. +-T implemented as ssh/rsync sometimes hang due to getting a tty. + Til QUOTING: FN="two spaces" @@ -309,7 +312,8 @@ distribute the arguments evenly if running -X. =head1 options -One char options not used: A F G K O Q R T Z +One char options not used: A F G K O Q R T Z c f +-T terminal =head1 sem diff --git a/doc/release_new_version b/doc/release_new_version index 1f32f453..4f35979a 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -22,7 +22,7 @@ rm -fr autom4te.cache aclocal.m4 config.h config.h.in config.log Makefile.in mis rm -rf src/Makefile.in autoreconf --install -W gnu ./configure -make && sudo make install +make -j && sudo make install == Testsuite == @@ -89,9 +89,14 @@ https://savannah.gnu.org/news/approve.php?group=parallel http://www.gnu.org/software/parallel/ http://www.gnu.org/software/parallel/man.html +http://www.gnu.org/software/parallel/sql.html +http://www.gnu.org/software/parallel/sem.html +http://www.gnu.org/software/parallel/niceload.html -pod2html src/parallel > ../parallel-web/parallel/man.html +pod2html src/parallel.pod > ../parallel-web/parallel/man.html pod2html src/sql > ../parallel-web/parallel/sql.html +pod2html src/niceload > ../parallel-web/parallel/niceload.html +pod2html src/sem.pod > ../parallel-web/parallel/sem.html cd ../parallel-web/parallel cvs up cvs ci @@ -125,11 +130,13 @@ cc:Peter Simons , Sandro Cazzaniga , Andrew McFague , Steven M. Christensen , Chris Howey , Fethican Coşkuner , Rogério Brito , Jonathan Palardy , - Koen Vervloesem , R. Tyler Croy + Koen Vervloesem , R. Tyler Croy , + ryoichiro.suzuki@gmail.com,kerick@shiftedbit.net, + Christian Faulhammer -Subject: GNU Parallel 20101202 released +Subject: GNU Parallel 20101222 released -GNU Parallel 20101202 has been released. It is available for +GNU Parallel 20101222 has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: @@ -146,13 +153,26 @@ New in this release: * Implemented --nice set the niceness of jobs running both locally and remotely. +* Implemented --dry-run to print the job without running it. + +* Implemented --tty as the old default of assigning a tty to the first + job causes problems. + * Review with focus on clusters. Thanks to Taylor Gillespie http://www.unixpronews.com/unixpronews-49-20101019GNUParallelSpeedUpProcessingWithMulticoresClusters.html +* Review with focus on protein similarity. + http://kevinformatics.tumblr.com/post/2142473893/cluster-like-computing-using-gnu-parallel + +* Review in Spanish. + http://gr3p.com/2010/12/gnu-parallel-acelera-tus-scripts-en-linux + +* Quite a few bug fixes and man page updates. + = About GNU Parallel = GNU Parallel is a shell tool for executing jobs in parallel using one -or more machines. A job is typically a single command or a small +or more computers. A job is typically a single command or a small script that has to be run for each of the lines in the input. The typical input is a list of files, a list of hosts, a list of users, a list of URLs, or a list of tables. diff --git a/src/Makefile.am b/src/Makefile.am index a810920b..5d3306df 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,21 +18,21 @@ niceload.1: niceload Makefile pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/niceload > $(srcdir)/niceload.1 -parallel.html: parallel Makefile - pod2html $(srcdir)/parallel > $(srcdir)/parallel.html - rm -f $(srcdir)/pod2htm* +parallel.html: parallel Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/parallel.pod > $(srcdir)/parallel.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* -sem.html: sem.pod Makefile - pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html - rm -f $(srcdir)/pod2htm* +sem.html: sem.pod Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* -sql.html: sql Makefile - pod2html $(srcdir)/sql > $(srcdir)/sql.html - rm -f $(srcdir)/pod2htm* +sql.html: sql Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/sql > $(srcdir)/sql.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* -niceload.html: niceload Makefile - pod2html $(srcdir)/niceload > $(srcdir)/niceload.html - rm -f $(srcdir)/pod2htm* +niceload.html: niceload Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/niceload > $(srcdir)/niceload.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* sem: parallel ln -fs parallel sem diff --git a/src/Makefile.in b/src/Makefile.in index 97d9a127..e2a71a8c 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -463,21 +463,21 @@ niceload.1: niceload Makefile pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/niceload > $(srcdir)/niceload.1 -parallel.html: parallel Makefile - pod2html $(srcdir)/parallel > $(srcdir)/parallel.html - rm -f $(srcdir)/pod2htm* +parallel.html: parallel Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/parallel.pod > $(srcdir)/parallel.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* -sem.html: sem.pod Makefile - pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html - rm -f $(srcdir)/pod2htm* +sem.html: sem.pod Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/sem.pod > $(srcdir)/sem.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* -sql.html: sql Makefile - pod2html $(srcdir)/sql > $(srcdir)/sql.html - rm -f $(srcdir)/pod2htm* +sql.html: sql Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/sql > $(srcdir)/sql.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* -niceload.html: niceload Makefile - pod2html $(srcdir)/niceload > $(srcdir)/niceload.html - rm -f $(srcdir)/pod2htm* +niceload.html: niceload Makefile sem + ./sem --fg --id pod2html pod2html $(srcdir)/niceload > $(srcdir)/niceload.html + ./sem --fg --id pod2html rm -f $(srcdir)/pod2htm* sem: parallel ln -fs parallel sem diff --git a/src/parallel b/src/parallel index 1f7fd487..1461365a 100755 --- a/src/parallel +++ b/src/parallel @@ -142,6 +142,7 @@ sub get_options_from_array { "workdir|W=s" => \$::opt_workdir, "tmpdir=s" => \$::opt_tmpdir, "tempdir=s" => \$::opt_tmpdir, + "tty|T" => \$::opt_tty, "halt-on-error|H=s" => \$::opt_halt_on_error, "retries=i" => \$::opt_retries, "dry-run|dryrun" => \$::opt_dryrun, @@ -192,7 +193,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20101217; + $Global::version = 20101220; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -234,7 +235,9 @@ sub parse_options { if(defined $::opt_U) { $Global::replace{'{.}'} = $::opt_U; } if(defined $::opt_i and $::opt_i) { $Global::replace{'{}'} = $::opt_i; } if(defined $::opt_basenamereplace) { $Global::replace{'{/}'} = $::opt_basenamereplace; } - if(defined $::opt_basenameextensionreplace) { $Global::replace{'{/.}'} = $::opt_basenameextensionreplace; } + if(defined $::opt_basenameextensionreplace) { + $Global::replace{'{/.}'} = $::opt_basenameextensionreplace; + } if(defined $::opt_E and $::opt_E) { $Global::end_of_file_string = $::opt_E; } if(defined $::opt_n and $::opt_n) { $Global::max_number_of_args = $::opt_n; } if(defined $::opt_N and $::opt_N) { $Global::max_number_of_args = $::opt_N; } @@ -245,8 +248,12 @@ sub parse_options { if(defined $::opt_arg_sep) { $Global::arg_sep = $::opt_arg_sep; } if(defined $::opt_arg_file_sep) { $Global::arg_file_sep = $::opt_arg_file_sep; } if(defined $::opt_number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); } - if(defined $::opt_number_of_cores) { print SSHLogin::no_of_cores(),"\n"; wait_and_exit(0); } - if(defined $::opt_max_line_length_allowed) { print CommandMaxLength::real_max_length(),"\n"; wait_and_exit(0); } + if(defined $::opt_number_of_cores) { + print SSHLogin::no_of_cores(),"\n"; wait_and_exit(0); + } + if(defined $::opt_max_line_length_allowed) { + print Limits::Command::real_max_length(),"\n"; wait_and_exit(0); + } if(defined $::opt_version) { version(); wait_and_exit(0); } if(defined $::opt_show_limits) { show_limits(); } if(defined @::opt_sshlogin) { @Global::sshlogin = @::opt_sshlogin; } @@ -258,6 +265,16 @@ sub parse_options { if(defined $::opt_fg) { $Global::semaphore = 1; } if(defined $::opt_bg) { $Global::semaphore = 1; } if(defined $::opt_wait) { $Global::semaphore = 1; } + if(defined $::opt_tty) { + # Defaults for --tty: -j1 -u + # Can be overridden with -jXXX -g + if(not defined $::opt_P) { + $::opt_P = 1; + } + if(not defined $::opt_g) { + $Global::grouped = 0; + } + } if(defined @::opt_trc) { push @Global::ret_files, @::opt_trc; $::opt_transfer = 1; @@ -625,7 +642,7 @@ sub init_run_jobs { } } -sub login_and_host { +sub __login_and_host { # Returns: # login@hostname my $sshlogin = shift; @@ -757,9 +774,9 @@ sub progress { $status = $eta . join(" ",map { sprintf("%s:%d/%d/%d%%", - $sshlogin{$_}, + $sshlogin{$_}, $Global::host{$_}->jobs_running(), - ($Global::host{$_}->jobs_completed()||0), + ($Global::host{$_}->jobs_completed()||0), ($Global::host{$_}->jobs_running()+ ($Global::host{$_}->jobs_completed()||0))*100 / $Global::total_started) } @@ -771,9 +788,9 @@ sub progress { $status = $eta . join(" ",map { sprintf("%s:%d/%d/%d%%", - $workerno{$_}, + $workerno{$_}, $Global::host{$_}->jobs_running(), - ($Global::host{$_}->jobs_completed()||0), + ($Global::host{$_}->jobs_completed()||0), ($Global::host{$_}->jobs_running()+ ($Global::host{$_}->jobs_completed()||0))*100 / $Global::total_started) } @@ -933,7 +950,7 @@ sub get_job_with_sshlogin { if($::oodebug and $Global::JobQueue->empty()) { Carp::confess("get_job_with_sshlogin should never be called if empty"); } - + my ($job) = $Global::JobQueue->get(); if(not defined $job) { # No more jobs @@ -1089,7 +1106,7 @@ sub cleanup_basefile { } sub __SIGNAL_HANDLING__ {} - + sub list_running_jobs { # Returns: N/A for my $v (values %Global::running) { @@ -1369,7 +1386,7 @@ sub string { sub jobs_running { my $self = shift; - + return ($self->{'jobs_running'} || "0"); } @@ -1383,10 +1400,10 @@ sub dec_jobs_running { $self->{'jobs_running'}--; } -sub set_jobs_running { - my $self = shift; - $self->{'jobs_running'} = 0; -} +#sub set_jobs_running { +# my $self = shift; +# $self->{'jobs_running'} = shift; +#} sub set_maxlength { my $self = shift; @@ -1786,7 +1803,7 @@ sub ncpus { if($ncpu =~ /^[0-9]+$/) { $self->{'ncpus'} = $ncpu; } else { - print $Global::original_stderr + print $Global::original_stderr ("Warning: Could not figure out ", "number of cpus on $serverlogin. Using 1\n"); $self->{'ncpus'} = 1; @@ -1794,7 +1811,7 @@ sub ncpus { } } return $self->{'ncpus'}; -} +} sub no_of_cpus { # Returns: @@ -2045,7 +2062,7 @@ sub get { return undef; } } -} +} sub unget { my $self = shift; @@ -2083,7 +2100,7 @@ sub quote_args { my $self = shift; return $self->{'commandlinequeue'}->quote_args(); } - + package Job; @@ -2098,8 +2115,8 @@ sub new { 'stderr' => undef, 'pid' => undef, # hash of { SSHLogins => number of times the command failed there } - 'failed' => undef, - 'sshlogin' => undef, + 'failed' => undef, + 'sshlogin' => undef, # The commandline wrapped with rsync and ssh 'sshlogin_wrap' => undef, 'exitstatus' => undef, @@ -2378,8 +2395,8 @@ sub sshcleanup { join(" ",map { ::shell_quote_scalar($workdir."/".$_) } @subworkdirs); } my $relpath = ($file !~ m:^/:); # Is the path relative? - my $cleandir = ($relpath ? $workdir : ""); - $cleancmd .= "$sshcmd $serverlogin rm -f ".::shell_quote_scalar($file.$removeworkdir).";"; + my $cleandir = ($relpath ? $workdir."/" : ""); + $cleancmd .= "$sshcmd $serverlogin rm -f ".::shell_quote_scalar($cleandir.$file.$removeworkdir).";"; } return $cleancmd; } @@ -2499,7 +2516,7 @@ sub start { # Re-open to avoid complaining open STDIN, "<&", $Global::original_stdin or die "Can't dup \$Global::original_stdin: $!"; - } elsif (not $Global::tty_taken and -c "/dev/tty" and + } elsif ($::opt_tty and not $Global::tty_taken and -c "/dev/tty" and open(DEVTTY, "/dev/tty")) { # Give /dev/tty to the command if no one else is using it $pid = ::open3("<&DEVTTY", ">&STDOUT", ">&STDERR", $command) || @@ -2712,7 +2729,7 @@ sub push { push @{$self->{'arg_list'}}, $record; #::my_dump($record); my $arg_no = ($self->number_of_args()-1) * ($#$record+1); - + for my $arg (@$record) { $arg_no++; if(defined $arg) { @@ -2770,7 +2787,7 @@ sub len { # Add the length for each time for my $replstring (keys %{$self->{'replacecount'}}) { if(defined $self->{'len'}{$replstring}) { - $len += $self->{'len'}{$replstring} * + $len += $self->{'len'}{$replstring} * $self->{'replacecount'}{$replstring}; } if($Global::replace{$replstring}) { @@ -2791,7 +2808,7 @@ sub len { sub multi_regexp { if(not $CommandLine::multi_regexp) { - $CommandLine::multi_regexp = + $CommandLine::multi_regexp = "(?:". join("|",map {my $a=$_; $a =~ s/(\W)/\\$1/g; $a} ($Global::replace{"{}"}, @@ -2812,7 +2829,7 @@ sub number_of_replacements { my $sum = 0; my $cmd = $command; my $multi_regexp = multi_regexp(); - my $replacement_regexp = + my $replacement_regexp = "(?:". '\{\d+/?\.?\}'. # {n}, {n.} {n/.} {n/} '|'. @@ -2827,7 +2844,7 @@ sub number_of_replacements { } $sum++; } - + my $number_of_context_groups = 0; my $no_args; my $context; @@ -2911,11 +2928,11 @@ sub replace_placeholders { CORE::push @used_multi, $replacementfunction; if($self->{'context_replace'}) { for my $n (0 .. $#args) { - $replace_context[$n]{$replacementfunction} = + $replace_context[$n]{$replacementfunction} = $args[$n]->replace($replacementfunction); } } else { - CORE::push(@{$replace_multi{$replacementfunction}}, + CORE::push(@{$replace_multi{$replacementfunction}}, map { $args[$_]->replace($replacementfunction) } 0 .. $#args); } @@ -3238,7 +3255,6 @@ sub get { } else { push @out_record, Arg->new(""); } - } return \@out_record; } else { @@ -3348,7 +3364,7 @@ sub unget { sub empty { my $self = shift; - my $empty = (not @Global::unget_argv + my $empty = (not @Global::unget_argv and not @{$self->{'unget'}}); for my $fh (@{$self->{'fhs'}}) { $empty &&= eof($fh); @@ -3498,7 +3514,7 @@ sub release { if($self->nlinks() == 1) { unlink $self->{'idfile'}; rmdir $self->{'lockdir'}; - } + } $self->unlock(); } ::debug("released $self->{'pid'}\n"); diff --git a/src/parallel.pod b/src/parallel.pod index 1f46c14c..f344b792 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -246,18 +246,18 @@ Implies B<--semaphore>. =item B<--cleanup> Remove transferred files. B<--cleanup> will remove the transferred files -on the remote server after processing is done. +on the remote computer after processing is done. find log -name '*gz' | parallel \ --sshlogin server.example.com --transfer --return {.}.bz2 \ --cleanup "zcat {} | bzip -9 >{.}.bz2" -With B<--transfer> the file transferred to the remote server will be -removed on the remote server. Directories created will not be removed +With B<--transfer> the file transferred to the remote computer will be +removed on the remote computer. Directories created will not be removed - even if they are empty. -With B<--return> the file transferred from the remote server will be -removed on the remote server. Directories created will not be removed +With B<--return> the file transferred from the remote computer will be +removed on the remote computer. Directories created will not be removed - even if they are empty. B<--cleanup> is ignored when not used with B<--transfer> or B<--return>. @@ -493,12 +493,12 @@ Implies B<-X> unless B<-m> is set. =item B<--load> I (experimental) -Do not start new jobs on a given machine unless the load is less than +Do not start new jobs on a given computer unless the load is less than I. I uses the same syntax as B<--jobs>, so I<100%> is a valid setting. The load average is only sampled every 10 seconds to avoid stressing -small machines. +small computers. =item B<--controlmaster> (experimental) @@ -610,8 +610,8 @@ with 'y' or 'Y'. Implies B<-t>. Use profile I for options. This is useful if you want to have multiple profiles. You could have one profile for running jobs in -parallel on the local machine and a different profile for running jobs -on remote machines. See the section PROFILE FILES for examples. +parallel on the local computer and a different profile for running jobs +on remote computers. See the section PROFILE FILES for examples. I corresponds to the file ~/.parallel/I. @@ -644,23 +644,23 @@ reason (such as network failure). =item B<--return> I -Transfer files from remote servers. B<--return> is used with -B<--sshlogin> when the arguments are files on the remote servers. When +Transfer files from remote computers. B<--return> is used with +B<--sshlogin> when the arguments are files on the remote computers. When processing is done the file I will be transferred -from the remote server using B and will be put relative to +from the remote computer using B and will be put relative to the default login dir. E.g. echo foo/bar.txt | parallel \ --sshlogin server.example.com --return {.}.out touch {.}.out -This will transfer the file I<$HOME/foo/bar.out> from the server +This will transfer the file I<$HOME/foo/bar.out> from the computer I to the file I after running B on I. echo /tmp/foo/bar.txt | parallel \ --sshlogin server.example.com --return {.}.out touch {.}.out -This will transfer the file I from the server +This will transfer the file I from the computer I to the file I after running B on I. @@ -742,9 +742,9 @@ when called with B<--shebang>). =item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> -Distribute jobs to remote servers. The jobs will be run on a list of -remote servers. GNU B will determine the number of CPU -cores on the remote servers and run the number of jobs as specified by +Distribute jobs to remote computers. The jobs will be run on a list of +remote computers. GNU B will determine the number of CPU +cores on the remote computers and run the number of jobs as specified by B<-j>. If the number I is given GNU B will use this number for number of CPU cores on the host. Normally I will not be needed. @@ -803,6 +803,17 @@ Silent. The job to be run will not be printed. This is the default. Can be reversed with B<-v>. +=item B<--tty> (beta testing) + +=item B<-T> (beta testing) + +Open terminal tty. If GNU B is used for starting an +interactive program then this option may be needed. It will start only +one job at a time (i.e. B<-j1>), not buffer the output (i.e. B<-u>), +and it will open a tty for the job. When the job is done, the next job +will get the tty. + + =item B<--tmpdir> I Directory for temporary files. GNU B normally buffers output @@ -822,22 +833,22 @@ See also B<-v> and B<-p>. =item B<--transfer> -Transfer files to remote servers. B<--transfer> is used with +Transfer files to remote computers. B<--transfer> is used with B<--sshlogin> when the arguments are files and should be transferred to -the remote servers. The files will be transferred using B and +the remote computers. The files will be transferred using B and will be put relative to the default login dir. E.g. echo foo/bar.txt | parallel \ --sshlogin server.example.com --transfer wc -This will transfer the file I to the server +This will transfer the file I to the computer I to the file I<$HOME/foo/bar.txt> before running B on I. echo /tmp/foo/bar.txt | parallel \ --sshlogin server.example.com --transfer wc -This will transfer the file I to the server +This will transfer the file I to the computer I to the file I before running B on I. @@ -929,9 +940,9 @@ Print the version GNU B and exit. =item B<-W> I (beta testing) Files transferred using B<--transfer> and B<--return> will be relative -to I on remote machines, and the command will be executed in +to I on remote computers, and the command will be executed in that dir. The special workdir B<...> will create a workdir in -B<~/.parallel/tmp/> on the remote machines and will be removed if +B<~/.parallel/tmp/> on the remote computers and will be removed if using B<--cleanup>. @@ -1323,16 +1334,16 @@ GNU B will try to determine the number of CPU cores on each of the remote computers, so B<-j+0> will run one job per CPU core - even if the remote computers do not have the same number of CPU cores. -If the number of CPU cores on the remote servers is not identified +If the number of CPU cores on the remote computers is not identified correctly the number of CPU cores can be added in front. Here the -server has 8 CPU cores. +computer has 8 CPU cores. seq 1 10 | parallel --sshlogin 8/server.example.com echo =head1 EXAMPLE: Transferring of files -To recompress gzipped files with B using a remote server run: +To recompress gzipped files with B using a remote computer run: find logs/ -name '*.gz' | \ parallel --sshlogin server.example.com \ @@ -1363,7 +1374,7 @@ computer and the files transferred from the remote computer: parallel --sshlogin server.example.com \ --transfer --return {.}.bz2 --cleanup "zcat {} | bzip2 -9 >{.}.bz2" -If you want run on several servers add the servers to I<--sshlogin> +If you want run on several computers add the computers to I<--sshlogin> either using ',' or multiple I<--sshlogin>: find logs/ -name '*.gz' | \ @@ -1513,9 +1524,9 @@ same time: You can use GNU Parallel to start interactive programs like emacs or vi: -B +B -B +B If there are more files than will fit on a single command line, the editor will be started again with the remaining files. @@ -1792,7 +1803,7 @@ Example: Profile for running a perl script before every command: Note how the $ and " need to be quoted using \. Example: Profile for running distributed jobs with B on the -remote machines: +remote computers: echo -S .. nice > ~/.parallel/dist parallel -J dist --trc {.}.bz2 bzip2 -9 ::: * @@ -2385,14 +2396,14 @@ using GNU B: ClusterSSH solves a different problem than GNU B. ClusterSSH runs the same command with the same arguments on a list of -machines - one per machine. This is typically used for administrating -several machines that are almost identical. +computers - one per computer. This is typically used for administrating +several computers that are almost identical. GNU B runs the same (or different) commands with different -arguments in parallel possibly using remote machines to help -computing. If more than one machine is listed in B<-S> GNU B may +arguments in parallel possibly using remote computers to help +computing. If more than one computer is listed in B<-S> GNU B may only use one of these (e.g. if there are 8 jobs to be run and one -machine has 8 cores). +computer has 8 cores). GNU B can be used as a poor-man's version of ClusterSSH: diff --git a/src/sem.pod b/src/sem.pod index 546525b8..2f1048ee 100755 --- a/src/sem.pod +++ b/src/sem.pod @@ -81,6 +81,17 @@ available. done sem --wait +=head1 EXAMPLE: Protecting pod2html from itself + +pod2html creates two files: pod2htmd.tmp and pod2htmi.tmp which it +does not clean up. It uses these two files for a short time. But if +you run multiple pod2html in parallel (e.g. in a Makefile with make +-j) you need to protect pod2html from running twice at the same +time. B running as a mutex will do just that: + + sem --fg --id pod2html pod2html foo.pod > foo.html + sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp + =head1 BUGS diff --git a/src/sql b/src/sql index 68956b5c..dde7ecf6 100755 --- a/src/sql +++ b/src/sql @@ -528,7 +528,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20101206; + $Global::version = 20101220; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/test15.sh b/testsuite/tests-to-run/test15.sh index acd5b174..93ca90cb 100755 --- a/testsuite/tests-to-run/test15.sh +++ b/testsuite/tests-to-run/test15.sh @@ -91,7 +91,7 @@ echo 3 | xargs -P 1 -n 1 -a files cat - echo 'parallel Expect: 3 1 via psedotty 2' cat >/tmp/parallel-script-for-script </tmp/parallel-script-for-script2 </tmp/parallel-script-for-script </tmp/parallel-script-for-script2 </tmp/test17 +echo '# Create some weirdly files in /tmp' +mkdir -p /tmp/parallel.file +cat /tmp/test17 | parallel -k /bin/echo file{} '>'/tmp/parallel.file{}.file +cat /tmp/test17 | parallel -k /bin/echo /tmp/parallel.file{}.file >/tmp/test17abs +cat /tmp/test17 | parallel -k /bin/echo tmp/parallel.file{}.file >/tmp/test17rel + +echo '### --transfer - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' +cat /tmp/test17abs | parallel -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {} +# One of these should give the empty dir /tmp/parallel.file +echo good if no file +stdout ssh $SERVER1 ls '/tmp/parallel.file*' +# The other: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' + +echo '### --transfer - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' +cd / +cat /tmp/test17rel | parallel -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {} +# Should give: No such file or directory +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' + +echo '### --transfer --cleanup - abspath' +cat /tmp/test17abs | parallel -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} +echo good if no file +# Should give: No such file or directory +stdout ssh $SERVER1 ls '/tmp/parallel.file*' +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' + +echo '### --transfer --cleanup - relpath' +cat /tmp/test17rel | parallel -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} +# Should give: No such file or directory +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --return - abspath' +rm -rf /tmp/parallel.file*out +cat /tmp/test17abs | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 echo {} ">"{.}.out +ls /tmp/parallel.file*out + +echo '### --return - relpath' +rm -rf /tmp/parallel.file*out +cat /tmp/test17rel | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out +ls tmp/parallel.file*out + +echo '### --return - multiple files' +rm -rf tmp/parallel.file*out tmp/parallel.file*done +cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' +ls tmp/parallel.file*out tmp/parallel.file*done + +echo '### --return --cleanup - abspath' +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done +cat /tmp/test17abs | parallel -k --return {.}.out --return {}.done --cleanup \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK + +echo '### --return --cleanup - relpath' +rm -rf tmp/parallel.file*out tmp/parallel.file*done +cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' +ls tmp/parallel.file*out tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --return --cleanup - multiple returns' +rm -rf tmp/parallel.file*out tmp/parallel.file*done +cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --transfer --return --cleanup - abspath' +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done +cat /tmp/test17abs | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ + --sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK + + +echo '### --transfer --return --cleanup - relpath' +rm -rf tmp/parallel.file*out tmp/parallel.file*done +cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --transfer --return --cleanup - multiple files' +rm -rf tmp/parallel.file*out tmp/parallel.file*done +cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --trc - abspath' +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done +cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK + +echo '### --trc - relpath' +rm -rf tmp/parallel.file*out tmp/parallel.file*done +cat /tmp/test17rel | parallel -k --trc {.}.out --trc {}.done \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' +ls tmp/parallel.file*out tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --trc - multiple files' +rm -rf /tmp/parallel.file*out /tmp/parallel.file*done +cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' +ls /tmp/parallel.file*out /tmp/parallel.file*done +echo good if no file +stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK + +echo '### --transfer --cleanup - multiple argument files' +parallel -kv --transfer --cleanup -Sparallel@$SERVER2 cat {2} {1} :::: /tmp/test17rel <(sort -r /tmp/test17abs) +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK + diff --git a/testsuite/wanted-results/test19 b/testsuite/wanted-results/test19 index da414b7b..d41fd7aa 100644 --- a/testsuite/wanted-results/test19 +++ b/testsuite/wanted-results/test19 @@ -85,7 +85,7 @@ parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./ 'newline2.out parallel-server3 rsync --server --sender -lDrRze.iLsf --remove-source-files . ././tmp/parallel.file.' 'newline2.out2 -parallel-server3 rm -f tmp/parallel.file.' +parallel-server3 rm -f ./tmp/parallel.file.' 'newline2; rmdir 2>/dev/null ./tmp -l parallel parallel-server2 rsync --server -lDErRze.iLsf . . parallel@parallel-server2 PARALLEL_SEQ=1;export PARALLEL_SEQ;PARALLEL_PID=00000;export PARALLEL_PID; cat tmp/parallel.file.' @@ -97,7 +97,7 @@ parallel@parallel-server2 PARALLEL_SEQ=1;export PARALLEL_SEQ;PARALLEL_PID=00000; 'newline1.out -l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ././tmp/parallel.file.' 'newline1.out2 -parallel@parallel-server2 rm -f tmp/parallel.file.' +parallel@parallel-server2 rm -f ./tmp/parallel.file.' 'newline1; rmdir 2>/dev/null ./tmp ### Test use special ssh with > 9 simultaneous 1 diff --git a/testsuite/wanted-results/test47 b/testsuite/wanted-results/test47 new file mode 100644 index 00000000..8989c537 --- /dev/null +++ b/testsuite/wanted-results/test47 @@ -0,0 +1,127 @@ +### Test --transfer --return --cleanup +# Create some weirdly files in /tmp +### --transfer - abspath +file1 +file2 +good if no file +ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access /tmp/parallel.file*: No such file or directory +### --transfer - relpath +file1 +file2 +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory +### --transfer --cleanup - abspath +file1 +file2 +good if no file +ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access /tmp/parallel.file*: No such file or directory +### --transfer --cleanup - relpath +file1 +file2 +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --return - abspath +/tmp/parallel.file1.out +/tmp/parallel.file2.out +### --return - relpath +tmp/parallel.file1.out +tmp/parallel.file2.out +### --return - multiple files +tmp/parallel.file1.file.done +tmp/parallel.file1.out +tmp/parallel.file2.file.done +tmp/parallel.file2.out +### --return --cleanup - abspath +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +good if no file +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +### --return --cleanup - relpath +tmp/parallel.file1.file.done +tmp/parallel.file1.out +tmp/parallel.file2.file.done +tmp/parallel.file2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --return --cleanup - multiple returns +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --transfer --return --cleanup - abspath +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +good if no file +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +### --transfer --return --cleanup - relpath +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --transfer --return --cleanup - multiple files +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --trc - abspath +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +good if no file +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +### --trc - relpath +tmp/parallel.file1.file.done +tmp/parallel.file1.out +tmp/parallel.file2.file.done +tmp/parallel.file2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --trc - multiple files +/tmp/parallel.file1.file.done +/tmp/parallel.file1.out +/tmp/parallel.file2.file.done +/tmp/parallel.file2.out +good if no file +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +ls: cannot access /tmp/parallel.file*: No such file or directory +OK +### --transfer --cleanup - multiple argument files +cat /tmp/parallel.file2.file tmp/parallel.file1.file +file2 +file1 +cat /tmp/parallel.file1.file tmp/parallel.file2.file +file1 +file2 +ls: cannot access /tmp/parallel.file*: No such file or directory +OK