diff --git a/doc/release_new_version b/doc/release_new_version index c876b8af..2a464f8d 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -196,9 +196,9 @@ file:///home/tange/privat/parallel/doc/release_new_version from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org -Subject: GNU Parallel 20170422 ('Санкт-Петербу́рг') released <<[stable]>> +Subject: GNU Parallel 20170522 ('') released <<[stable]>> -GNU Parallel 20170422 ('Санкт-Петербу́рг') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20170522 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -214,22 +214,26 @@ New in this release: http://meta.askubuntu.com/a/16750/22307 http://meta.serverfault.com/a/9040/45704 -http://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-017-1593-0 +http://pubs.rsc.org/-/content/articlelanding/2017/cp/c7cp01662j#!divAbstract -https://128.84.21.199/pdf/1703.09026.pdf +ftp://frapftp.fire.ca.gov/outgoing/transfer/PUC_Map1/Utility%20Fire%20map%201%20final%20review%20and%20development%20report21616.pdf -https://wiki.ncsa.illinois.edu/display/ROGER/Using+GNU+Parallel +https://arxiv.org/pdf/1704.02257.pdf -https://github.com/lucascbeyeler/zmbackup -http://helpful.knobs-dials.com/index.php/Find_and_xargs_and_parallel#Parallel +http://www.sciencedirect.com/science/article/pii/S0044848616312194 -https://github.com/mlangill/microbiome_helper/wiki/Quick-Introduction-to-GNU-Parallel +https://www.aclweb.org/anthology/E/E17/E17-1050.pdf -http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0174575 +http://www.atmos-meas-tech.net/10/1425/2017/amt-10-1425-2017.pdf -https://www.slideshare.net/sharatsc/data-science-at-the-command-line +http://amslaurea.unibo.it/12261/1/nurrito_eugenio_tesi_fisica_magistrale.pdf -http://www.jianshu.com/p/67b0665490ac +http://biorxiv.org/content/biorxiv/early/2017/04/27/131680.full.pdf + +http://www.nature.com/nprot/journal/v12/n5/full/nprot.2017.022.html?WT.feed_name=subjects_biotechnology + + +* <> * <> diff --git a/src/Makefile.am b/src/Makefile.am index 6c8bb90a..5f5a153c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,9 +23,6 @@ doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \ parcat.pdf parset.pdf endif -parset.pod: parset - cp parset parset.pod - # Build documentation file if the tool to build exists. # Otherwise: Use the distributed version parallel.1: parallel.pod @@ -77,13 +74,13 @@ niceload.1: niceload.pod && mv $(srcdir)/niceload.1n $(srcdir)/niceload.1 \ || echo "Warning: pod2man not found. Using old niceload.1" -parcat.1: parcat +parcat.1: parcat.pod pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/parcat > $(srcdir)/parcat.1n \ && mv $(srcdir)/parcat.1n $(srcdir)/parcat.1 \ || echo "Warning: pod2man not found. Using old parcat.1" -parset.1: parset +parset.1: parset.pod pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/parset > $(srcdir)/parset.1n \ && mv $(srcdir)/parset.1n $(srcdir)/parset.1 \ @@ -194,7 +191,7 @@ parcat.texi: parcat pod2texi --output=$(srcdir)/parcat.texi $(srcdir)/parcat \ || echo "Warning: pod2texi not found. Using old parcat.texi" -parset.texi: parset +parset.texi: parset.pod pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset \ || echo "Warning: pod2texi not found. Using old parset.texi" @@ -234,7 +231,7 @@ parcat.pdf: parcat pod2pdf --output-file $(srcdir)/parcat.pdf $(srcdir)/parcat --title "GNU parcat" \ || echo "Warning: pod2pdf not found. Using old parcat.pdf" -parset.pdf: parset +parset.pdf: parset.pod pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset --title "GNU parset" \ || echo "Warning: pod2pdf not found. Using old parset.pdf" @@ -258,6 +255,7 @@ EXTRA_DIST = parallel sem sql niceload parcat parset env_parallel \ env_parallel.ash env_parallel.bash env_parallel.csh \ env_parallel.dash env_parallel.fish env_parallel.ksh \ env_parallel.pdksh env_parallel.sh env_parallel.tcsh \ - env_parallel.zsh sem.pod parallel.pod env_parallel.pod \ - niceload.pod parallel_tutorial.pod parallel_design.pod \ - parallel_alternatives.pod $(DISTCLEANFILES) + env_parallel.zsh parcat.pod parset.pod sem.pod parallel.pod \ + env_parallel.pod niceload.pod parallel_tutorial.pod \ + parallel_design.pod parallel_alternatives.pod \ + $(DISTCLEANFILES) diff --git a/src/Makefile.in b/src/Makefile.in index bc2479aa..53a5c01b 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -266,9 +266,10 @@ EXTRA_DIST = parallel sem sql niceload parcat parset env_parallel \ env_parallel.ash env_parallel.bash env_parallel.csh \ env_parallel.dash env_parallel.fish env_parallel.ksh \ env_parallel.pdksh env_parallel.sh env_parallel.tcsh \ - env_parallel.zsh sem.pod parallel.pod env_parallel.pod \ - niceload.pod parallel_tutorial.pod parallel_design.pod \ - parallel_alternatives.pod $(DISTCLEANFILES) + env_parallel.zsh parcat.pod parset.pod sem.pod parallel.pod \ + env_parallel.pod niceload.pod parallel_tutorial.pod \ + parallel_design.pod parallel_alternatives.pod \ + $(DISTCLEANFILES) all: all-am @@ -611,9 +612,6 @@ install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true $(LN_S) parallel $(DESTDIR)$(bindir)/sem -parset.pod: parset - cp parset parset.pod - # Build documentation file if the tool to build exists. # Otherwise: Use the distributed version parallel.1: parallel.pod @@ -664,13 +662,13 @@ niceload.1: niceload.pod && mv $(srcdir)/niceload.1n $(srcdir)/niceload.1 \ || echo "Warning: pod2man not found. Using old niceload.1" -parcat.1: parcat +parcat.1: parcat.pod pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/parcat > $(srcdir)/parcat.1n \ && mv $(srcdir)/parcat.1n $(srcdir)/parcat.1 \ || echo "Warning: pod2man not found. Using old parcat.1" -parset.1: parset +parset.1: parset.pod pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/parset > $(srcdir)/parset.1n \ && mv $(srcdir)/parset.1n $(srcdir)/parset.1 \ @@ -781,7 +779,7 @@ parcat.texi: parcat pod2texi --output=$(srcdir)/parcat.texi $(srcdir)/parcat \ || echo "Warning: pod2texi not found. Using old parcat.texi" -parset.texi: parset +parset.texi: parset.pod pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset \ || echo "Warning: pod2texi not found. Using old parset.texi" @@ -821,7 +819,7 @@ parcat.pdf: parcat pod2pdf --output-file $(srcdir)/parcat.pdf $(srcdir)/parcat --title "GNU parcat" \ || echo "Warning: pod2pdf not found. Using old parcat.pdf" -parset.pdf: parset +parset.pdf: parset.pod pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset --title "GNU parset" \ || echo "Warning: pod2pdf not found. Using old parset.pdf" diff --git a/src/niceload b/src/niceload index dade7ad3..67534fbf 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20170422; +$Global::version = 20170423; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index f1a63f27..3af9a76d 100755 --- a/src/parallel +++ b/src/parallel @@ -266,7 +266,7 @@ sub pipe_tee_setup { } # cat foo | tee t1 t2 t3 t4 t5 > /dev/null if(not fork()){ - # Let tee inheirit our stdin + # Let tee inherit our stdin # and redirect stdout to null open STDOUT, ">","/dev/null"; exec "tee",@fifos; @@ -300,7 +300,8 @@ sub pipe_part_files { # Make @cat_prepends my @cat_prepends = (); for(my $i=0; $i<$#pos; $i++) { - push @cat_prepends, cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1]); + push(@cat_prepends, + cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1])); } return @cat_prepends; } @@ -487,7 +488,8 @@ sub spreadstdin { if($Global::max_number_of_args) { # -N => (start..*?end){n} # -L -N => (start..*?end){n*l} - my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1); + my $read_n_lines = + $Global::max_number_of_args * ($Global::max_lines || 1); while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) { # Copy to modifiable variable my $b = $1; @@ -510,7 +512,8 @@ sub spreadstdin { if($Global::max_number_of_args) { # -N => (start..*?end){n} my $i = 0; - my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1); + my $read_n_lines = + $Global::max_number_of_args * ($Global::max_lines || 1); while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1) { $i += length $recend; # find the actual splitting location $anything_written += @@ -547,7 +550,8 @@ sub spreadstdin { ::debug("init", "Done reading input\n"); # If there is anything left in the buffer write it - write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf); + write_record_to_pipe($chunk_number++, \$header, \$buf, $recstart, + $recend, length $buf); if($opt::retries) { $Global::no_more_input = 1; @@ -1099,7 +1103,8 @@ sub parse_options { if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2 $Global::debug = $opt::D; - $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh"; + $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) + || $ENV{'SHELL'} || "/bin/sh"; $Global::cshell = $Global::shell =~ m:/csh:; if(defined $opt::X) { $Global::ContextReplace = 1; } if(defined $opt::silent) { $Global::verbose = 0; } @@ -1114,7 +1119,9 @@ sub parse_options { if(defined $opt::verbose) { $Global::stderr_verbose = 1; } parse_replacement_string_options(); if(defined $opt::eof) { $Global::end_of_file_string = $opt::eof; } - if(defined $opt::max_args) { $Global::max_number_of_args = $opt::max_args; } + if(defined $opt::max_args) { + $Global::max_number_of_args = $opt::max_args; + } if(defined $opt::timeout) { $Global::timeoutq = TimeoutQueue->new($opt::timeout); } @@ -1363,7 +1370,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170422; + $Global::version = 20170423; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1752,7 +1759,7 @@ sub open_csv { ::die_bug("Can't dup STDOUT in csv: $!"); # Do not print any other output to STDOUT # by forcing all other output to /dev/null - open my $fd, ">", "/dev/null" or + open my $fd, ">", "/dev/null" or ::die_bug("Can't >/dev/null in csv: $!"); $Global::fd{1} = $fd; $Global::fd{2} = $fd; @@ -1852,7 +1859,8 @@ sub read_options { } my $script = shell_quote_scalar(shift @ARGV); # exec myself to split $ARGV[0] into separate fields - exec "$0 --internal-pipe-means-argfiles @options @parser $script ::: @ARGV"; + exec "$0 --internal-pipe-means-argfiles @options @parser $script ". + "::: @ARGV"; } } if($ARGV[0] =~ / --shebang(-?wrap)? /) { @@ -1886,7 +1894,8 @@ sub read_options { } for my $profile (@profiles) { if(-r $profile) { - open (my $in_fh, "<", $profile) || ::die_bug("read-profile: $profile"); + open (my $in_fh, "<", $profile) || + ::die_bug("read-profile: $profile"); while(<$in_fh>) { /^\s*\#/ and next; chomp; @@ -2138,11 +2147,12 @@ sub shell_quote_scalar { } sub shell_quote_file { - # Quote the string so shell will not expand any special chars and prepend ./ if needed + # Quote the string so shell will not expand any special chars + # and prepend ./ if needed # Input: # $filename = filename to be shell quoted # Returns: - # $quoted_filename = filename quoted with \ as needed by the shell and ./ if needed + # $quoted_filename = filename quoted with \ and ./ if needed my $a = shell_quote_scalar(shift); if(defined $a) { if($a =~ m:^/: or $a =~ m:^\./:) { @@ -2283,9 +2293,12 @@ sub set_fh_blocking { my $fh = shift; $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; my $flags; - fcntl($fh, &F_GETFL, $flags) || die $!; # Get the current flags on the filehandle - $flags &= ~&O_NONBLOCK; # Remove non-blocking from the flags - fcntl($fh, &F_SETFL, $flags) || die $!; # Set the flags on the filehandle + # Get the current flags on the filehandle + fcntl($fh, &F_GETFL, $flags) || die $!; + # Remove non-blocking from the flags + $flags &= ~&O_NONBLOCK; + # Set the flags on the filehandle + fcntl($fh, &F_SETFL, $flags) || die $!; } sub set_fh_non_blocking { @@ -2297,9 +2310,12 @@ sub set_fh_non_blocking { my $fh = shift; $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; my $flags; - fcntl($fh, &F_GETFL, $flags) || die $!; # Get the current flags on the filehandle - $flags |= &O_NONBLOCK; # Add non-blocking to the flags - fcntl($fh, &F_SETFL, $flags) || die $!; # Set the flags on the filehandle + # Get the current flags on the filehandle + fcntl($fh, &F_GETFL, $flags) || die $!; + # Add non-blocking to the flags + $flags |= &O_NONBLOCK; + # Set the flags on the filehandle + fcntl($fh, &F_SETFL, $flags) || die $!; } @@ -7032,7 +7048,8 @@ sub runtime { # Returns: # Run time in seconds with 3 decimals my $self = shift; - return sprintf("%.3f",int(($self->endtime() - $self->starttime())*1000)/1000); + return sprintf("%.3f", + int(($self->endtime() - $self->starttime())*1000)/1000); } sub endtime { @@ -7330,9 +7347,15 @@ sub set_sshlogin { $self->{'sshlogin'} = $sshlogin; delete $self->{'sshlogin_wrap'}; # If sshlogin is changed the wrap is wrong delete $self->{'wrapped'}; - $opt::sqlworker and - $Global::sql->update("SET Host = ? WHERE Seq = ".$self->seq(), - $sshlogin->string()); + + if($opt::sqlworker) { + # Identify worker as --sqlworker often runs on different machines + my $host = $sshlogin->string(); + if($host eq ":") { + $host = ::hostname(); + } + $Global::sql->update("SET Host = ? WHERE Seq = ".$self->seq(), $host); + } } sub sshlogin { @@ -7536,7 +7559,8 @@ sub sshlogin_wrap { push @vars, grep { not defined $ignore{$_} } keys %ENV; @vars = grep { not /^_$/ } @vars; } else { - ::error("Run '$Global::progname --record-env' in a clean environment first."); + ::error("Run '$Global::progname --record-env' ". + "in a clean environment first."); ::wait_and_exit(255); } } @@ -7795,12 +7819,15 @@ sub sshreturn { $nobasedir =~ s:.*/\./::; $cd = ::shell_quote_file(::dirname($nobasedir)); my $rsync_cd = '--rsync-path='.::shell_quote_scalar("cd $wd$cd; rsync"); - my $basename = ::shell_quote_scalar(::shell_quote_file(::basename($file))); + my $basename = + ::shell_quote_scalar(::shell_quote_file(::basename($file))); # --return # mkdir -p /home/tange/dir/subdir/; - # rsync (--protocol 30) -rlDzR --rsync-path="cd /home/tange/dir/subdir/; rsync" - # server:file.gz /home/tange/dir/subdir/ - $pre .= "mkdir -p $basedir$cd && ".$sshlogin->rsync()." $rsync_cd $rsync_opt $serverlogin:". + # rsync (--protocol 30) -rlDzR + # --rsync-path="cd /home/tange/dir/subdir/; rsync" + # server:file.gz /home/tange/dir/subdir/ + $pre .= "mkdir -p $basedir$cd && ". $sshlogin->rsync(). + " $rsync_cd $rsync_opt $serverlogin:". $basename . " ".$basedir.$cd.";"; } return $pre; @@ -8998,7 +9025,8 @@ sub populate { if($opt::sqlmaster) { # Insert the V1..Vn for this $seq in SQL table instead of generating one - $Global::sql->insert_records($self->seq(),$self->{'command'},$self->{'arg_list_flat_orig'}); + $Global::sql->insert_records($self->seq(), $self->{'command'}, + $self->{'arg_list_flat_orig'}); } } @@ -9045,11 +9073,13 @@ sub pop { if($perlexpr =~ /^(\d+) /) { # Positional defined($record->[$1-1]) or next; - $self->{'len'}{$perlexpr} -= length $record->[$1-1]->replace($perlexpr,$quote_arg,$self); + $self->{'len'}{$perlexpr} -= + length $record->[$1-1]->replace($perlexpr,$quote_arg,$self); } else { for my $arg (@$record) { if(defined $arg) { - $self->{'len'}{$perlexpr} -= length $arg->replace($perlexpr,$quote_arg,$self); + $self->{'len'}{$perlexpr} -= + length $arg->replace($perlexpr,$quote_arg,$self); } } } @@ -9321,10 +9351,10 @@ sub replaced { my $context_replace; my $perl_expressions_as_re; my @arg; - my %words_containing_replacement_strings; + my %words_with_rpl_strings; - sub fish_out_words_containing_replacement_strings { - if(not $words_containing_replacement_strings{$context_replace,@target}) { + sub fish_out_words_with_rpl_strings { + if(not $words_with_rpl_strings{$context_replace,@target}) { my %word; for (@target) { my $tt = $_; @@ -9357,9 +9387,9 @@ sub replaced { } } } - @{$words_containing_replacement_strings{$context_replace,@target}} = keys %word + @{$words_with_rpl_strings{$context_replace,@target}} = keys %word } - return @{$words_containing_replacement_strings{$context_replace,@target}}; + return @{$words_with_rpl_strings{$context_replace,@target}}; } sub replace_placeholders { @@ -9377,7 +9407,7 @@ sub replaced { my $quote = shift; my $quote_arg = shift; my %replace; - # -X = context replace (fish_out_words_containing_replacement_strings) + # -X = context replace (fish_out_words_with_rpl_strings) $context_replace = $self->{'context_replace'}; @target = @$targetref; ::debug("replace", "Replace @target\n"); @@ -9404,7 +9434,7 @@ sub replaced { join("|", map {s/^-?\d+//; "\Q$_\E"} keys %{$self->{'replacecount'}}); # Fish out the words that have replacement strings in them for my $word ( - fish_out_words_containing_replacement_strings()) { + fish_out_words_with_rpl_strings()) { # word = AB \257< perlexpr \257> CD \257< perlexpr \257> EF ::debug("replace", "Replacing in $word\n"); my $normal_replace; @@ -9505,7 +9535,8 @@ sub new { $_ or next; # Disallow \257 to avoid nested {= {= =} =} if(/\257/) { - ::error("Command cannot contain the character \257. Use a function for that."); + ::error("Command cannot contain the character \257. ". + "Use a function for that."); ::wait_and_exit(255); } # Needs to match rightmost left parens (Perl defaults to leftmost) @@ -9525,7 +9556,8 @@ sub new { # # Replace $$1 in {= perl expr =} with groupings in short hand string # - # ppar --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;' echo {/.tar/.gz} ::: UU.tar.gz + # --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;' + # echo {/.tar/.gz} ::: UU.tar.gz my ($prefix,$grp_regexp,$postfix) = $rpl =~ /^( [^(]* ) # Prefix - e.g. {%% ( \(.*\) )? # Group capture regexp - e.g (.*) @@ -9533,7 +9565,8 @@ sub new { /x; $grp_regexp ||= ''; my $rplval = $Global::rpl{$rpl}; - while(s{( (?: ^|\257> ) [^\257]*? ) # Don't replace after \257 unless \257> + while(s{( (?: ^|\257> ) [^\257]*? ) + # Don't replace after \257 unless \257> \Q$prefix\E $grp_regexp \Q$postfix\E} { # The start remains the same @@ -9876,7 +9909,7 @@ sub real_max_length { # Find the max_length of a command line # Returns: # The maximal command line length - # Use an upper bound of 8 MB if the shell allows for for infinite long lengths + # Use an upper bound of 8 MB if the shell allows for infinite long lengths my $upper = 8_000_000; my $len = 8; do { @@ -9938,7 +9971,8 @@ sub tmux_length { my @out; for my $l (1, 2020, 16320, 100000, $len) { my $tmpfile = ::tmpname("tms"); - my $tmuxcmd = $ENV{'PARALLEL_TMUX'}." -S $tmpfile new-session -d -n echo $l". + my $tmuxcmd = $ENV{'PARALLEL_TMUX'}. + " -S $tmpfile new-session -d -n echo $l". ("x"x$l). " && echo $l; rm -f $tmpfile"; push @out, ::qqx($tmuxcmd); ::rm($tmpfile); @@ -10093,7 +10127,8 @@ sub unget { sub empty { my $self = shift; - my $empty = (not @{$self->{'unget'}} and $self->{'arg_sub_queue'}->empty()); + my $empty = (not @{$self->{'unget'}} and + $self->{'arg_sub_queue'}->empty()); ::debug("run", "RecordColQueue->empty $empty"); return $empty; } @@ -10273,7 +10308,8 @@ sub nest_get { # Is input source --link'ed to the next? $opt::linkinputsource[$fhn+1]); } - $combarg[2*$fhno] = [$len,$len]; # Find only combinations with this new entry + # Find only combinations with this new entry + $combarg[2*$fhno] = [$len,$len]; # map combinations # [ 1, 3, 7 ], [ 2, 4, 1 ] # => @@ -10695,8 +10731,12 @@ sub get_alias { "$ENV{HOME}/.dburl.aliases", "/etc/sql/aliases", "$path/dburl.aliases", "$path/dburl.aliases.dist"); for my $alias_file (@search) { + # local $/ needed if -0 set + local $/ = "\n"; if(-r $alias_file) { - push @urlalias, `cat "$alias_file"`; + open(my $in, "<", $alias_file) || die; + push @urlalias, <$in>; + close $in; } } my ($alias_part,$rest) = $alias=~/(:\w*)(.*)/; @@ -10743,7 +10783,7 @@ sub check_permissions { sub parse_dburl { my $url = shift; my %options = (); - # sql:mysql://[[user][:password]@][host][:port]/[database[/table][?sql query]] + # sql:mysql://[[user][:password]@][host][:port]/[database[/table][?query]] if($url=~m!(?:sql:)? # You can prefix with 'sql:' ((?:oracle|ora|mysql|pg|postgres|postgresql)(?:s|ssl|)| @@ -10780,12 +10820,12 @@ sub parse_dburl { $options{database} = ::undef_if_empty(uri_unescape($6)); $options{table} = ::undef_if_empty(uri_unescape($7)); $options{query} = ::undef_if_empty(uri_unescape($8)); - ::debug("sql","dburl $url\n"); - ::debug("sql","databasedriver ",$options{databasedriver}, " user ", $options{user}, - " password ", $options{password}, " host ", $options{host}, - " port ", $options{port}, " database ", $options{database}, - " table ",$options{table}," query ",$options{query}, "\n"); - + ::debug("sql", "dburl $url\n"); + ::debug("sql", "databasedriver ", $options{databasedriver}, + " user ", $options{user}, + " password ", $options{password}, " host ", $options{host}, + " port ", $options{port}, " database ", $options{database}, + " table ", $options{table}, " query ", $options{query}, "\n"); } else { ::error("$url is not a valid DBURL"); exit 255; @@ -10848,7 +10888,7 @@ sub run { last; } if($DBI::errstr =~ /locked/) { - ::debug("sql","Lock retry: $lockretry"); + ::debug("sql", "Lock retry: $lockretry"); $lockretry++; ::usleep(rand()*300); } elsif(not $sth) { @@ -10904,7 +10944,8 @@ sub output { my $self = shift; my $commandline = shift; - $self->update("SET Stdout = ?, Stderr = ? WHERE Seq = ".$commandline->seq(), + $self->update("SET Stdout = ?, Stderr = ? WHERE Seq = ". + $commandline->seq(), join("",@{$commandline->{'output'}{1}}), join("",@{$commandline->{'output'}{2}})); } @@ -11037,7 +11078,8 @@ sub finished { my $self = shift; if($opt::wait or $Global::start_sqlworker) { my $table = $self->table(); - my $rv = $self->get("select Seq,Exitval from $table where Exitval <= -1000 limit 1"); + my $rv = $self->get("select Seq,Exitval from $table ". + "where Exitval <= -1000 limit 1"); return not $rv->[0]; } else { return 1; @@ -11251,7 +11293,8 @@ sub lock { } } else { if($total_sleep/1000 > 30) { - ::warning("Semaphore stuck for 30 seconds. Consider using --semaphoretimeout."); + ::warning("Semaphore stuck for 30 seconds. ". + "Consider using --semaphoretimeout."); } } } diff --git a/src/parallel.pod b/src/parallel.pod index b05bc51e..790b566a 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -341,7 +341,7 @@ B<:::> and B<::::> can be mixed. So these are equivalent: ::: 1 2 3 -=item B<:::+> I +=item B<:::+> I (alpha testing) Like B<:::> but linked like B<--link> to the previous input source. @@ -362,7 +362,7 @@ B<:::> and B<::::> can be mixed. See B<-a>, B<:::> and B<--link>. -=item B<::::+> I +=item B<::::+> I (alpha testing) Like B<::::> but linked like B<--link> to the previous input source. @@ -418,7 +418,7 @@ string that is not in the command line. See also: B<:::>. -=item B<--bar> +=item B<--bar> (alpha testing) Show progress as a progress bar. In the bar is shown: % of jobs completed, estimated seconds left, and number of jobs started. @@ -656,7 +656,7 @@ variables, arrays, and functions) use B. See also: B<--record-env>. -=item B<--eta> +=item B<--eta> (alpha testing) Show the estimated number of seconds before finishing. This forces GNU B to read all jobs before starting to find the number of @@ -744,9 +744,9 @@ See also: B<--line-buffer> B<--ungroup> Print a summary of the options to GNU B and exit. -=item B<--halt-on-error> I +=item B<--halt-on-error> I (alpha testing) -=item B<--halt> I +=item B<--halt> I (alpha testing) When should GNU B terminate? In some situations it makes no sense to run all jobs. GNU B should simply give up as soon @@ -891,7 +891,7 @@ specified, and for B<-I {}> otherwise. This option is deprecated; use B<-I> instead. -=item B<--joblog> I +=item B<--joblog> I (alpha testing) Logfile for executed jobs. Save a list of the executed jobs to I in the following TAB separated format: sequence number, @@ -1076,7 +1076,7 @@ Arguments will be recycled if one input source has more arguments than the other See also B<--header>, B<:::+>, B<::::+>. -=item B<--load> I +=item B<--load> I (alpha testing) Do not start new jobs on a given computer unless the number of running processes on the computer is less than I. I uses @@ -1254,7 +1254,7 @@ control on the command line (used by GNU B internally when called with B<--sshlogin>). -=item B<--plus> (alpha testing) +=item B<--plus> (beta testing) Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} {...} {/..} {/...} {##}. The idea being that '{+foo}' matches the opposite of @@ -2061,6 +2061,9 @@ should be the same as given by B<--sqlmaster>. If you have more than one B<--sqlworker> jobs may be run more than once. +If B<--sqlworker> runs on the local machine, the hostname in the SQL +table will not be ':' but instead the hostname of the machine. + =item B<--ssh> I @@ -2233,7 +2236,7 @@ B<{}>. B<--tagstring> is ignored when using B<-u>, B<--onall>, and B<--nonall>. -=item B<--tee> (alpha testing) +=item B<--tee> (beta testing) Pipe all data to all jobs. Used with B<--pipe>/B<--pipepart> and B<:::>. @@ -2274,13 +2277,13 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--tmux> +=item B<--tmux> (alpha testing) Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. -=item B<--tmuxpane> +=item B<--tmuxpane> (alpha testing) Use B for output but put output into panes in the first window. Useful if you want to monitor the progress of less than 100 concurrent diff --git a/src/parcat b/src/parcat index be4d4c3e..820c17ac 100755 --- a/src/parcat +++ b/src/parcat @@ -1,156 +1,5 @@ #!/usr/bin/perl -=head1 NAME - -parcat - cat files or fifos in parallel - -=head1 SYNOPSIS - -B file(s) - -=head1 DESCRIPTION - -GNU B reads files or fifos in parallel. It writes full lines -so there will be no problem with mixed-half-lines which you risk if -you use: - - (cat file1 & cat file2 &) | ... - - -=head1 EXAMPLES - -=head2 Do be done - - mkfifo slot-{1..5}-digit-{0..9} - parallel -j5 'seq 100000 | grep {} > slot-{%}-digit-{}' ::: {0..9} & - parallel parcat slot-{1..5}-digit-{} '>' digit-{} ::: {0..9} - -=head1 REPORTING BUGS - -GNU B is part of GNU B. Report bugs to . - - -=head1 AUTHOR - -Copyright (C) 2016,2017 Ole Tange, http://ole.tange.dk and Free -Software Foundation, Inc. - -=head1 LICENSE - -Copyright (C) 2007,2008,2009,2010,2011 Free Software Foundation, Inc. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -at your option any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -=head2 Documentation license I - -Permission is granted to copy, distribute and/or modify this documentation -under the terms of the GNU Free Documentation License, Version 1.3 or -any later version published by the Free Software Foundation; with no -Invariant Sections, with no Front-Cover Texts, and with no Back-Cover -Texts. A copy of the license is included in the file fdl.txt. - -=head2 Documentation license II - -You are free: - -=over 9 - -=item B - -to copy, distribute and transmit the work - -=item B - -to adapt the work - -=back - -Under the following conditions: - -=over 9 - -=item B - -You must attribute the work in the manner specified by the author or -licensor (but not in any way that suggests that they endorse you or -your use of the work). - -=item B - -If you alter, transform, or build upon this work, you may distribute -the resulting work only under the same, similar or a compatible -license. - -=back - -With the understanding that: - -=over 9 - -=item B - -Any of the above conditions can be waived if you get permission from -the copyright holder. - -=item B - -Where the work or any of its elements is in the public domain under -applicable law, that status is in no way affected by the license. - -=item B - -In no way are any of the following rights affected by the license: - -=over 9 - -=item * - -Your fair dealing or fair use rights, or other applicable -copyright exceptions and limitations; - -=item * - -The author's moral rights; - -=item * - -Rights other persons may have either in the work itself or in -how the work is used, such as publicity or privacy rights. - -=back - -=item B - -For any reuse or distribution, you must make clear to others the -license terms of this work. - -=back - -A copy of the full license is included in the file as cc-by-sa.txt. - -=head1 DEPENDENCIES - -GNU B uses Perl. - - -=head1 SEE ALSO - -B(1), B(1) - -=cut - - use Symbol qw(gensym); use IPC::Open3; use POSIX qw(:errno_h); diff --git a/src/parcat.pod b/src/parcat.pod new file mode 100644 index 00000000..60065268 --- /dev/null +++ b/src/parcat.pod @@ -0,0 +1,151 @@ +#!/usr/bin/perl + +=head1 NAME + +parcat - cat files or fifos in parallel + +=head1 SYNOPSIS + +B file(s) + +=head1 DESCRIPTION + +GNU B reads files or fifos in parallel. It writes full lines +so there will be no problem with mixed-half-lines which you risk if +you use: + + (cat file1 & cat file2 &) | ... + + +=head1 EXAMPLES + +=head2 Do be done + + mkfifo slot-{1..5}-digit-{0..9} + parallel -j5 'seq 100000 | grep {} > slot-{%}-digit-{}' ::: {0..9} & + parallel parcat slot-{1..5}-digit-{} '>' digit-{} ::: {0..9} + +=head1 REPORTING BUGS + +GNU B is part of GNU B. Report bugs to . + + +=head1 AUTHOR + +Copyright (C) 2016,2017 Ole Tange, http://ole.tange.dk and Free +Software Foundation, Inc. + +=head1 LICENSE + +Copyright (C) 2007,2008,2009,2010,2011 Free Software Foundation, Inc. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +at your option any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +=head2 Documentation license I + +Permission is granted to copy, distribute and/or modify this documentation +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, with no Front-Cover Texts, and with no Back-Cover +Texts. A copy of the license is included in the file fdl.txt. + +=head2 Documentation license II + +You are free: + +=over 9 + +=item B + +to copy, distribute and transmit the work + +=item B + +to adapt the work + +=back + +Under the following conditions: + +=over 9 + +=item B + +You must attribute the work in the manner specified by the author or +licensor (but not in any way that suggests that they endorse you or +your use of the work). + +=item B + +If you alter, transform, or build upon this work, you may distribute +the resulting work only under the same, similar or a compatible +license. + +=back + +With the understanding that: + +=over 9 + +=item B + +Any of the above conditions can be waived if you get permission from +the copyright holder. + +=item B + +Where the work or any of its elements is in the public domain under +applicable law, that status is in no way affected by the license. + +=item B + +In no way are any of the following rights affected by the license: + +=over 9 + +=item * + +Your fair dealing or fair use rights, or other applicable +copyright exceptions and limitations; + +=item * + +The author's moral rights; + +=item * + +Rights other persons may have either in the work itself or in +how the work is used, such as publicity or privacy rights. + +=back + +=item B + +For any reuse or distribution, you must make clear to others the +license terms of this work. + +=back + +A copy of the full license is included in the file as cc-by-sa.txt. + +=head1 DEPENDENCIES + +GNU B uses Perl. + + +=head1 SEE ALSO + +B(1), B(1) + +=cut diff --git a/src/parset.pod b/src/parset.pod index 8b428c0c..cabb8ffb 100644 --- a/src/parset.pod +++ b/src/parset.pod @@ -25,6 +25,8 @@ the destination variable and made to an array. If I contains multiple names separated by ',' or space, the names will be the destination variables. +B is not production ready and alpha quality. + =head1 OPTIONS @@ -102,9 +104,6 @@ Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk Copyright (C) 2010,2011,2012,2013,2014,2015,2016,2017 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. -Parts of the manual concerning B compatibility is inspired by -the manual of B from GNU findutils 4.4.2. - =head1 LICENSE @@ -223,9 +222,7 @@ B uses GNU B. =head1 SEE ALSO -B(1), B(1), -B(1). - +B(1), B(1), B(1). =cut diff --git a/src/sql b/src/sql index c67e6cc9..e26ebed9 100755 --- a/src/sql +++ b/src/sql @@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20170422; + $Global::version = 20170423; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 457a3360..c57fd392 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -233,6 +233,14 @@ line2"' 'echo "command2"' } +par_sqlworker_hostname() { + echo 'bug #50901: --sqlworker should use hostname in the joblog instead of :' + parallel --sqlmaster :my/hostname echo ::: 1 2 3 + parallel -k --sqlworker :my/hostname + sql :my 'select host from hostname;' +} + + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 8e759e9f..d254ebdc 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -743,3 +743,11 @@ par_result_replace /tmp/par__49983-baz C par_result_replace /tmp/par__49983-baz C/seq par_result_replace /tmp/par__49983-baz C/stderr par_result_replace /tmp/par__49983-baz C/stdout +par_sqlworker_hostname bug #50901: --sqlworker should use hostname in the joblog instead of : +par_sqlworker_hostname 1 +par_sqlworker_hostname 2 +par_sqlworker_hostname 3 +par_sqlworker_hostname host +par_sqlworker_hostname aspire +par_sqlworker_hostname aspire +par_sqlworker_hostname aspire