From 96ba577d8561de5183b20e7b1c9f05c05f88d620 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 15 Aug 2013 19:38:39 +0200 Subject: [PATCH] src/parallel: Fixed bug #39787: --xargs broken; incl test. --delay fixed; incl test. --- src/parallel | 35 +++++++++++----------- src/parallel.pod | 4 +-- src/parallel.texi | 4 +-- testsuite/tests-to-run/parallel-local23.sh | 11 +++++-- testsuite/tests-to-run/parallel-local6.sh | 2 +- testsuite/wanted-results/parallel-local23 | 6 ++++ 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/parallel b/src/parallel index 9d9b8d46..839f85a5 100755 --- a/src/parallel +++ b/src/parallel @@ -51,7 +51,7 @@ parse_options(); my $number_of_args; if($Global::max_number_of_args) { $number_of_args=$Global::max_number_of_args; -} elsif ($opt::X or $opt::m) { +} elsif ($opt::X or $opt::m or $opt::xargs) { $number_of_args = undef; } else { $number_of_args = 1; @@ -728,7 +728,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20130814; + $Global::version = 20130815; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -990,12 +990,13 @@ sub env_quote { } sub record_env { - # Record current %ENV-keys in ~/.parallel/recorded_env + # Record current %ENV-keys in ~/.parallel/ignored_vars # Returns: N/A - if(open(my $vars_fh, ">", $ENV{'HOME'} . "/.parallel/recorded_env")) { + my $ignore_filename = $ENV{'HOME'} . "/.parallel/ignored_vars"; + if(open(my $vars_fh, ">", $ignore_filename)) { print $vars_fh map { $_,"\n" } keys %ENV; } else { - ::error("Cannot write to ".$ENV{'HOME'} . "/.parallel/recorded_env\n"); + ::error("Cannot write to $ignore_filename\n"); ::wait_and_exit(255); } } @@ -1011,7 +1012,7 @@ sub parse_env_var { } if(grep { /^_$/ } @vars) { # Include all vars that are not in a clean environment - if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/recorded_env")) { + if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) { my @ignore = <$vars_fh>; chomp @ignore; my %ignore; @@ -4394,10 +4395,7 @@ sub start { $Global::timeoutq->insert($job); } if($opt::delay) { - my $now = ::now(); - $Global::last_job_started_at ||= $now; - ::usleep(($opt::delay - ($now - $Global::last_job_started_at))*1000); - $Global::last_job_started_at = $now; + $Global::JobQueue->empty() or ::usleep($opt::delay*1000); } return $job; } else { @@ -4726,6 +4724,9 @@ sub populate { } $self->push($next_arg); if($self->len() >= Limits::Command::max_length()) { + # Command length is now > max_length + # If there are arguments: remove the last + # If there are no arguments: Error # TODO stuff about -x opt_x if($self->number_of_args() > 1) { # There is something to work on @@ -4733,13 +4734,13 @@ sub populate { last; } else { my $args = join(" ", map { $_->orig() } @$next_arg); - print STDERR ("$Global::progname: Command line too ", - "long (", $self->len(), " >= ", - Limits::Command::max_length(), - ") at number ", - $self->{'arg_queue'}->arg_number(), - ": ". - (substr($args,0,50))."...\n"); + ::error("Command line too long (", + $self->len(), " >= ", + Limits::Command::max_length(), + ") at number ", + $self->{'arg_queue'}->arg_number(), + ": ". + (substr($args,0,50))."...\n"); $self->{'arg_queue'}->unget($self->pop()); ::wait_and_exit(255); } diff --git a/src/parallel.pod b/src/parallel.pod index 754a85a5..52f6d509 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -468,7 +468,7 @@ In Bash I can also be a Bash function - just remember to B the function. The variable '_' is special. It will copy all enviroment variables -except for the ones mentioned in ~/.parallel/recorded_env. +except for the ones mentioned in ~/.parallel/ignored_vars. See also: B<--record-env>. @@ -957,7 +957,7 @@ If used with B<--pipe> this is slow. =item B<--record-env> (alpha testing) -Record current environment variables in ~/.parallel/recorded_env. This +Record current environment variables in ~/.parallel/ignored_vars. This is useful before using B<--env _>. See also B<--env>. diff --git a/src/parallel.texi b/src/parallel.texi index f538a6f8..b9345e92 100644 --- a/src/parallel.texi +++ b/src/parallel.texi @@ -496,7 +496,7 @@ In Bash @emph{var} can also be a Bash function - just remember to @strong{export -f} the function. The variable '_' is special. It will copy all enviroment variables -except for the ones mentioned in ~/.parallel/recorded_env. +except for the ones mentioned in ~/.parallel/ignored_vars. See also: @strong{--record-env}. @@ -1020,7 +1020,7 @@ If used with @strong{--pipe} this is slow. @item @strong{--record-env} (alpha testing) @anchor{@strong{--record-env} (alpha testing)} -Record current environment variables in ~/.parallel/recorded_env. This +Record current environment variables in ~/.parallel/ignored_vars. This is useful before using @strong{--env _}. See also @strong{--env}. diff --git a/testsuite/tests-to-run/parallel-local23.sh b/testsuite/tests-to-run/parallel-local23.sh index f6e68719..2d9f24d7 100755 --- a/testsuite/tests-to-run/parallel-local23.sh +++ b/testsuite/tests-to-run/parallel-local23.sh @@ -6,7 +6,7 @@ cp -a input-files/testdir2 tmp cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1 echo '### Test filenames containing UTF-8'; cd tmp; - find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; + find . -name '*.jpg' | nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; find |grep -v CVS | sort; echo '### bug #39554: Feature request: line buffered output'; @@ -18,11 +18,18 @@ echo '### bug #39554: Feature request: line buffered output --tag'; echo echo '### test round-robin'; - seq 1000 | parallel --block 1k --pipe --round-robin wc | sort + nice seq 1000 | parallel --block 1k --pipe --round-robin wc | sort echo '### --version must have higher priority than retired options' parallel --version -g -Y -U -W -T | tail +echo '### bug #39787: --xargs broken' + perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo | perl -ne 'print length $_,"\n"' + +echo '### --delay should grow by 2 sec per arg' +stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 | perl -ne '$_ >= 2 and $_ <= 4 and print "OK\n"' +stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 3 | perl -ne '$_ >= 4 and $_ <= 6 and print "OK\n"' + EOF rm -rf tmp diff --git a/testsuite/tests-to-run/parallel-local6.sh b/testsuite/tests-to-run/parallel-local6.sh index a194c323..55e1c857 100755 --- a/testsuite/tests-to-run/parallel-local6.sh +++ b/testsuite/tests-to-run/parallel-local6.sh @@ -5,7 +5,7 @@ parallel --joblog /tmp/parallel_joblog_exitval 'sleep {} && echo sleep was not k parallel --joblog /tmp/parallel_joblog_signal 'sleep {}' ::: 100 2>/dev/null & sleep 1 killall -6 sleep -sleep 1 +sleep 1.5 grep -q 134 /tmp/parallel_joblog_exitval && echo exitval OK grep -q '[^0-9]6[^0-9]' /tmp/parallel_joblog_signal && echo signal OK diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 9dc029d8..f8d41e44 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -42,3 +42,9 @@ When using GNU Parallel for a publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. +### bug #39787: --xargs broken +131064 +37830 +### --delay should grow by 2 sec per arg +OK +OK