tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: --dryrun implemented. --return of multiple files with

Browse source 
-X. --return of files containing space. Passes testsuite.
This commit is contained in:
Ole Tange 2010-12-14 09:40:42 +01:00
parent 3b3c344097
commit bff9531219
10 changed files with 116 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
doc/FUTURE_IDEAS
View file

@ -1,6 +1,15 @@
--command, -c, --file, and -f now obsoleted. --eta works again.
Bugfix in testsuite for --retries.
Lots of dead code removed.
parallel: --dryrun implemented. --return of multiple files with
-X. --return of files containing space.
== Bug ==
--trc {}' 'done
echo 1' '2 | parallel -vv --trc {}' 'done -S nlv.pi.dk touch {}' 'done
(echo 1' '2;echo a' 'b) | parallel -qvv --trc {}' 'done -S nlv.pi.dk touch {}' 'done
== Bug? ==

3
doc/release_new_version
View file

@ -134,6 +134,9 @@ download at: http://ftp.gnu.org/gnu/parallel/
New in this release:
* GNU niceload is now part of GNU Parallel. GNU niceload slows down a
program if the load average is above a certain limit.
* Implemented --tmpdir to buffer standard output and standard error in
a different place.

2
src/niceload
View file

@ -2,7 +2,7 @@
=head1 NAME
niceload - run a program when the load is below a certain limit
niceload - slow down a program when the load average is above a certain limit
=head1 SYNOPSIS

90
src/parallel
View file

@ -136,6 +136,7 @@ sub get_options_from_array {
"tempdir=s" => \$::opt_tmpdir,
"halt-on-error|H=s" => \$::opt_halt_on_error,
"retries=i" => \$::opt_retries,
"dry-run|dryrun" => \$::opt_dryrun,
"progress" => \$::opt_progress,
"eta" => \$::opt_eta,
"arg-sep|argsep=s" => \$::opt_arg_sep,
@ -2301,7 +2302,7 @@ sub return {
my $self = shift;
my @return = ();
for my $return (@{$self->{'commandline'}{'return_files'}}) {
CORE::push @return, $self->{'commandline'}->replace_placeholders($return);
CORE::push @return, $self->{'commandline'}->replace_placeholders($return,1);
}
return @return;
}
@ -2452,14 +2453,17 @@ sub start {
print $Global::original_stderr "$command\n";
}
}
if($Global::verbose and not $Global::grouped) {
if($Global::verbose == 1) {
if(($::opt_dryrun or $Global::verbose) and not $Global::grouped) {
if($Global::verbose <= 1) {
print STDOUT $job->replaced(),"\n";
} else {
# Verbose level > 1: Print the rsync and stuff
print STDOUT $command,"\n";
}
}
if($::opt_dryrun) {
$command = "true";
}
$Global::total_running++;
$Global::total_started++;
if(not $job->seq()) {
@ -2536,8 +2540,8 @@ sub print {
my $err = $self->stderr();
my ($command) = $self->sshlogin_wrap();
if($Global::verbose and $Global::grouped) {
if($Global::verbose == 1) {
if(($::opt_dryrun or $Global::verbose) and $Global::grouped) {
if($Global::verbose <= 1) {
print STDOUT $self->replaced(),"\n";
} else {
# Verbose level > 1: Print the rsync and stuff
@ -2838,7 +2842,7 @@ sub number_of_replacements {
sub replaced {
my $self = shift;
if(not defined $self->{'replaced'}) {
$self->{'replaced'} = $self->replace_placeholders($self->{'command'});
$self->{'replaced'} = $self->replace_placeholders($self->{'command'},0);
if($::opt_nice) {
# Prepend nice -n19 bash -c
# and quote
@ -2854,8 +2858,9 @@ sub replaced {
sub replace_placeholders {
my $self = shift;
my($target) = shift;
my($context_replace) = $self->{'context_replace'};
my $target = shift;
my $quote_special_chars = shift;
my $context_replace = $self->{'context_replace'};
my $context_regexp = $context_replace ? '\S*' : ''; # Regexp to match surrounding context
if($self->number_of_args() == 0) {
@ -2897,13 +2902,13 @@ sub replace_placeholders {
0 .. $#args);
}
} else {
die;
die('This should not happen. Contact <parallel@gnu.org>.');
}
}
my $replacements = 0;
if(%replace_single) {
my $single_regexp = join('|', map { $_=~s/(\W)/\\$1/g; $_} sort keys %replace_single);
my $single_regexp = join('|', map { $_=~s/(\W)/\\$1/g; $_} sort keys %replace_single);
$replacements += ($target =~ s/($single_regexp)/$replace_single{$1}/ge);
}
my $orig_target = $target;
@ -2913,29 +2918,62 @@ sub replace_placeholders {
$a=~s/(\W)/\\$1/g; $a
} @used_multi);
my %wordargs;
while($target =~ s/(\S*($multi_regexp)\S*)/\0/o) {
my $wordarg = $1;
my $pattern = $2;
if($self->{'context_replace'}) {
my $substituted = $wordarg;
my @all=();
for my $argref (@replace_context) {
# for each argument convert a{}b to a1b a2b
if($quote_special_chars) {
while($target =~ s/(.*($multi_regexp).*)/\0/o) {
my $wordarg = $1;
my $pattern = $2;
if($self->{'context_replace'}) {
my $substituted = $wordarg;
$substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g;
CORE::push @all,$substituted;
my @all=();
for my $argref (@replace_context) {
# for each argument convert a{}b to a1b a2b
my $substituted = $wordarg;
$substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g;
CORE::push @all,$substituted;
}
$wordargs{$wordarg} = join" ",@all;
return @all;
} else {
my $substituted = $wordarg;
$substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg;
$wordargs{$wordarg} = $substituted;
}
}
} else {
while($target =~ s/(\S*($multi_regexp)\S*)/\0/o) {
my $wordarg = $1;
my $pattern = $2;
if($self->{'context_replace'}) {
my $substituted = $wordarg;
my @all=();
for my $argref (@replace_context) {
# for each argument convert a{}b to a1b a2b
my $substituted = $wordarg;
$substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g;
CORE::push @all,$substituted;
}
$wordargs{$wordarg} = join" ",@all;
} else {
my $substituted = $wordarg;
$substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg;
$wordargs{$wordarg} = $substituted;
}
$wordargs{$wordarg} = join" ",@all;
} else {
my $substituted = $wordarg;
$substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg;
$wordargs{$wordarg} = $substituted;
}
}
my @k=keys %wordargs;
for(@k) {s/(\W)/\\$1/g};
my $regexp=join("|",@k);
$orig_target =~s/($regexp)/$wordargs{$1}/g;
if($quote_special_chars) {
# When --return'ing a file with added special chars
# they need to be quoted.
# E.g. --trc 'a {}'
# Not really pretty. Can this be done better?
$orig_target =~s/($regexp)/::shell_unquote($wordargs{$1})/ge;
$orig_target = ::shell_quote_scalar($orig_target);
} else {
$orig_target =~s/($regexp)/$wordargs{$1}/g;
}
}
return $orig_target;
}

2
testsuite/tests-to-run/test12.sh
View file

@ -1,6 +1,6 @@
#!/bin/bash
# Test if we can deal with output > 4 GB
echo | niceload -l=1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
echo | niceload -l 1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
# dd does not work with niceload (no idea why)
#echo | parallel 'dd if=/dev/zero count=43 bs=100000k; echo 1; echo 2' | md5sum

7
testsuite/tests-to-run/test42.sh
View file

@ -27,9 +27,12 @@ SERVER1=parallel-server3
SERVER2=parallel-server2
rm -rf tmp
echo "### Test combined --return {/}_{/.}_{#/.}_{#/}_{#.}"
echo "### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"
stdout parallel -k -Xv --cleanup --return tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file -S parallel@$SERVER2 \
mkdir -p tmp/{/}_{/.}_{2/.}_{2/}_{2.} \;touch tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file \
::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6
::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 'number 7' 'number <8|8>'
find tmp
rm -rf tmp
echo "### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"
echo "### But we will wait for a real world scenario"

14
testsuite/tests-to-run/test46.sh Normal file
View file

@ -0,0 +1,14 @@
#!/bin/bash
SERVER1=parallel-server3
SERVER2=parallel-server2
echo '### Test --trc with space added in filename'
echo original > '/tmp/parallel space file'
echo '/tmp/parallel space file' | parallel --trc "{} more space" -S $SERVER1 cat {} ">{}\\ more\\ space"
cat '/tmp/parallel space file more space'
echo '### Test --trc with >|< added in filename'
echo original > '/tmp/parallel space file'
echo '/tmp/parallel space file' | parallel --trc "{} >|<" -S $SERVER1 cat {} ">{}\\ \\>\\|\\<"
cat '/tmp/parallel space file >|<'

2
testsuite/wanted-results/niceload01 Normal file
View file

@ -0,0 +1,2 @@
### Test niceload
..........

12
testsuite/wanted-results/test42
View file

@ -24,13 +24,16 @@ number2
number2.c
### Test {#.}
a/number2
### Test combined --return {/}_{/.}_{#/.}_{#/}_{#.}
### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
Warning: using -X or -m with --sshlogin may fail
mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file
mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 tmp/number\ 7_number\ 7_number2_number2.c_a/number2 tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file tmp/number\ 7_number\ 7_number2_number2.c_a/number2/file tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2/file
tmp
tmp/number6_number6_number2_number2.c_a
tmp/number6_number6_number2_number2.c_a/number2
tmp/number6_number6_number2_number2.c_a/number2/file
tmp/number <8|8>_number <8|8>_number2_number2.c_a
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2
tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2/file
tmp/number3.c_number3_number2_number2.c_a
tmp/number3.c_number3_number2_number2.c_a/number2
tmp/number3.c_number3_number2_number2.c_a/number2/file
@ -46,3 +49,8 @@ tmp/number1.c_number1_number2_number2.c_a/number2/file
tmp/number4_number4_number2_number2.c_a
tmp/number4_number4_number2_number2.c_a/number2
tmp/number4_number4_number2_number2.c_a/number2/file
tmp/number 7_number 7_number2_number2.c_a
tmp/number 7_number 7_number2_number2.c_a/number2
tmp/number 7_number 7_number2_number2.c_a/number2/file
### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
### But we will wait for a real world scenario

4
testsuite/wanted-results/test46 Normal file
View file

@ -0,0 +1,4 @@
### Test --trc with space added in filename
original
### Test --trc with >|< added in filename
original