diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 439c93d6..d88d3bc7 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,6 +1,15 @@ ---command, -c, --file, and -f now obsoleted. --eta works again. -Bugfix in testsuite for --retries. -Lots of dead code removed. +parallel: --dryrun implemented. --return of multiple files with +-X. --return of files containing space. + + +== Bug == + +--trc {}' 'done + +echo 1' '2 | parallel -vv --trc {}' 'done -S nlv.pi.dk touch {}' 'done + +(echo 1' '2;echo a' 'b) | parallel -qvv --trc {}' 'done -S nlv.pi.dk touch {}' 'done + == Bug? == diff --git a/doc/release_new_version b/doc/release_new_version index 3d22ef7f..1f32f453 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -134,6 +134,9 @@ download at: http://ftp.gnu.org/gnu/parallel/ New in this release: +* GNU niceload is now part of GNU Parallel. GNU niceload slows down a + program if the load average is above a certain limit. + * Implemented --tmpdir to buffer standard output and standard error in a different place. diff --git a/src/niceload b/src/niceload index 7a2998c2..0f46583d 100755 --- a/src/niceload +++ b/src/niceload @@ -2,7 +2,7 @@ =head1 NAME -niceload - run a program when the load is below a certain limit +niceload - slow down a program when the load average is above a certain limit =head1 SYNOPSIS diff --git a/src/parallel b/src/parallel index 4087ff66..b68d2ce6 100755 --- a/src/parallel +++ b/src/parallel @@ -136,6 +136,7 @@ sub get_options_from_array { "tempdir=s" => \$::opt_tmpdir, "halt-on-error|H=s" => \$::opt_halt_on_error, "retries=i" => \$::opt_retries, + "dry-run|dryrun" => \$::opt_dryrun, "progress" => \$::opt_progress, "eta" => \$::opt_eta, "arg-sep|argsep=s" => \$::opt_arg_sep, @@ -2301,7 +2302,7 @@ sub return { my $self = shift; my @return = (); for my $return (@{$self->{'commandline'}{'return_files'}}) { - CORE::push @return, $self->{'commandline'}->replace_placeholders($return); + CORE::push @return, $self->{'commandline'}->replace_placeholders($return,1); } return @return; } @@ -2452,14 +2453,17 @@ sub start { print $Global::original_stderr "$command\n"; } } - if($Global::verbose and not $Global::grouped) { - if($Global::verbose == 1) { + if(($::opt_dryrun or $Global::verbose) and not $Global::grouped) { + if($Global::verbose <= 1) { print STDOUT $job->replaced(),"\n"; } else { # Verbose level > 1: Print the rsync and stuff print STDOUT $command,"\n"; } } + if($::opt_dryrun) { + $command = "true"; + } $Global::total_running++; $Global::total_started++; if(not $job->seq()) { @@ -2536,8 +2540,8 @@ sub print { my $err = $self->stderr(); my ($command) = $self->sshlogin_wrap(); - if($Global::verbose and $Global::grouped) { - if($Global::verbose == 1) { + if(($::opt_dryrun or $Global::verbose) and $Global::grouped) { + if($Global::verbose <= 1) { print STDOUT $self->replaced(),"\n"; } else { # Verbose level > 1: Print the rsync and stuff @@ -2838,7 +2842,7 @@ sub number_of_replacements { sub replaced { my $self = shift; if(not defined $self->{'replaced'}) { - $self->{'replaced'} = $self->replace_placeholders($self->{'command'}); + $self->{'replaced'} = $self->replace_placeholders($self->{'command'},0); if($::opt_nice) { # Prepend nice -n19 bash -c # and quote @@ -2854,8 +2858,9 @@ sub replaced { sub replace_placeholders { my $self = shift; - my($target) = shift; - my($context_replace) = $self->{'context_replace'}; + my $target = shift; + my $quote_special_chars = shift; + my $context_replace = $self->{'context_replace'}; my $context_regexp = $context_replace ? '\S*' : ''; # Regexp to match surrounding context if($self->number_of_args() == 0) { @@ -2897,13 +2902,13 @@ sub replace_placeholders { 0 .. $#args); } } else { - die; + die('This should not happen. Contact .'); } } my $replacements = 0; if(%replace_single) { - my $single_regexp = join('|', map { $_=~s/(\W)/\\$1/g; $_} sort keys %replace_single); + my $single_regexp = join('|', map { $_=~s/(\W)/\\$1/g; $_} sort keys %replace_single); $replacements += ($target =~ s/($single_regexp)/$replace_single{$1}/ge); } my $orig_target = $target; @@ -2913,29 +2918,62 @@ sub replace_placeholders { $a=~s/(\W)/\\$1/g; $a } @used_multi); my %wordargs; - while($target =~ s/(\S*($multi_regexp)\S*)/\0/o) { - my $wordarg = $1; - my $pattern = $2; - if($self->{'context_replace'}) { - my $substituted = $wordarg; - my @all=(); - for my $argref (@replace_context) { - # for each argument convert a{}b to a1b a2b + if($quote_special_chars) { + while($target =~ s/(.*($multi_regexp).*)/\0/o) { + my $wordarg = $1; + my $pattern = $2; + if($self->{'context_replace'}) { my $substituted = $wordarg; - $substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g; - CORE::push @all,$substituted; + my @all=(); + for my $argref (@replace_context) { + # for each argument convert a{}b to a1b a2b + my $substituted = $wordarg; + $substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g; + CORE::push @all,$substituted; + } + $wordargs{$wordarg} = join" ",@all; + return @all; + } else { + my $substituted = $wordarg; + $substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg; + $wordargs{$wordarg} = $substituted; + } + } + } else { + while($target =~ s/(\S*($multi_regexp)\S*)/\0/o) { + my $wordarg = $1; + my $pattern = $2; + if($self->{'context_replace'}) { + my $substituted = $wordarg; + my @all=(); + for my $argref (@replace_context) { + # for each argument convert a{}b to a1b a2b + my $substituted = $wordarg; + $substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g; + CORE::push @all,$substituted; + } + $wordargs{$wordarg} = join" ",@all; + } else { + my $substituted = $wordarg; + $substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg; + $wordargs{$wordarg} = $substituted; } - $wordargs{$wordarg} = join" ",@all; - } else { - my $substituted = $wordarg; - $substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg; - $wordargs{$wordarg} = $substituted; } } + my @k=keys %wordargs; for(@k) {s/(\W)/\\$1/g}; my $regexp=join("|",@k); - $orig_target =~s/($regexp)/$wordargs{$1}/g; + if($quote_special_chars) { + # When --return'ing a file with added special chars + # they need to be quoted. + # E.g. --trc 'a {}' + # Not really pretty. Can this be done better? + $orig_target =~s/($regexp)/::shell_unquote($wordargs{$1})/ge; + $orig_target = ::shell_quote_scalar($orig_target); + } else { + $orig_target =~s/($regexp)/$wordargs{$1}/g; + } } return $orig_target; } diff --git a/testsuite/tests-to-run/test12.sh b/testsuite/tests-to-run/test12.sh index c948babe..7a7e728b 100755 --- a/testsuite/tests-to-run/test12.sh +++ b/testsuite/tests-to-run/test12.sh @@ -1,6 +1,6 @@ #!/bin/bash # Test if we can deal with output > 4 GB -echo | niceload -l=1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum +echo | niceload -l 1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum # dd does not work with niceload (no idea why) #echo | parallel 'dd if=/dev/zero count=43 bs=100000k; echo 1; echo 2' | md5sum diff --git a/testsuite/tests-to-run/test42.sh b/testsuite/tests-to-run/test42.sh index 7251fef3..e002f16d 100644 --- a/testsuite/tests-to-run/test42.sh +++ b/testsuite/tests-to-run/test42.sh @@ -27,9 +27,12 @@ SERVER1=parallel-server3 SERVER2=parallel-server2 rm -rf tmp -echo "### Test combined --return {/}_{/.}_{#/.}_{#/}_{#.}" +echo "### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space" stdout parallel -k -Xv --cleanup --return tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file -S parallel@$SERVER2 \ mkdir -p tmp/{/}_{/.}_{2/.}_{2/}_{2.} \;touch tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file \ -::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 +::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 'number 7' 'number <8|8>' find tmp rm -rf tmp + +echo "### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space" +echo "### But we will wait for a real world scenario" diff --git a/testsuite/tests-to-run/test46.sh b/testsuite/tests-to-run/test46.sh new file mode 100644 index 00000000..bc4e5c1f --- /dev/null +++ b/testsuite/tests-to-run/test46.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +SERVER1=parallel-server3 +SERVER2=parallel-server2 + +echo '### Test --trc with space added in filename' +echo original > '/tmp/parallel space file' +echo '/tmp/parallel space file' | parallel --trc "{} more space" -S $SERVER1 cat {} ">{}\\ more\\ space" +cat '/tmp/parallel space file more space' + +echo '### Test --trc with >|< added in filename' +echo original > '/tmp/parallel space file' +echo '/tmp/parallel space file' | parallel --trc "{} >|<" -S $SERVER1 cat {} ">{}\\ \\>\\|\\<" +cat '/tmp/parallel space file >|<' diff --git a/testsuite/wanted-results/niceload01 b/testsuite/wanted-results/niceload01 new file mode 100644 index 00000000..8aebce90 --- /dev/null +++ b/testsuite/wanted-results/niceload01 @@ -0,0 +1,2 @@ +### Test niceload +.......... diff --git a/testsuite/wanted-results/test42 b/testsuite/wanted-results/test42 index 9543c158..4a893b5e 100644 --- a/testsuite/wanted-results/test42 +++ b/testsuite/wanted-results/test42 @@ -24,13 +24,16 @@ number2 number2.c ### Test {#.} a/number2 -### Test combined --return {/}_{/.}_{#/.}_{#/}_{#.} +### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space Warning: using -X or -m with --sshlogin may fail -mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file +mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 tmp/number\ 7_number\ 7_number2_number2.c_a/number2 tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file tmp/number\ 7_number\ 7_number2_number2.c_a/number2/file tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2/file tmp tmp/number6_number6_number2_number2.c_a tmp/number6_number6_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2/file +tmp/number <8|8>_number <8|8>_number2_number2.c_a +tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2 +tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2/file @@ -46,3 +49,8 @@ tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a tmp/number4_number4_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2/file +tmp/number 7_number 7_number2_number2.c_a +tmp/number 7_number 7_number2_number2.c_a/number2 +tmp/number 7_number 7_number2_number2.c_a/number2/file +### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space +### But we will wait for a real world scenario diff --git a/testsuite/wanted-results/test46 b/testsuite/wanted-results/test46 new file mode 100644 index 00000000..86b2c96c --- /dev/null +++ b/testsuite/wanted-results/test46 @@ -0,0 +1,4 @@ +### Test --trc with space added in filename +original +### Test --trc with >|< added in filename +original