parallel: --dryrun implemented. --return of multiple files with

-X. --return of files containing space. Passes testsuite.

doc/FUTURE_IDEAS

@@ -1,6 +1,15 @@
---command, -c, --file, and -f now obsoleted. --eta works again.
+parallel: --dryrun implemented. --return of multiple files with
-Bugfix in testsuite for --retries.
+-X. --return of files containing space.
-Lots of dead code removed.
+
+
+== Bug ==
+
+--trc {}' 'done
+
+echo 1' '2 | parallel -vv --trc {}' 'done -S nlv.pi.dk touch {}' 'done
+
+(echo 1' '2;echo a' 'b) | parallel -qvv --trc {}' 'done -S nlv.pi.dk touch {}' 'done
+
 
 == Bug? ==
 

doc/release_new_version

@@ -134,6 +134,9 @@ download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
+* GNU niceload is now part of GNU Parallel. GNU niceload slows down a
+  program if the load average is above a certain limit.
+
 * Implemented --tmpdir to buffer standard output and standard error in
   a different place.
 

src/niceload

@@ -2,7 +2,7 @@
 
 =head1 NAME
 
-niceload - run a program when the load is below a certain limit
+niceload - slow down a program when the load average is above a certain limit
 
 =head1 SYNOPSIS
 

src/parallel

@@ -136,6 +136,7 @@ sub get_options_from_array {
 	 "tempdir=s" => \$::opt_tmpdir,
 	 "halt-on-error|H=s" => \$::opt_halt_on_error,
 	 "retries=i" => \$::opt_retries,
+	 "dry-run|dryrun" => \$::opt_dryrun,
 	 "progress" => \$::opt_progress,
 	 "eta" => \$::opt_eta,
 	 "arg-sep|argsep=s" => \$::opt_arg_sep,
@@ -2301,7 +2302,7 @@ sub return {
     my $self = shift;
     my @return = ();
     for my $return (@{$self->{'commandline'}{'return_files'}}) {
-	CORE::push @return, $self->{'commandline'}->replace_placeholders($return);
+	CORE::push @return, $self->{'commandline'}->replace_placeholders($return,1);
     }
     return @return;
 }
@@ -2452,14 +2453,17 @@ sub start {
 	    print $Global::original_stderr "$command\n";
 	}
     }
-    if($Global::verbose and not $Global::grouped) {
+    if(($::opt_dryrun or $Global::verbose) and not $Global::grouped) {
-	if($Global::verbose == 1) {
+	if($Global::verbose <= 1) {
 	    print STDOUT $job->replaced(),"\n";
 	} else {
 	    # Verbose level > 1: Print the rsync and stuff
 	    print STDOUT $command,"\n";
 	}
     }
+    if($::opt_dryrun) {
+	$command = "true";
+    }
     $Global::total_running++;
     $Global::total_started++;
     if(not $job->seq()) {
@@ -2536,8 +2540,8 @@ sub print {
     my $err = $self->stderr();
     my ($command) = $self->sshlogin_wrap();
 
-    if($Global::verbose and $Global::grouped) {
+    if(($::opt_dryrun or $Global::verbose) and $Global::grouped) {
-	if($Global::verbose == 1) {
+	if($Global::verbose <= 1) {
 	    print STDOUT $self->replaced(),"\n";
 	} else {
 	    # Verbose level > 1: Print the rsync and stuff
@@ -2838,7 +2842,7 @@ sub number_of_replacements {
 sub replaced {
     my $self = shift;
     if(not defined $self->{'replaced'}) {
-	$self->{'replaced'} = $self->replace_placeholders($self->{'command'});
+	$self->{'replaced'} = $self->replace_placeholders($self->{'command'},0);
 	if($::opt_nice) {
 	    # Prepend nice -n19 bash -c
 	    # and quote
@@ -2854,8 +2858,9 @@ sub replaced {
 
 sub replace_placeholders {
     my $self = shift;
-    my($target) = shift;
+    my $target = shift;
-    my($context_replace) = $self->{'context_replace'};
+    my $quote_special_chars = shift;
+    my $context_replace = $self->{'context_replace'};
 
     my $context_regexp = $context_replace ? '\S*' : ''; # Regexp to match surrounding context
     if($self->number_of_args() == 0) {
@@ -2897,7 +2902,7 @@ sub replace_placeholders {
 			   0 .. $#args);
 	    }
 	} else {
-	    die;
+	    die('This should not happen. Contact <parallel@gnu.org>.');
 	}
     }
 
@@ -2913,6 +2918,28 @@ sub replace_placeholders {
 	    $a=~s/(\W)/\\$1/g; $a
 				} @used_multi);
 	my %wordargs;
+	if($quote_special_chars) {
+	    while($target =~ s/(.*($multi_regexp).*)/\0/o) {
+		my $wordarg = $1;
+		my $pattern = $2;
+		if($self->{'context_replace'}) {
+		    my $substituted = $wordarg;
+		    my @all=();
+		    for my $argref (@replace_context) {
+			# for each argument convert a{}b to a1b a2b
+			my $substituted = $wordarg;
+			$substituted =~ s/($multi_regexp)/$argref->{$Global::replace_rev{$1}}/g;
+			CORE::push @all,$substituted;
+		    }
+		    $wordargs{$wordarg} = join" ",@all;
+		    return @all;
+		} else {
+		    my $substituted = $wordarg;
+		    $substituted =~ s/($multi_regexp)/join(" ",map {$_} @{$replace_multi{$Global::replace_rev{$1}}})/eg;
+		    $wordargs{$wordarg} = $substituted;
+		}
+	    }
+	} else {
 	    while($target =~ s/(\S*($multi_regexp)\S*)/\0/o) {
 		my $wordarg = $1;
 		my $pattern = $2;
@@ -2932,11 +2959,22 @@ sub replace_placeholders {
 		    $wordargs{$wordarg} = $substituted;
 		}
 	    }
+	}
+
 	my @k=keys %wordargs;
 	for(@k) {s/(\W)/\\$1/g};
 	my $regexp=join("|",@k);
+	if($quote_special_chars) {
+	    # When --return'ing a file with added special chars
+	    # they need to be quoted.
+	    # E.g. --trc 'a {}'
+	    # Not really pretty. Can this be done better?
+	    $orig_target =~s/($regexp)/::shell_unquote($wordargs{$1})/ge;
+	    $orig_target = ::shell_quote_scalar($orig_target);
+	} else {
 	    $orig_target =~s/($regexp)/$wordargs{$1}/g;
 	}
+    }
     return $orig_target;
 }
 

testsuite/tests-to-run/test12.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 # Test if we can deal with output > 4 GB
-echo | niceload -l=1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
+echo | niceload -l 1.5 parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
 # dd does not work with niceload (no idea why)
 #echo | parallel 'dd if=/dev/zero count=43 bs=100000k; echo 1; echo 2' | md5sum

testsuite/tests-to-run/test42.sh

@@ -27,9 +27,12 @@ SERVER1=parallel-server3
 SERVER2=parallel-server2
 rm -rf tmp
 
-echo "### Test combined --return {/}_{/.}_{#/.}_{#/}_{#.}"
+echo "### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"
 stdout parallel -k -Xv --cleanup --return tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file -S parallel@$SERVER2 \
 mkdir -p tmp/{/}_{/.}_{2/.}_{2/}_{2.} \;touch tmp/{/}_{/.}_{2/.}_{2/}_{2.}/file \
-::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6
+::: /a/number1.c a/number2.c number3.c /a/number4 a/number5 number6 'number 7' 'number <8|8>'
 find tmp
 rm -rf tmp
+
+echo "### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space"
+echo "### But we will wait for a real world scenario"

testsuite/tests-to-run/test46.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+SERVER1=parallel-server3
+SERVER2=parallel-server2
+
+echo '### Test --trc with space added in filename'
+echo original > '/tmp/parallel space file'
+echo '/tmp/parallel space file' | parallel --trc "{} more space" -S $SERVER1 cat {} ">{}\\ more\\ space"
+cat '/tmp/parallel space file more space'
+
+echo '### Test --trc with >|< added in filename'
+echo original > '/tmp/parallel space file'
+echo '/tmp/parallel space file' | parallel --trc "{} >|<" -S $SERVER1 cat {} ">{}\\ \\>\\|\\<"
+cat '/tmp/parallel space file >|<'

testsuite/wanted-results/niceload01

@@ -0,0 +1,2 @@
+### Test niceload
+..........

testsuite/wanted-results/test42

@@ -24,13 +24,16 @@ number2
 number2.c
 ### Test {#.}
 a/number2
-### Test combined --return {/}_{/.}_{#/.}_{#/}_{#.}
+### Test combined -X --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
 Warning: using -X or -m with --sshlogin may fail
-mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file
+mkdir -p tmp/number1.c_number1_number2_number2.c_a/number2 tmp/number2.c_number2_number2_number2.c_a/number2 tmp/number3.c_number3_number2_number2.c_a/number2 tmp/number4_number4_number2_number2.c_a/number2 tmp/number5_number5_number2_number2.c_a/number2 tmp/number6_number6_number2_number2.c_a/number2 tmp/number\ 7_number\ 7_number2_number2.c_a/number2 tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2 ;touch tmp/number1.c_number1_number2_number2.c_a/number2/file tmp/number2.c_number2_number2_number2.c_a/number2/file tmp/number3.c_number3_number2_number2.c_a/number2/file tmp/number4_number4_number2_number2.c_a/number2/file tmp/number5_number5_number2_number2.c_a/number2/file tmp/number6_number6_number2_number2.c_a/number2/file tmp/number\ 7_number\ 7_number2_number2.c_a/number2/file tmp/number\ \<8\|8\>_number\ \<8\|8\>_number2_number2.c_a/number2/file
 tmp
 tmp/number6_number6_number2_number2.c_a
 tmp/number6_number6_number2_number2.c_a/number2
 tmp/number6_number6_number2_number2.c_a/number2/file
+tmp/number <8|8>_number <8|8>_number2_number2.c_a
+tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2
+tmp/number <8|8>_number <8|8>_number2_number2.c_a/number2/file
 tmp/number3.c_number3_number2_number2.c_a
 tmp/number3.c_number3_number2_number2.c_a/number2
 tmp/number3.c_number3_number2_number2.c_a/number2/file
@@ -46,3 +49,8 @@ tmp/number1.c_number1_number2_number2.c_a/number2/file
 tmp/number4_number4_number2_number2.c_a
 tmp/number4_number4_number2_number2.c_a/number2
 tmp/number4_number4_number2_number2.c_a/number2/file
+tmp/number 7_number 7_number2_number2.c_a
+tmp/number 7_number 7_number2_number2.c_a/number2
+tmp/number 7_number 7_number2_number2.c_a/number2/file
+### Here we ought to test -m --return {/}_{/.}_{#/.}_{#/}_{#.} with files containing space
+### But we will wait for a real world scenario

testsuite/wanted-results/test46

@@ -0,0 +1,4 @@
+### Test --trc with space added in filename
+original
+### Test --trc with >|< added in filename
+original