Bugfix: --keeporder with -j100% mixed up order of first job

parallel: Unittest for --retries. Passes unittest.

NEWS

@@ -1,3 +1,29 @@
+20100922
+
+* See GNU Parallel live at FSCONS 2010-11-07:
+  http://www.fscons.org/fs/gnu-parallel
+
+* Untested Debian and xUbuntu packages available through OpenSUSE
+  build service:
+  https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
+
+* Using --retries a job will be retried on another computer if it
+  fails. This is useful if some jobs fail for no apparent reason (such
+  as network failure).
+
+* BSD xargs -o (open /dev/tty) is now default for the job running in
+  foreground. Useful for interactive commands like:
+  ls | parallel -Xuj1 vi
+
+* GNU sql now supports SQLite.
+
+* Renamed .dburl.aliases to .sql/aliases and /etc/sql/aliases.
+
+* GNU sql now support --list-tables
+
+* Alias for DBURL can contain '?query' part with %-quoting.
+
+
 20100906
 
 * Using --shebang GNU Parallel can be used as the parser for a script.

doc/FUTURE_IDEAS

@@ -1,24 +1,7 @@
 == Retry ==
 
---retries n
+Bugfix: --keeporder with -j100% mixed up order of first job
-If a job fails, retry it on another computer. Do this n times. If there are fewer than n computers in --sshlogin GNU parallel will re-use the computers. This is useful if some jobs fail for no apparent reason (such as network failure).
+parallel: Unittest for --retries
-
-When the command dies and fails:
-Update $Global::failed_command{$test_failed}
-If max n has been reached for all: fail
-
-What if all jobs in the queue failed for this sshlogin so only undef is left for this sshlogin?
-
-BUG: Rækkeflg er forkert.
-(seq 0 1) | parallel -kj100%  -S 1/:,2/parallel@server2 -vq perl -e 'sleep 1;print "job{}\n";exit({})' 
-OK: (seq 0 1) | parallel -kj2  -S 1/:,2/parallel@server2 -vq perl -e 'sleep 1;print "job{}\n";exit({})' 
-Fejlen eksisterer også i 20100906 og 20100822
-
-perl -e sleep\ 1\;print\ \"job0\\n\"\;exit\(0\)ARRAY(0x965e928)perl -e sleep\ 1\;print\ \"job1\\n\"\;exit\(1\)ARRAY(0x9727bc0)perl -e sleep\ 1\;print\ \"job2\\n\"\;exit\(2\)ARRAY(0x9727a70)Limited to procs: 3
-Wanted procs: 1
-Time to fork ten procs:  (processes so far: 1)
-perl -e sleep\ 1\;print\ \"job1\\n\"\;exit\(1\)ARRAY(0x9727bc0)perl -e sleep\ 1\;print\ \"job2\\n\"\;exit\(2\)ARRAY(0x9727a70)perl -e sleep\ 1\;print\ \"job0\\n\"\;exit\(0\)ARRAY(0x965e928)Limited to procs: 1
-Running jobs on parallel@server2: 0
 
 == FSCONS ==
 
@@ -96,11 +79,12 @@ find music-files -type f | parallel -j+0 lame {} -o {.}.mp3
 # Give us progress information
 # For 'foo.wav' call the output file 'foo.mp3'
 
-find music-files -type f | parallel -j+0 -S :,computer1.examle.com,computer2.example.com \
+find music-files -type f | parallel -j+0 -S :,server1,server2 \
 --eta --trc {.}.mp3 lame {} -o {.}.mp3
 
 # Colsep
 # sem
+# --retry
 
 
 =head1 YouTube video

doc/release_new_version

@@ -83,7 +83,7 @@ https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel
 
 == Update Twitter ==
 
-#GNU #parallel 20100906 released. See what is new in this release
+#GNU #parallel 20100922 released. See what is new in this release
 http://nd.gd/2j Watch the intro video http://nd.gd/0s
 
 https://savannah.gnu.org/news/?group=parallel
@@ -116,23 +116,37 @@ cc:Peter Simons <simons@cryp.to>, Sandro Cazzaniga <kharec@mandriva.org>,
    Phil Sung <psung@alum.mit.edu>, Michael Shigorin <mike@altlinux.org>,
    Andrew McFague <amcfague@wgen.net>, Steven M. Christensen <sunfreeware@gmail.com>
 
-Subject: GNU Parallel 2010XXXX released
+Subject: GNU Parallel 20100922 released
 
-GNU Parallel 2010XXXX has been released. It is available for
+GNU Parallel 20100922 has been released. It is available for
 download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
+* See GNU Parallel live at FSCONS 2010-11-07:
+  http://www.fscons.org/fs/gnu-parallel
+
 * Untested Debian and xUbuntu packages available through OpenSUSE
   build service:
   https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
 
+* Using --retries a job will be retried on another computer if it
+  fails. This is useful if some jobs fail for no apparent reason (such
+  as network failure).
+
 * BSD xargs -o (open /dev/tty) is now default for the job running in
   foreground. Useful for interactive commands like:
   ls | parallel -Xuj1 vi
 
+* GNU sql now supports SQLite.
+
 * Renamed .dburl.aliases to .sql/aliases and /etc/sql/aliases.
 
+* GNU sql now support --list-tables
+
+* Alias for DBURL can contain '?query' part with %-quoting.
+
+
 = About GNU Parallel =
 
 GNU Parallel is a shell tool for executing jobs in parallel using one

src/parallel

@@ -729,7 +729,7 @@ Can be reversed with B<-v>.
 Print the command line on the standard error output before executing
 it.
 
-See also B<-v>.
+See also B<-v> and B<-p>.
 
 
 =item B<--transfer>
@@ -1397,7 +1397,10 @@ because > and | need to be interpreted by the shell.
 
 If you get errors like:
 
-B<sh: -c: line 0: syntax error near unexpected token>
+  sh: -c: line 0: syntax error near unexpected token
+  sh: Syntax error: Unterminated quoted string
+  sh: -c: line 0: unexpected EOF while looking for matching `''
+  sh: -c: line 1: syntax error: unexpected end of file
 
 then you might try using B<-q>.
 
@@ -2180,7 +2183,7 @@ if($Global::semaphore) {
 if($::opt_halt_on_error) {
     wait_and_exit($Global::halt_on_error_exitstatus);
 } else {
-    wait_and_exit(min($Global::exitstatus,254));
+    wait_and_exit(min(undef_as_zero($Global::exitstatus),254));
 }
 
 sub acquire_semaphore {
@@ -2266,7 +2269,7 @@ sub parse_options {
 	       "X" => \$::opt_X,
 	       "v" => \$::opt_v,
 	       "silent" => \$::opt_silent,
-	       "keeporder|k" => \$::opt_k,
+	       "keep-order|keeporder|k" => \$::opt_k,
 	       "group|g" => \$::opt_g,
 	       "ungroup|u" => \$::opt_u,
 	       "command|c" => \$::opt_c,
@@ -2980,10 +2983,8 @@ sub compute_number_of_processes {
     my $sshlogin = shift;
     my $wanted_processes = user_requested_processes($opt_P,$sshlogin);
     debug("Wanted procs: $wanted_processes\n");
-    debug("unget before:",@Global::unget_next_command_line);
     my $system_limit =
 	processes_available_by_system_limit($wanted_processes,$sshlogin);
-    debug("unget after:",@Global::unget_next_command_line);
     debug("Limited to procs: $system_limit\n");
     return $system_limit;
 }
@@ -3371,12 +3372,12 @@ sub no_of_cores_solaris {
 sub min {
     # Returns:
     #   Minimum value of array
-    my $min = shift;
+    my $min;
-    my @args = @_;
+    for (@_) {
-    for my $a (@args) {
 	# Skip undefs
-	defined $a or next;
+	defined $_ or next;
-	$min = ($min < $a) ? $min : $a;
+	defined $min or do { $min = $_; next; }; # Set $_ to the first non-undef
+	$min = ($min < $_) ? $min : $_;
     }
     return $min;
 }
@@ -3384,12 +3385,12 @@ sub min {
 sub max {
     # Returns:
     #   Maximum value of array
-    my $max = shift;
+    my $max;
-    my @args = @_;
+    for (@_) {
-    for my $a (@args) {
 	# Skip undefs
-	defined $a or next;
+	defined $_ or next;
-	$max = ($max > $a) ? $max : $a;
+	defined $max or do { $max = $_; next; }; # Set $_ to the first non-undef
+	$max = ($max > $_) ? $max : $_;
     }
     return $max;
 }
@@ -3944,7 +3945,7 @@ sub get_command_line {
 
 sub unget_command_line {
     # Returns: N/A
-    push @Global::unget_next_command_line, @_;
+    unshift @Global::unget_next_command_line, @_;
 }
 
 sub more_arguments {

unittest/tests-to-run/test13.sh

@@ -5,5 +5,11 @@ ulimit -n 50
 (echo "sleep 3; echo begin"; seq 1 30 | parallel -kq echo "sleep 1; echo {}"; echo "echo end") \
 | parallel -k -j0
 
+echo '### Test --keep-order'
+(seq 0 2) | parallel --keep-order -j100% -S 1/:,2/parallel@server2 -q perl -e 'sleep 1;print "job{}\n";exit({})'
+
+echo '### Test --keeporder'
+(seq 0 2) | parallel --keeporder -j100% -S 1/:,2/parallel@server2 -q perl -e 'sleep 1;print "job{}\n";exit({})'
+
 echo '### Test SIGTERM'
 (sleep 5; killall parallel -TERM) & seq 1 100 | parallel -k sleep 3';' echo

unittest/tests-to-run/test14.sh

@@ -1,22 +1,20 @@
 #!/bin/bash
 
-PAR=parallel
-
 # Test -I
-seq 1 10 | $PAR -k 'seq 1 {} | '$PAR' -k -I :: echo {} ::'
+seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'
 
-seq 1 10 | $PAR -k 'seq 1 {} | '$PAR' -X -k -I :: echo a{} b::'
+seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::'
 
-seq 1 10 | $PAR -k 'seq 1 {} | '$PAR' -m -k -I :: echo a{} b::'
+seq 1 10 | parallel -k 'seq 1 {} | parallel -m -k -I :: echo a{} b::'
 
-seq 1 60000 | $PAR -I :: -m echo a::b::c | \
+seq 1 60000 | parallel -I :: -m echo a::b::c | \
   mop -q "|sort |md5sum" :par
 CHAR=$(cat ~/.mop/:par | wc -c)
 LINES=$(cat ~/.mop/:par | wc -l)
 echo -n "Chars per line ($CHAR/$LINES): "
 echo "$CHAR/$LINES" | bc
 
-seq 1 60000 | $PAR -I :: -X echo a::b::c | \
+seq 1 60000 | parallel -I :: -X echo a::b::c | \
   mop -q "|sort |md5sum" :par
 CHAR=$(cat ~/.mop/:par | wc -c)
 LINES=$(cat ~/.mop/:par | wc -l)

unittest/tests-to-run/test20.sh

@@ -1,16 +1,14 @@
 #!/bin/bash
 
-PAR=parallel
-
 echo '### Test --number-of-cpus'
-$PAR --number-of-cpus
+parallel --number-of-cpus
 
 echo '### Test --number-of-cores'
-$PAR --number-of-cores
+parallel --number-of-cores
 
 echo '### Test --use-cpus-instead-of-cores'
-(seq 1 4 | $PAR --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done &
+(seq 1 4 | parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done &
-(seq 1 4 | $PAR -j100% sleep) && echo cores done &
+(seq 1 4 | parallel -j100% sleep) && echo cores done &
 echo 'Cores should complete first on machines with less than 4 physical CPUs'
 wait
 

unittest/wanted-results/test13

@@ -31,6 +31,14 @@ begin
 29
 30
 end
+### Test --keep-order
+job0
+job1
+job2
+### Test --keeporder
+job0
+job1
+job2
 ### Test SIGTERM
 1
 2

unittest/wanted-results/test32

@@ -0,0 +1,27 @@
+### Test of --retries
+echo 1
+1
+echo 2
+2
+echo 3
+3
+echo 4
+4
+echo 5
+5
+echo 6
+6
+echo 7
+7
+echo 8
+8
+echo 9
+9
+echo 10
+10
+### Test of --retries - it should run 13 jobs in total
+13
+### Test of --retries - it should run 25 jobs in total
+25
+### Test of --retries - it should run 49 jobs in total
+49