mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
Bugfix: --keeporder with -j100% mixed up order of first job
parallel: Unittest for --retries. Passes unittest.
This commit is contained in:
Ole Tange
parent
2e572c00c9
commit
59cb631264
26
NEWS
26
NEWS
|
|
@ -1,3 +1,29 @@
|
|||
20100922
|
||||
|
||||
* See GNU Parallel live at FSCONS 2010-11-07:
|
||||
http://www.fscons.org/fs/gnu-parallel
|
||||
|
||||
* Untested Debian and xUbuntu packages available through OpenSUSE
|
||||
build service:
|
||||
https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
|
||||
|
||||
* Using --retries a job will be retried on another computer if it
|
||||
fails. This is useful if some jobs fail for no apparent reason (such
|
||||
as network failure).
|
||||
|
||||
* BSD xargs -o (open /dev/tty) is now default for the job running in
|
||||
foreground. Useful for interactive commands like:
|
||||
ls | parallel -Xuj1 vi
|
||||
|
||||
* GNU sql now supports SQLite.
|
||||
|
||||
* Renamed .dburl.aliases to .sql/aliases and /etc/sql/aliases.
|
||||
|
||||
* GNU sql now support --list-tables
|
||||
|
||||
* Alias for DBURL can contain '?query' part with %-quoting.
|
||||
|
||||
|
||||
20100906
|
||||
|
||||
* Using --shebang GNU Parallel can be used as the parser for a script.
|
||||
|
|
|
|||
|
|
@ -1,24 +1,7 @@
|
|||
== Retry ==
|
||||
|
||||
--retries n
|
||||
If a job fails, retry it on another computer. Do this n times. If there are fewer than n computers in --sshlogin GNU parallel will re-use the computers. This is useful if some jobs fail for no apparent reason (such as network failure).
|
||||
|
||||
When the command dies and fails:
|
||||
Update $Global::failed_command{$test_failed}
|
||||
If max n has been reached for all: fail
|
||||
|
||||
What if all jobs in the queue failed for this sshlogin so only undef is left for this sshlogin?
|
||||
|
||||
BUG: Rækkeflg er forkert.
|
||||
(seq 0 1) | parallel -kj100% -S 1/:,2/parallel@server2 -vq perl -e 'sleep 1;print "job{}\n";exit({})'
|
||||
OK: (seq 0 1) | parallel -kj2 -S 1/:,2/parallel@server2 -vq perl -e 'sleep 1;print "job{}\n";exit({})'
|
||||
Fejlen eksisterer også i 20100906 og 20100822
|
||||
|
||||
perl -e sleep\ 1\;print\ \"job0\\n\"\;exit\(0\)ARRAY(0x965e928)perl -e sleep\ 1\;print\ \"job1\\n\"\;exit\(1\)ARRAY(0x9727bc0)perl -e sleep\ 1\;print\ \"job2\\n\"\;exit\(2\)ARRAY(0x9727a70)Limited to procs: 3
|
||||
Wanted procs: 1
|
||||
Time to fork ten procs: (processes so far: 1)
|
||||
perl -e sleep\ 1\;print\ \"job1\\n\"\;exit\(1\)ARRAY(0x9727bc0)perl -e sleep\ 1\;print\ \"job2\\n\"\;exit\(2\)ARRAY(0x9727a70)perl -e sleep\ 1\;print\ \"job0\\n\"\;exit\(0\)ARRAY(0x965e928)Limited to procs: 1
|
||||
Running jobs on parallel@server2: 0
|
||||
Bugfix: --keeporder with -j100% mixed up order of first job
|
||||
parallel: Unittest for --retries
|
||||
|
||||
== FSCONS ==
|
||||
|
||||
|
|
@ -96,11 +79,12 @@ find music-files -type f | parallel -j+0 lame {} -o {.}.mp3
|
|||
# Give us progress information
|
||||
# For 'foo.wav' call the output file 'foo.mp3'
|
||||
|
||||
find music-files -type f | parallel -j+0 -S :,computer1.examle.com,computer2.example.com \
|
||||
find music-files -type f | parallel -j+0 -S :,server1,server2 \
|
||||
--eta --trc {.}.mp3 lame {} -o {.}.mp3
|
||||
|
||||
# Colsep
|
||||
# sem
|
||||
# --retry
|
||||
|
||||
|
||||
=head1 YouTube video
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel
|
|||
|
||||
== Update Twitter ==
|
||||
|
||||
#GNU #parallel 20100906 released. See what is new in this release
|
||||
#GNU #parallel 20100922 released. See what is new in this release
|
||||
http://nd.gd/2j Watch the intro video http://nd.gd/0s
|
||||
|
||||
https://savannah.gnu.org/news/?group=parallel
|
||||
|
|
@ -116,23 +116,37 @@ cc:Peter Simons <simons@cryp.to>, Sandro Cazzaniga <kharec@mandriva.org>,
|
|||
Phil Sung <psung@alum.mit.edu>, Michael Shigorin <mike@altlinux.org>,
|
||||
Andrew McFague <amcfague@wgen.net>, Steven M. Christensen <sunfreeware@gmail.com>
|
||||
|
||||
Subject: GNU Parallel 2010XXXX released
|
||||
Subject: GNU Parallel 20100922 released
|
||||
|
||||
GNU Parallel 2010XXXX has been released. It is available for
|
||||
GNU Parallel 20100922 has been released. It is available for
|
||||
download at: http://ftp.gnu.org/gnu/parallel/
|
||||
|
||||
New in this release:
|
||||
|
||||
* See GNU Parallel live at FSCONS 2010-11-07:
|
||||
http://www.fscons.org/fs/gnu-parallel
|
||||
|
||||
* Untested Debian and xUbuntu packages available through OpenSUSE
|
||||
build service:
|
||||
https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
|
||||
|
||||
* Using --retries a job will be retried on another computer if it
|
||||
fails. This is useful if some jobs fail for no apparent reason (such
|
||||
as network failure).
|
||||
|
||||
* BSD xargs -o (open /dev/tty) is now default for the job running in
|
||||
foreground. Useful for interactive commands like:
|
||||
ls | parallel -Xuj1 vi
|
||||
|
||||
* GNU sql now supports SQLite.
|
||||
|
||||
* Renamed .dburl.aliases to .sql/aliases and /etc/sql/aliases.
|
||||
|
||||
* GNU sql now support --list-tables
|
||||
|
||||
* Alias for DBURL can contain '?query' part with %-quoting.
|
||||
|
||||
|
||||
= About GNU Parallel =
|
||||
|
||||
GNU Parallel is a shell tool for executing jobs in parallel using one
|
||||
|
|
|
|||
35
src/parallel
35
src/parallel
|
|
@ -729,7 +729,7 @@ Can be reversed with B<-v>.
|
|||
Print the command line on the standard error output before executing
|
||||
it.
|
||||
|
||||
See also B<-v>.
|
||||
See also B<-v> and B<-p>.
|
||||
|
||||
|
||||
=item B<--transfer>
|
||||
|
|
@ -1397,7 +1397,10 @@ because > and | need to be interpreted by the shell.
|
|||
|
||||
If you get errors like:
|
||||
|
||||
B<sh: -c: line 0: syntax error near unexpected token>
|
||||
sh: -c: line 0: syntax error near unexpected token
|
||||
sh: Syntax error: Unterminated quoted string
|
||||
sh: -c: line 0: unexpected EOF while looking for matching `''
|
||||
sh: -c: line 1: syntax error: unexpected end of file
|
||||
|
||||
then you might try using B<-q>.
|
||||
|
||||
|
|
@ -2180,7 +2183,7 @@ if($Global::semaphore) {
|
|||
if($::opt_halt_on_error) {
|
||||
wait_and_exit($Global::halt_on_error_exitstatus);
|
||||
} else {
|
||||
wait_and_exit(min($Global::exitstatus,254));
|
||||
wait_and_exit(min(undef_as_zero($Global::exitstatus),254));
|
||||
}
|
||||
|
||||
sub acquire_semaphore {
|
||||
|
|
@ -2266,7 +2269,7 @@ sub parse_options {
|
|||
"X" => \$::opt_X,
|
||||
"v" => \$::opt_v,
|
||||
"silent" => \$::opt_silent,
|
||||
"keeporder|k" => \$::opt_k,
|
||||
"keep-order|keeporder|k" => \$::opt_k,
|
||||
"group|g" => \$::opt_g,
|
||||
"ungroup|u" => \$::opt_u,
|
||||
"command|c" => \$::opt_c,
|
||||
|
|
@ -2980,10 +2983,8 @@ sub compute_number_of_processes {
|
|||
my $sshlogin = shift;
|
||||
my $wanted_processes = user_requested_processes($opt_P,$sshlogin);
|
||||
debug("Wanted procs: $wanted_processes\n");
|
||||
debug("unget before:",@Global::unget_next_command_line);
|
||||
my $system_limit =
|
||||
processes_available_by_system_limit($wanted_processes,$sshlogin);
|
||||
debug("unget after:",@Global::unget_next_command_line);
|
||||
debug("Limited to procs: $system_limit\n");
|
||||
return $system_limit;
|
||||
}
|
||||
|
|
@ -3371,12 +3372,12 @@ sub no_of_cores_solaris {
|
|||
sub min {
|
||||
# Returns:
|
||||
# Minimum value of array
|
||||
my $min = shift;
|
||||
my @args = @_;
|
||||
for my $a (@args) {
|
||||
my $min;
|
||||
for (@_) {
|
||||
# Skip undefs
|
||||
defined $a or next;
|
||||
$min = ($min < $a) ? $min : $a;
|
||||
defined $_ or next;
|
||||
defined $min or do { $min = $_; next; }; # Set $_ to the first non-undef
|
||||
$min = ($min < $_) ? $min : $_;
|
||||
}
|
||||
return $min;
|
||||
}
|
||||
|
|
@ -3384,12 +3385,12 @@ sub min {
|
|||
sub max {
|
||||
# Returns:
|
||||
# Maximum value of array
|
||||
my $max = shift;
|
||||
my @args = @_;
|
||||
for my $a (@args) {
|
||||
my $max;
|
||||
for (@_) {
|
||||
# Skip undefs
|
||||
defined $a or next;
|
||||
$max = ($max > $a) ? $max : $a;
|
||||
defined $_ or next;
|
||||
defined $max or do { $max = $_; next; }; # Set $_ to the first non-undef
|
||||
$max = ($max > $_) ? $max : $_;
|
||||
}
|
||||
return $max;
|
||||
}
|
||||
|
|
@ -3944,7 +3945,7 @@ sub get_command_line {
|
|||
|
||||
sub unget_command_line {
|
||||
# Returns: N/A
|
||||
push @Global::unget_next_command_line, @_;
|
||||
unshift @Global::unget_next_command_line, @_;
|
||||
}
|
||||
|
||||
sub more_arguments {
|
||||
|
|
|
|||
|
|
@ -5,5 +5,11 @@ ulimit -n 50
|
|||
(echo "sleep 3; echo begin"; seq 1 30 | parallel -kq echo "sleep 1; echo {}"; echo "echo end") \
|
||||
| parallel -k -j0
|
||||
|
||||
echo '### Test --keep-order'
|
||||
(seq 0 2) | parallel --keep-order -j100% -S 1/:,2/parallel@server2 -q perl -e 'sleep 1;print "job{}\n";exit({})'
|
||||
|
||||
echo '### Test --keeporder'
|
||||
(seq 0 2) | parallel --keeporder -j100% -S 1/:,2/parallel@server2 -q perl -e 'sleep 1;print "job{}\n";exit({})'
|
||||
|
||||
echo '### Test SIGTERM'
|
||||
(sleep 5; killall parallel -TERM) & seq 1 100 | parallel -k sleep 3';' echo
|
||||
|
|
|
|||
|
|
@ -1,22 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
PAR=parallel
|
||||
|
||||
# Test -I
|
||||
seq 1 10 | $PAR -k 'seq 1 {} | '$PAR' -k -I :: echo {} ::'
|
||||
seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'
|
||||
|
||||
seq 1 10 | $PAR -k 'seq 1 {} | '$PAR' -X -k -I :: echo a{} b::'
|
||||
seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::'
|
||||
|
||||
seq 1 10 | $PAR -k 'seq 1 {} | '$PAR' -m -k -I :: echo a{} b::'
|
||||
seq 1 10 | parallel -k 'seq 1 {} | parallel -m -k -I :: echo a{} b::'
|
||||
|
||||
seq 1 60000 | $PAR -I :: -m echo a::b::c | \
|
||||
seq 1 60000 | parallel -I :: -m echo a::b::c | \
|
||||
mop -q "|sort |md5sum" :par
|
||||
CHAR=$(cat ~/.mop/:par | wc -c)
|
||||
LINES=$(cat ~/.mop/:par | wc -l)
|
||||
echo -n "Chars per line ($CHAR/$LINES): "
|
||||
echo "$CHAR/$LINES" | bc
|
||||
|
||||
seq 1 60000 | $PAR -I :: -X echo a::b::c | \
|
||||
seq 1 60000 | parallel -I :: -X echo a::b::c | \
|
||||
mop -q "|sort |md5sum" :par
|
||||
CHAR=$(cat ~/.mop/:par | wc -c)
|
||||
LINES=$(cat ~/.mop/:par | wc -l)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,14 @@
|
|||
#!/bin/bash
|
||||
|
||||
PAR=parallel
|
||||
|
||||
echo '### Test --number-of-cpus'
|
||||
$PAR --number-of-cpus
|
||||
parallel --number-of-cpus
|
||||
|
||||
echo '### Test --number-of-cores'
|
||||
$PAR --number-of-cores
|
||||
parallel --number-of-cores
|
||||
|
||||
echo '### Test --use-cpus-instead-of-cores'
|
||||
(seq 1 4 | $PAR --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done &
|
||||
(seq 1 4 | $PAR -j100% sleep) && echo cores done &
|
||||
(seq 1 4 | parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done &
|
||||
(seq 1 4 | parallel -j100% sleep) && echo cores done &
|
||||
echo 'Cores should complete first on machines with less than 4 physical CPUs'
|
||||
wait
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,14 @@ begin
|
|||
29
|
||||
30
|
||||
end
|
||||
### Test --keep-order
|
||||
job0
|
||||
job1
|
||||
job2
|
||||
### Test --keeporder
|
||||
job0
|
||||
job1
|
||||
job2
|
||||
### Test SIGTERM
|
||||
1
|
||||
2
|
||||
|
|
|
|||
27
unittest/wanted-results/test32
Normal file
27
unittest/wanted-results/test32
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
### Test of --retries
|
||||
echo 1
|
||||
1
|
||||
echo 2
|
||||
2
|
||||
echo 3
|
||||
3
|
||||
echo 4
|
||||
4
|
||||
echo 5
|
||||
5
|
||||
echo 6
|
||||
6
|
||||
echo 7
|
||||
7
|
||||
echo 8
|
||||
8
|
||||
echo 9
|
||||
9
|
||||
echo 10
|
||||
10
|
||||
### Test of --retries - it should run 13 jobs in total
|
||||
13
|
||||
### Test of --retries - it should run 25 jobs in total
|
||||
25
|
||||
### Test of --retries - it should run 49 jobs in total
|
||||
49
|
||||
Loading…
Reference in a new issue