tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: warnings are now sent directly to stderr and not cached.

Browse source 
Test of --tmpdir works.
Testsuite passes.
This commit is contained in:
Ole Tange 2010-12-05 23:58:22 +01:00
parent f3b7af993a
commit ed6181773e
5 changed files with 105 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
src/parallel
View file

@ -605,8 +605,10 @@ to determine the number of CPU cores on remote computers).
=item B<--nice> I<niceness> (beta testing) =item B<--nice> I<niceness> (beta testing)
Run the command at this niceness. This is especially useful when Run the command at this niceness. For simple commands you can just add
running composed commands or commands on remote computers. B<nice> in front of the command. But if the command consists of more
sub commands (Like: ls|wc) then prepending B<nice> will not always
work. B<--nice> will make sure all sub commands are niced.
=item B<--interactive> =item B<--interactive>
@ -2528,6 +2530,7 @@ use Carp;
$::oodebug=0; $::oodebug=0;
$Global::original_sigterm = $SIG{TERM}; $Global::original_sigterm = $SIG{TERM};
$SIG{TERM} = sub {}; # Dummy until jobs really start $SIG{TERM} = sub {}; # Dummy until jobs really start
open $Global::original_stderr, ">&STDERR" or die "Can't dup STDERR: $!";
do_not_reap(); do_not_reap();
parse_options(); parse_options();
@ -3544,15 +3547,20 @@ sub parse_sshlogin {
if(not remote_hosts()) { if(not remote_hosts()) {
# There are no remote hosts # There are no remote hosts
if(defined @::opt_trc) { if(defined @::opt_trc) {
print STDERR "Warning: --trc ignored as there are no remote --sshlogin\n"; print $Global::original_stderr
"Warning: --trc ignored as there are no remote --sshlogin\n";
} elsif (defined $::opt_transfer) { } elsif (defined $::opt_transfer) {
print STDERR "Warning: --transfer ignored as there are no remote --sshlogin\n"; print $Global::original_stderr
"Warning: --transfer ignored as there are no remote --sshlogin\n";
} elsif (defined @::opt_return) { } elsif (defined @::opt_return) {
print STDERR "Warning: --return ignored as there are no remote --sshlogin\n"; print $Global::original_stderr
"Warning: --return ignored as there are no remote --sshlogin\n";
} elsif (defined $::opt_cleanup) { } elsif (defined $::opt_cleanup) {
print STDERR "Warning: --cleanup ignored as there are no remote --sshlogin\n"; print $Global::original_stderr
"Warning: --cleanup ignored as there are no remote --sshlogin\n";
} elsif (defined @::opt_basefile) { } elsif (defined @::opt_basefile) {
print STDERR "Warning: --basefile ignored as there are no remote --sshlogin\n"; print $Global::original_stderr
"Warning: --basefile ignored as there are no remote --sshlogin\n";
} }
} }
} }
@ -3612,14 +3620,14 @@ sub __SIGNAL_HANDLING__ {}
sub list_running_jobs { sub list_running_jobs {
# Returns: N/A # Returns: N/A
for my $v (values %Global::running) { for my $v (values %Global::running) {
print STDERR "$Global::progname: ",$v->replaced(),"\n"; print $Global::original_stderr "$Global::progname: ",$v->replaced(),"\n";
} }
} }
sub start_no_new_jobs { sub start_no_new_jobs {
# Returns: N/A # Returns: N/A
$SIG{TERM} = $Global::original_sigterm; $SIG{TERM} = $Global::original_sigterm;
print STDERR print $Global::original_stderr
("$Global::progname: SIGTERM received. No new jobs will be started.\n", ("$Global::progname: SIGTERM received. No new jobs will be started.\n",
"$Global::progname: Waiting for these ", scalar(keys %Global::running), "$Global::progname: Waiting for these ", scalar(keys %Global::running),
" jobs to finish. Send SIGTERM again to stop now.\n"); " jobs to finish. Send SIGTERM again to stop now.\n");
@ -3698,16 +3706,18 @@ sub reaper {
if($::opt_halt_on_error) { if($::opt_halt_on_error) {
if($::opt_halt_on_error == 1) { if($::opt_halt_on_error == 1) {
# If halt on error == 1 we should gracefully exit # If halt on error == 1 we should gracefully exit
print STDERR ("$Global::progname: Starting no more jobs. ", print $Global::original_stderr
"Waiting for ", scalar(keys %Global::running), ("$Global::progname: Starting no more jobs. ",
" jobs to finish. This job failed:\n", "Waiting for ", scalar(keys %Global::running),
$job->replaced(),"\n"); " jobs to finish. This job failed:\n",
$job->replaced(),"\n");
$Global::start_no_new_jobs++; $Global::start_no_new_jobs++;
$Global::halt_on_error_exitstatus = $job->exitstatus(); $Global::halt_on_error_exitstatus = $job->exitstatus();
} elsif($::opt_halt_on_error == 2) { } elsif($::opt_halt_on_error == 2) {
# If halt on error == 2 we should exit immediately # If halt on error == 2 we should exit immediately
print STDERR ("$Global::progname: This job failed:\n", print $Global::original_stderr
$job->replaced(),"\n"); ("$Global::progname: This job failed:\n",
$job->replaced(),"\n");
exit ($job->exitstatus()); exit ($job->exitstatus());
} }
} }
@ -3835,7 +3845,7 @@ sub my_dump {
if ($@) { if ($@) {
my $err = "Neither Data::Dump nor Data::Dumper is installed\n". my $err = "Neither Data::Dump nor Data::Dumper is installed\n".
"Not dumping output\n"; "Not dumping output\n";
print STDERR $err; print $Global::original_stderr $err;
return $err; return $err;
} else { } else {
return Dumper(@dump_this); return Dumper(@dump_this);
@ -4028,11 +4038,11 @@ sub compute_max_loadavg {
close IN; close IN;
$load = $self->compute_max_loadavg($opt_P_file); $load = $self->compute_max_loadavg($opt_P_file);
} else { } else {
print STDERR "Cannot open $loadspec\n"; print $Global::original_stderr "Cannot open $loadspec\n";
exit(255); exit(255);
} }
} else { } else {
print STDERR "Parsing of --load failed\n"; print $Global::original_stderr "Parsing of --load failed\n";
::die_usage(); ::die_usage();
} }
if($load < 0.01) { if($load < 0.01) {
@ -4132,19 +4142,22 @@ sub processes_available_by_system_limit {
# It took more than 2 second to fork ten processes. # It took more than 2 second to fork ten processes.
# Give the user a warning. He can press Ctrl-C if this # Give the user a warning. He can press Ctrl-C if this
# sucks. # sucks.
print STDERR ("Warning: Starting 10 extra processes takes > 2 sec.\n", print $Global::original_stderr
"Consider adjusting -j. Press CTRL-C to stop.\n"); ("Warning: Starting 10 extra processes takes > 2 sec.\n",
"Consider adjusting -j. Press CTRL-C to stop.\n");
$slow_spawining_warning_printed = 1; $slow_spawining_warning_printed = 1;
} }
} }
if($system_limit < $wanted_processes and not $more_filehandles) { if($system_limit < $wanted_processes and not $more_filehandles) {
print STDERR ("Warning: Only enough filehandles to run ", print $Global::original_stderr
$system_limit, " jobs in parallel. ", ("Warning: Only enough filehandles to run ",
"Raising ulimit -n may help\n"); $system_limit, " jobs in parallel. ",
"Raising ulimit -n may help\n");
} }
if($system_limit < $wanted_processes and $max_system_proc_reached) { if($system_limit < $wanted_processes and $max_system_proc_reached) {
print STDERR ("Warning: Only enough available processes to run ", print $Global::original_stderr
$system_limit, " jobs in parallel.\n"); ("Warning: Only enough available processes to run ",
$system_limit, " jobs in parallel.\n");
} }
# Cleanup: Close the files # Cleanup: Close the files
for (values %fh) { close $_ } for (values %fh) { close $_ }
@ -4178,7 +4191,7 @@ sub simultaneous_sshlogin_limit {
$self->simultaneous_sshlogin($wanted_processes)); $self->simultaneous_sshlogin($wanted_processes));
if($ssh_limit < $wanted_processes) { if($ssh_limit < $wanted_processes) {
my $serverlogin = $self->serverlogin(); my $serverlogin = $self->serverlogin();
print STDERR print $Global::original_stderr
("Warning: ssh to $serverlogin only allows ", ("Warning: ssh to $serverlogin only allows ",
"for $ssh_limit simultaneous logins.\n", "for $ssh_limit simultaneous logins.\n",
"You may raise this by changing ", "You may raise this by changing ",
@ -4252,11 +4265,11 @@ sub user_requested_processes {
close IN; close IN;
$processes = $self->user_requested_processes($opt_P_file); $processes = $self->user_requested_processes($opt_P_file);
} else { } else {
print STDERR "Cannot open $opt_P\n"; print $Global::original_stderr "Cannot open $opt_P\n";
exit(255); exit(255);
} }
} else { } else {
print STDERR "Parsing of --jobs/-j/--max-procs/-P failed\n"; print $Global::original_stderr "Parsing of --jobs/-j/--max-procs/-P failed\n";
::die_usage(); ::die_usage();
} }
if($processes < 1) { if($processes < 1) {
@ -4289,8 +4302,9 @@ sub ncpus {
if($ncpu =~ /^[0-9]+$/) { if($ncpu =~ /^[0-9]+$/) {
$self->{'ncpus'} = $ncpu; $self->{'ncpus'} = $ncpu;
} else { } else {
print STDERR ("Warning: Could not figure out ", print $Global::original_stderr
"number of cpus on $serverlogin. Using 1\n"); ("Warning: Could not figure out ",
"number of cpus on $serverlogin. Using 1\n");
$self->{'ncpus'} = 1; $self->{'ncpus'} = 1;
} }
} }
@ -4792,7 +4806,8 @@ sub sshtransfer {
$remote_workdir eq "." ? "true" : "ssh $serverlogin mkdir -p $rsync_destdir"; $remote_workdir eq "." ? "true" : "ssh $serverlogin mkdir -p $rsync_destdir";
$pre .= "$mkremote_workdir; rsync $rsync_opt ".::shell_quote_scalar($file)." $serverlogin:$rsync_destdir;"; $pre .= "$mkremote_workdir; rsync $rsync_opt ".::shell_quote_scalar($file)." $serverlogin:$rsync_destdir;";
} else { } else {
print STDERR "Warning: $file is not readable and will not be transferred\n"; print $Global::original_stderr
"Warning: $file is not readable and will not be transferred\n";
} }
} }
return $pre; return $pre;

3
testsuite/tests-to-run/test41.sh
View file

@ -5,4 +5,5 @@ SERVER2=parallel-server2
echo '### Test -M' echo '### Test -M'
seq 1 20 | parallel -k -M -S 9/$SERVER1,9/parallel@$SERVER2 echo seq 1 30 | parallel -j5 -k -M -S $SERVER1,parallel@$SERVER2 echo
seq 1 30 | parallel -j10 -k -M -S $SERVER1,parallel@$SERVER2 echo

32
testsuite/wanted-results/test22
View file

@ -11,9 +11,9 @@ sleep 2;false
1 1
sh: non_exist: command not found sh: non_exist: command not found
2 2
sh: non_exist: command not found
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
sleep 2;false sleep 2;false
sh: non_exist: command not found
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
sleep 4; non_exist sleep 4; non_exist
127 127
@ -23,31 +23,31 @@ sleep 2;false
### Test last dying print --halt-on-error ### Test last dying print --halt-on-error
0 0
1 1
parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
2 2
parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2
3 3
parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
4 4
parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4
5 5
parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5
6 6
parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6
7 7
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
8 8
0 0
9
parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2
parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4
parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5
parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8
9
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9
9 9

2
testsuite/wanted-results/test33
View file

@ -1,11 +1,11 @@
### Test of -j filename ### Test of -j filename
Parsing of --jobs/-j/--max-procs/-P failed
Usage: Usage:
parallel [options] [command [arguments]] < list_of_arguments parallel [options] [command [arguments]] < list_of_arguments
parallel [options] [command [arguments]] ::: arguments parallel [options] [command [arguments]] ::: arguments
parallel [options] [command [arguments]] :::: argfile(s) parallel [options] [command [arguments]] :::: argfile(s)
See 'man parallel' for the options See 'man parallel' for the options
Parsing of --jobs/-j/--max-procs/-P failed
### Test of -j filename ### Test of -j filename
sleep 0.7 sleep 0.7
sleep 0.8 sleep 0.8

40
testsuite/wanted-results/test41
View file

@ -19,3 +19,43 @@
18 18
19 19
20 20
21
22
23
24
25
26
27
28
29
30
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30