From 0ab644d156137beadc8d927047606bb689f5ea61 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 1 Jun 2015 01:11:32 +0200 Subject: [PATCH] parallel: kill process groups. Fails for --tty jobs. --- doc/release_new_version | 2 +- src/parallel | 45 ++++++++++++++-------- testsuite/wanted-results/parallel-tutorial | 2 +- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 28ba7cd7..eabdf77c 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -250,7 +250,7 @@ New in this release: * GNU Parallel is used in: https://github.com/d2207197/local-mapreduce - +* Job ad asking for GNU Parallel experience: http://searchjobs.intel.com/gdansk-pol/software-validation-engineer/63A06826DAF24797AB414DC146201C2E/job/ * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index b5555a95..787422c7 100755 --- a/src/parallel +++ b/src/parallel @@ -1076,7 +1076,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150524; + $Global::version = 20150531; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3178,6 +3178,8 @@ sub reaper { $job->print(); } if($job->should_we_halt() eq "now") { + # Kill children + ::kill_sleep_seq($job->pid()); ::killall(); ::wait_and_exit($Global::halt_exitstatus); } @@ -3222,17 +3224,20 @@ sub killall { $Global::start_no_new_jobs ||= 1; $Global::killall ||= 1; - # pids of the all children and (grand*)children - # before we start the blood bath - my @family_pids = family_pids(keys %Global::running); - my @pids = @family_pids; + kill_sleep_seq(keys %Global::running); +} + +sub kill_sleep_seq { # Send jobs TERM,TERM,KILL - my @term_seq = ("TERM",200,"TERM",200,"KILL",200); + # processgroups (-$pid) + my @pgrps = map { -$_ } @_; + my @term_seq = ("TERM",200,"TERM",100,"TERM",50,"KILL",25); while(@term_seq) { - @pids = kill_sleep(shift @term_seq, shift @term_seq, @pids); + @pgrps = kill_sleep(shift @term_seq, shift @term_seq, @pgrps); } } + sub kill_sleep { my ($signal, $sleep_max, @pids) = @_; ::debug("kill","kill_sleep $signal ",(join " ",sort @pids),"\n"); @@ -3261,7 +3266,7 @@ sub kill_sleep { $sleep *= 1.1; ::usleep($sleep); $sleepsum += $sleep; - # Remove (grand)*children that are dead + # Remove dead children @pids = grep { kill( 0, $_) } @pids; } return @pids; @@ -3750,7 +3755,7 @@ sub which { # ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh # posh rbash rc rush rzsh sash sh static-sh tcsh yash zsh my @shells = (qw(ash bash csh dash fdsh fish fizsh ksh - ksh93 mksh pdksh posh rbash rc rush rzsh + ksh93 lksh mksh pdksh posh rbash rc rush rzsh sash sh static-sh tcsh yash zsh -sh -csh), '-sh (sh)' # sh on FreeBSD ); @@ -6247,6 +6252,12 @@ sub timedout { } sub kill { + my $self = shift; + $self->set_exitstatus(-1); + ::kill_sleep_seq($self->pid()); +} + +sub _kill { # Kill the job. # Send the signals to (grand)*children and pid. # If no signals: TERM TERM KILL @@ -6265,9 +6276,9 @@ sub kill { for my $signal (@send_signals) { my $alive = 0; for my $pid (@family_pids) { - if(kill 0, $pid) { + if(CORE::kill 0, $pid) { # The job still running - kill $signal, $pid; + CORE::kill $signal, $pid; $alive = 1; ::debug("run","$pid is alive\n"); } @@ -6278,7 +6289,7 @@ sub kill { if($signal eq "TERM" and $alive) { # Wait up to 200 ms between TERMs - but only if any pids are alive my $sleep = 1; - for (my $sleepsum = 0; kill 0, $family_pids[0] and $sleepsum < 200; + for (my $sleepsum = 0; CORE::kill 0, $family_pids[0] and $sleepsum < 200; $sleepsum += $sleep) { $sleep = ::reap_usleep($sleep); } @@ -7077,6 +7088,8 @@ sub start { if($opt::dryrun) { $command = "true"; } + # Each child gets its own process group to make it safe to killall + my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_TMP'} = ::tmpname("par"); @@ -7086,7 +7099,7 @@ sub start { my ($stdin_fh); # The eval is needed to catch exception from open3 eval { - $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) || ::die_bug("open3-pipe"); 1; }; @@ -7098,7 +7111,7 @@ sub start { *IN = *STDIN; # The eval is needed to catch exception from open3 eval { - $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) || ::die_bug("open3-a"); 1; }; @@ -7111,7 +7124,7 @@ sub start { *IN = $devtty_fh; # The eval is needed to catch exception from open3 eval { - $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) || ::die_bug("open3-/dev/tty"); $Global::tty_taken = $pid; close $devtty_fh; @@ -7120,7 +7133,7 @@ sub start { } else { # The eval is needed to catch exception from open3 eval { - $pid = ::open3(::gensym, ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + $pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command) || ::die_bug("open3-gensym"); 1; }; diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 0d1be3da..71601c89 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -733,7 +733,7 @@ _ /bin/bash: my_func2: command not found parallel -vv -S $SERVER1 echo ::: bar -ssh parallel@lo exec perl\ -e\ \\\$ENV\\\{\\\"PARALLEL_PID\\\"\\\}\\\=\\\"000000\\\"\\\;\\\$ENV\\\{\\\"PARALLEL_SEQ\\\"\\\}\\\=\\\"1\\\"\\\;\\\$bashfunc\\\ \\\=\\\ \\\"\\\"\\\;@ARGV\\\=\\\"echo\\\ bar\\\"\\\;\\\$shell\\\=\\\"\\\$ENV\\\{SHELL\\\}\\\"\\\;\\\$tmpdir\\\=\\\"/tmp/parallel-tutorial\\\"\\\;do\\\{\\\$ENV\\\{PARALLEL_TMP\\\}\\\=\\\$tmpdir.\\\"/par\\\".join\\\"\\\",map\\\{\\\(0..9,\\\"a\\\"..\\\"z\\\",\\\"A\\\"..\\\"Z\\\"\\\)\\\[rand\\\(62\\\)\\\]\\\}\\\(1..5\\\)\\\;\\\}while\\\(-e\\\$ENV\\\{PARALLEL_TMP\\\}\\\)\\\;\\\$SIG\\\{CHLD\\\}\\\=sub\\\{\\\$done\\\=1\\\;\\\}\\\;\\\$pid\\\=fork\\\;unless\\\(\\\$pid\\\)\\\{setpgrp\\\;exec\\\$shell,\\\"-c\\\",\\\(\\\$bashfunc.\\\"@ARGV\\\"\\\)\\\;die\\\"exec:\\\$\\\!\\\\n\\\"\\\;\\\}do\\\{\\\$s\\\=\\\$s\\\<1\\\?0.001+\\\$s\\\*1.03:\\\$s\\\;select\\\(undef,undef,undef,\\\$s\\\)\\\;\\\}until\\\(\\\$done\\\|\\\|getppid\\\=\\\=1\\\)\\\;kill\\\(SIGHUP,-\\\$\\\{pid\\\}\\\)unless\\\$done\\\;wait\\\;exit\\\(\\\$\\\?\\\&127\\\?128+\\\(\\\$\\\?\\\&127\\\):1+\\\$\\\?\\\>\\\>8\\\); +ssh parallel@lo exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my($in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;eval$eval;'\' BASE64; bar my_func3() { echo in my_func $1 > $1.out