parallel: --memsuspend working.

This commit is contained in:
Ole Tange 2021-01-20 01:15:16 +01:00
parent b210039626
commit e81a0eba05
11 changed files with 122 additions and 181 deletions

View file

@ -1,5 +1,8 @@
Quote of the month:
GNU Parallel is great
-- Newton's Flaming Laser Sword @swordgoesfwoosh@twitter
Try GNU parallel it's awesome, and exactly the thing you are looking for. It allows you to set number of processes running among many things. I use it a lot, and can't recommend it enough.
-- mapettheone@reddit

View file

@ -192,7 +192,7 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20210122 ('Ask/Capitol Riots') released <<[stable]>>
Subject: GNU Parallel 20210122 ('Capitol Riots') released <<[stable]>>
GNU Parallel 20210122 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
@ -209,7 +209,7 @@ New in this release:
* $PARALLEL_ARGHOSTGROUPS and the replacement string {agrp} will give the hostgroup given on the argument when using --hostgroup.
* Handy time functions for {= =}: yyyy_mm_dd_hh_mm_ss() yyyy_mm_dd_hh_mm() yyyy_mm_dd() yyyymmddhhmmss() yyyymmddhhmm() yyyymmdd()
* Handy time functions for {= =}: yyyy_mm_dd_hh_mm_ss() yyyy_mm_dd_hh_mm() yyyy_mm_dd() yyyymmddhhmmss() yyyymmddhhmm() yyyymmdd() hash($str)
<<>>
@ -224,6 +224,8 @@ https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c
https://canvas.stanford.edu/courses/133091
https://www.youtube.com/watch?v=-tX0bFAYAxM
<<>>
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

View file

@ -2151,7 +2151,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20201223;
$Global::version = 20210113;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -4661,8 +4661,17 @@ sub save_original_signal_handler() {
$SIG{CONT} = sub {
# Set $SIG{TSTP} again (it is undef'ed in sigtstp() )
$SIG{TSTP} = \&sigtstp;
# Send continue signal to all children process groups
kill "CONT", map { -$_ } keys %Global::running;
for my $job (values %Global::running) {
if($job->suspended()) {
# Force jobs to suspend, if they are marked as suspended.
# --memsupspend can suspend a job that will be resumed
# if the user presses CTRL-Z followed by `fg`.
$job->suspend();
} else {
# Resume the rest of the jobs
$job->resume();
}
}
};
}
@ -4791,7 +4800,7 @@ sub reaper() {
::status_no_nl("\r",$progress{'status'});
}
debug("run", "done ");
debug("run", "jobdone \n");
return $stiff;
}
@ -5888,7 +5897,7 @@ sub reap_usleep() {
kill_youngster_if_not_enough_mem($opt::memfree*0.5);
}
if($opt::memsuspend) {
kill_youngster_if_not_enough_mem($opt::memsuspend*0.5);
suspend_young_if_not_enough_mem($opt::memsuspend);
}
if($opt::limit) {
kill_youngest_if_over_limit();
@ -5953,6 +5962,58 @@ sub kill_youngest_if_over_limit() {
}
}
sub suspend_young_if_not_enough_mem() {
# Check each $sshlogin if there is enough mem.
# If less than $limit free mem: suspend some of the young children
# Else: Resume all jobs
# Uses:
# %Global::running
my $limit = shift;
my %jobs_of;
my @sshlogins;
for my $job (values %Global::running) {
if(not $jobs_of{$job->sshlogin()}) {
push @sshlogins, $job->sshlogin();
}
push @{$jobs_of{$job->sshlogin()}}, $job;
}
for my $sshlogin (@sshlogins) {
my $free = $sshlogin->memfree();
if($free < 2*$limit) {
# Suspend all jobs (resume some of them later)
map { $_->suspended() or $_->suspend(); } @{$jobs_of{$sshlogin}};
my @jobs = sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}};
# how many should be running?
# limit*1 => 1;
# limit*1.5 => 2;
# limit*1.75 => 4;
# free < limit*(2-1/2^n);
# =>
# 1/(2-free/limit) < 2^n;
my $run = int(1/(2-$free/$limit));
$run = ::min($run,$#jobs);
# Resume the oldest running
for my $job ((sort { $a->seq() <=> $b->seq() } @jobs)[0..$run]) {
::debug("mem","\nResume ",$run+1, " jobs. Seq ",
$job->seq(), " resumed ",
$sshlogin->memfree()," < ",2*$limit);
$job->resume();
}
} else {
for my $job (@{$jobs_of{$sshlogin}}) {
if($job->suspended()) {
$job->resume();
::debug("mem","\nResume ",$#{$jobs_of{$sshlogin}}+1, " jobs. Seq ",
$job->seq(), " resumed ",
$sshlogin->memfree()," > ",2*$limit);
last;
}
}
}
}
}
sub kill_youngster_if_not_enough_mem() {
# Check each $sshlogin if there is enough mem.
# If less than 50% enough free mem: kill off the youngest child
@ -5977,11 +6038,7 @@ sub kill_youngster_if_not_enough_mem() {
@{$jobs_of{$sshlogin}}));
::debug("mem","\n", $job->seq(), "killed ",
$sshlogin->memfree()," < ",$limit);
if($opt::memsuspend) {
$job->suspend();
} else {
$job->kill();
}
$job->kill();
$sshlogin->memfree_recompute();
} else {
last;
@ -8684,8 +8741,6 @@ sub suspend($) {
my @pgrps = map { -$_ } $self->pid();
kill "STOP", @pgrps;
$self->set_suspended(1);
# push job onto start stack
$Global::JobQueue->unget($self);
}
sub set_suspended($$) {
@ -9723,11 +9778,6 @@ sub start($) {
}
my $job = shift;
if($job->suspended()) {
# Job is kill -STOP'ped: Restart it.
$job->resume();
return $job;
}
# Get the shell command to be executed (possibly with ssh infront).
my $command = $job->wrapped();
my $pid;
@ -12379,6 +12429,11 @@ sub pQ($) {
return $ret;
}
sub hash($) {
$Global::use{"DBI"} ||= eval "use B; 1;";
B::hash(@_);
}
sub total_jobs() {
return $Global::JobQueue->total_jobs();
}
@ -12402,6 +12457,24 @@ sub total_jobs() {
# Do not quote this arg
$Global::unquote_arg = 1;
}
sub yyyy_mm_dd_hh_mm_ss() {
::strftime("%Y-%m-%dT%H:%M:%S", localtime(time()));
}
sub yyyy_mm_dd_hh_mm() {
::strftime("%Y-%m-%dT%H:%M", localtime(time()));
}
sub yyyy_mm_dd() {
::strftime("%Y-%m-%d", localtime(time()));
}
sub yyyymmddhhmmss() {
::strftime("%Y%m%d%H%M%S", localtime(time()));
}
sub yyyymmddhhmm() {
::strftime("%Y%m%d%H%M", localtime(time()));
}
sub yyyymmdd() {
::strftime("%Y%m%d", localtime(time()));
}
sub replace($$$$) {
# Calculates the corresponding value for a given perl expression
@ -13448,8 +13521,8 @@ sub main() {
while(start_more_jobs()) {}
spreadstdin();
} else {
# Reap one - start one
while(reaper() + start_more_jobs()) {}
# Reap the finished jobs - start more
while(reapers() + start_more_jobs()) {}
}
::debug("init", "Start draining\n");
drain_job_queue(@command);

View file

@ -2877,7 +2877,8 @@ with replacement strings. Such as:
that can be used like:
parallel --header : --tmpl my.tmpl {#}.t myprog {#}.t ::: x 1 2 3 ::: y 1 2 3
parallel --header : --tmpl my.tmpl {#}.t myprog {#}.t \
::: x 1 2 3 ::: y 1 2 3
Filtering may also be supported as:

View file

@ -941,6 +941,20 @@ par_json() {
perl -pe 's/\d/0/g'
}
par_hash_and_time_functions() {
echo '### Functions for replacement string'
parallel echo '{= $_=join(" ",
yyyy_mm_dd_hh_mm_ss(),
yyyy_mm_dd_hh_mm(),
yyyy_mm_dd(),
yyyymmddhhmmss(),
yyyymmddhhmm(),
yyyymmdd()) =}' ::: 1 |
perl -pe 's/\d/9/g'
parallel echo '{= $_=hash($_) =}' ::: 1 |
perl -pe 's/[a-f0-9]/X/g'
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |

View file

@ -31,11 +31,6 @@ par_10000_m_X() {
seq 10000 | perl -pe 's/$/.gif/' |
parallel -j1 -km echo a{}b{.}c{.} |
parallel -k --pipe --tee ::: wc md5sum
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | wc -l
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c | wc -l
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b | wc -l
}
par_10000_5_rpl_X() {
@ -49,7 +44,6 @@ par_10000_5_rpl_X() {
par_X_I_meta() {
echo '### Test -X -I with shell meta chars'
seq 10000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l
seq 10000 | parallel -j1 -I '<>' -X echo 'a<>b<>c<>' | wc -l
seq 10000 | parallel -j1 -I '<' -X echo 'a<b<c<' | wc -l

View file

@ -74,8 +74,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
# Timings are often off
s/^(\d)$/9/;
s/^(\d\d)$/99/;
# Sometime these vars are not present
s/^PAM_KWALLET5*_LOGIN$//;
# Remove variable names - they vary
s/^[A-Z][A-Z0-9_]*\s$//;
# Fails often due to race
s/cat: input_file: No such file or directory\n//;
s{rsync: link_stat ".*/home/parallel/input_file.out" .*\n}{};
@ -93,11 +93,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
s/doi.*=.*//;
s/url.*= .*doi.org.*//;
s/.Feel free to use .nocite.*//;
' |
perl -ne '/GTK2_RC_FILES/ and next;
/GTK_RC_FILES/ and next;
print' |
uniq
' | uniq
# 3+3 .par files (from --files), 1 .tms-file from tmux attach
find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l
find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | parallel rm

View file

@ -193,12 +193,13 @@ par_halt_one_job 1
par_halt_one_job 2
par_halt_one_job parallel: This job failed:
par_halt_one_job echo 2;exit 1
par_halt_one_job parallel: Starting no more jobs. Waiting for 0 jobs to finish.
par_halt_one_job should run 1 = job 1
par_halt_one_job 1
par_halt_one_job parallel: This job failed:
par_halt_one_job echo 1;exit 1
par_halt_one_job parallel: Starting no more jobs. Waiting for 0 jobs to finish.
par_hash_and_time_functions ### Functions for replacement string
par_hash_and_time_functions 9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99999999999999 999999999999 99999999
par_hash_and_time_functions XxXXXXXXXX
par_help ### Test --help and -h: Help output (just check we get the same amount of lines)
par_help Output from -h and --help
par_help 37

View file

@ -7,11 +7,6 @@ par_10000_5_rpl_X 1
par_10000_m_X ### Test -m with 10000 args
par_10000_m_X 2 29996 186684
par_10000_m_X c606aec1723ee5cc15f2a1b95d83d3cf -
par_10000_m_X 3
par_10000_m_X 2
par_10000_m_X 2
par_10000_m_X 2
par_10000_m_X 1
par_X_I_meta ### Test -X -I with shell meta chars
par_X_I_meta 2
par_X_I_meta 2

View file

@ -3,6 +3,7 @@ echo '### --env _'
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR test
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR test
echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
### --env _ with explicit mentioning of normally ignored var $DEBEMAIL
fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test

View file

@ -28,17 +28,8 @@ sleep .3
/usr/bin/bash: -c: line 0: ` perl -e 'for(1..10){print "$_\n"}') > num_%header'
perl -e 'print "HHHHAAABBBCCC"' > fixedlen
parallel echo ::: A B C
A
B
C
parallel -a abc-file echo
A
B
C
cat abc-file | parallel echo
A
B
C
parallel echo ::: A B C ::: D E F
A D
A E
@ -143,16 +134,8 @@ C D
C E
C F
parallel -d _ echo :::: abc_-file
A
B
C
parallel -d '\0' echo :::: abc0-file
A
B
C
parallel -E stop echo ::: A B stop C D
A
B
(echo 1; echo; echo 2) | parallel --no-run-if-empty echo
9
parallel ::: ls 'echo foo' pwd
@ -185,9 +168,7 @@ A/B
parallel echo {/} ::: A/B.C
B.C
parallel echo {//} ::: A/B.C
A
parallel echo {/.} ::: A/B.C
B
parallel echo {#} ::: A B C
9
parallel -j 2 echo {%} ::: A B C
@ -199,9 +180,7 @@ A/B
parallel --basenamereplace ,, echo ,, ::: A/B.C
B.C
parallel --dirnamereplace ,, echo ,, ::: A/B.C
A
parallel --basenameextensionreplace ,, echo ,, ::: A/B.C
B
parallel --seqreplace ,, echo ,, ::: A B C
9
parallel -j2 --slotreplace ,, echo ,, ::: A B C
@ -365,7 +344,6 @@ G H
parallel -N0 echo foo ::: 1 2 3
foo
perl -e 'print "@ARGV\n"' A
A
parallel perl -e 'print "@ARGV\n"' ::: This wont work
Warning: Input is read from the terminal. You either know what you
Warning: are doing (in which case: YOU ARE AWESOME!) or you forgot
@ -420,9 +398,6 @@ echo A
echo B
echo C
parallel --verbose echo {} ::: A B C
A
B
C
echo A
echo B
echo C
@ -468,9 +443,6 @@ echo C
parallel --tmpdir /var/tmp --files echo ::: A B C
/var/tmp/parXXXXX.par
parallel --results outdir echo ::: A B C
A
B
C
outdir/1/A/seq
outdir/1/A/stderr
outdir/1/A/stdout
@ -804,87 +776,6 @@ in my_func baz
cat ~/.parallel/ignored_vars|sort
BASH_FUNC_run_once%%
BASH_FUNC_run_test%%
COLORFGBG
COLORTERM
DBUS_SESSION_BUS_ADDRESS
DEBEMAIL
DEBFULLNAME
DESKTOP_SESSION
DISPLAY
EDITOR
GOPATH
GPG_AGENT_INFO
HISTCONTROL
HOME
KDE_FULL_SESSION
KDE_SESSION_UID
KDE_SESSION_VERSION
KONSOLE_DBUS_SERVICE
KONSOLE_DBUS_SESSION
KONSOLE_DBUS_WINDOW
KONSOLE_VERSION
LANG
LANGUAGE
LC_ALL
LESS
LESSCLOSE
LESSOPEN
LOGNAME
LS_COLORS
MAKEFLAGS
MAKELEVEL
MAKE_TERMERR
MAKE_TERMOUT
MFLAGS
OLDPWD
ORACLE_HOME
ORACLE_SID
PARALLEL
PARALLEL_HOME
PARALLEL_JOBSLOT
PARALLEL_PID
PARALLEL_RSYNC_OPTS
PARALLEL_SEQ
PARALLEL_SSHHOST
PARALLEL_SSHLOGIN
PARALLEL_TMP
PATH
PERL_MB_OPT
PERL_MM_OPT
PROFILEHOME
PWD
QT_ACCESSIBILITY
QT_AUTO_SCREEN_SCALE_FACTOR
SERVER1
SERVER2
SESSION_MANAGER
SHELL
SHELL_SESSION_ID
SHLVL
SSH_AGENT_PID
SSH_AUTH_SOCK
TERM
TIMEOUT
TMPDIR
TRIES
USER
VISUAL
WINDOWID
XAUTHORITY
XCURSOR_SIZE
XCURSOR_THEME
XDG_CONFIG_DIRS
XDG_CURRENT_DESKTOP
XDG_DATA_DIRS
XDG_RUNTIME_DIR
XDG_SEAT
XDG_SEAT_PATH
XDG_SESSION_CLASS
XDG_SESSION_DESKTOP
XDG_SESSION_ID
XDG_SESSION_PATH
XDG_SESSION_TYPE
XDG_VTNR
_
mysqlrootpass
# The function is only copied if using Bash
@ -1133,61 +1024,35 @@ parallel: Error: echo is not a valid DBURL
cat fixedlen | parallel --pipe --header .{4} --block 3 --recend '' \
'echo start; cat; echo'
start
HHHHAAA
start
HHHHBBB
start
HHHHCCC
echo /foo, bar/, /baz, qux/, | \
parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
JOB1
/foo, END
JOB2
bar/, END
JOB3
/baz, END
JOB4
qux/,
END
echo /foo, bar/, /baz, qux/, | \
parallel -kN1 --recstart / --pipe echo JOB{#}\;cat\;echo END
JOB1
/foo, barEND
JOB2
/, END
JOB3
/baz, quxEND
JOB4
/,
END
echo /foo, bar/, /baz, qux/, | \
parallel -kN1 --recend ', ' --recstart / --pipe \
echo JOB{#}\;cat\;echo END
JOB1
/foo, bar/, END
JOB2
/baz, qux/,
END
echo foo,bar,_baz,__qux, | \
parallel -kN1 --regexp --recend ,_+ --pipe \
echo JOB{#}\;cat\;echo END
JOB1
foo,bar,_END
JOB2
baz,__END
JOB3
qux,
END
echo foo,bar,_baz,__qux, | \
parallel -kN1 --rrs --regexp --recend ,_+ --pipe \
echo JOB{#}\;cat\;echo END
JOB1
foo,barEND
JOB2
bazEND
JOB3
qux,
END
cat num_%header | \
parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat
cat: num_%header: No such file or directory
@ -1384,7 +1249,6 @@ When using programs that use GNU Parallel to process data for publication
please cite as described in 'parallel --citation'.
parallel --minversion VERSION && \
echo Your version is at least VERSION.
VERSION
Your version is at least VERSION.
parallel --citation
Academic tradition requires you to cite works you base your article on.
@ -1416,9 +1280,6 @@ mentioned in the release notes of next version of GNU Parallel.
echo '--nice 17' > ~/.parallel/nicetimeout
echo '--timeout 300%' >> ~/.parallel/nicetimeout
parallel --profile nicetimeout echo ::: A B C
A
B
C
echo '-vv --dry-run' > ~/.parallel/dryverbose
parallel --profile dryverbose --profile nicetimeout echo ::: A B C
echo A