parallel: --memsuspend working.

doc/haikus

@@ -1,5 +1,8 @@
 Quote of the month:
 
+  GNU Parallel is great
+    -- Newton's Flaming Laser Sword @swordgoesfwoosh@twitter
+
   Try GNU parallel it's awesome, and exactly the thing you are looking for. It allows you to set number of processes running among many things. I use it a lot, and can't recommend it enough.
     -- mapettheone@reddit
 

doc/release_new_version

@@ -192,7 +192,7 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20210122 ('Ask/Capitol Riots') released <<[stable]>>
+Subject: GNU Parallel 20210122 ('Capitol Riots') released <<[stable]>>
 
 GNU Parallel 20210122 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
@@ -209,7 +209,7 @@ New in this release:
 
 * $PARALLEL_ARGHOSTGROUPS and the replacement string {agrp} will give the hostgroup given on the argument when using --hostgroup.
 
-* Handy time functions for {= =}: yyyy_mm_dd_hh_mm_ss() yyyy_mm_dd_hh_mm() yyyy_mm_dd() yyyymmddhhmmss() yyyymmddhhmm() yyyymmdd()
+* Handy time functions for {= =}: yyyy_mm_dd_hh_mm_ss() yyyy_mm_dd_hh_mm() yyyy_mm_dd() yyyymmddhhmmss() yyyymmddhhmm() yyyymmdd() hash($str)
 
 <<>>
 
@@ -224,6 +224,8 @@ https://qiita.com/hana_shin/items/53c3c78525c9c758ae7c
 
 https://canvas.stanford.edu/courses/133091
 
+https://www.youtube.com/watch?v=-tX0bFAYAxM
+
 <<>>
 
 Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

src/parallel

@@ -2151,7 +2151,7 @@ sub check_invalid_option_combinations() {
 
 sub init_globals() {
     # Defaults:
-    $Global::version = 20201223;
+    $Global::version = 20210113;
     $Global::progname = 'parallel';
     $::name = "GNU Parallel";
     $Global::infinity = 2**31;
@@ -4661,8 +4661,17 @@ sub save_original_signal_handler() {
     $SIG{CONT} = sub {
 	# Set $SIG{TSTP} again (it is undef'ed in sigtstp() )
 	$SIG{TSTP} = \&sigtstp;
-	# Send continue signal to all children process groups
+	for my $job (values %Global::running) {
-	kill "CONT", map { -$_ } keys %Global::running;
+	    if($job->suspended()) {
+		# Force jobs to suspend, if they are marked as suspended.
+		# --memsupspend can suspend a job that will be resumed
+		# if the user presses CTRL-Z followed by `fg`.
+		$job->suspend();
+	    } else {
+		# Resume the rest of the jobs
+		$job->resume();
+	    }
+	}
     };
 }
 
@@ -4791,7 +4800,7 @@ sub reaper() {
 	::status_no_nl("\r",$progress{'status'});
     }
 
-    debug("run", "done ");
+    debug("run", "jobdone \n");
     return $stiff;
 }
 
@@ -5888,7 +5897,7 @@ sub reap_usleep() {
 	    kill_youngster_if_not_enough_mem($opt::memfree*0.5);
 	}
 	if($opt::memsuspend) {
-	    kill_youngster_if_not_enough_mem($opt::memsuspend*0.5);
+	    suspend_young_if_not_enough_mem($opt::memsuspend);
 	}
 	if($opt::limit) {
 	    kill_youngest_if_over_limit();
@@ -5953,6 +5962,58 @@ sub kill_youngest_if_over_limit() {
     }
 }
 
+sub suspend_young_if_not_enough_mem() {
+    # Check each $sshlogin if there is enough mem.
+    # If less than $limit free mem: suspend some of the young children
+    # Else: Resume all jobs
+    # Uses:
+    #   %Global::running
+    my $limit = shift;
+    my %jobs_of;
+    my @sshlogins;
+
+    for my $job (values %Global::running) {
+	if(not $jobs_of{$job->sshlogin()}) {
+	    push @sshlogins, $job->sshlogin();
+	}
+	push @{$jobs_of{$job->sshlogin()}}, $job;
+    }
+    for my $sshlogin (@sshlogins) {
+	my $free = $sshlogin->memfree();
+	if($free < 2*$limit) {
+	    # Suspend all jobs (resume some of them later)
+	    map { $_->suspended() or $_->suspend(); } @{$jobs_of{$sshlogin}};
+	    my @jobs = sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}};
+	    # how many should be running?
+	    # limit*1 => 1;
+	    # limit*1.5 => 2;
+	    # limit*1.75 => 4;
+	    # free < limit*(2-1/2^n);
+	    # =>
+	    # 1/(2-free/limit) < 2^n;
+	    my $run = int(1/(2-$free/$limit));
+	    $run = ::min($run,$#jobs);
+	    # Resume the oldest running
+	    for my $job ((sort { $a->seq() <=> $b->seq() } @jobs)[0..$run]) {
+		::debug("mem","\nResume ",$run+1, " jobs. Seq ",
+			$job->seq(), " resumed ",
+			$sshlogin->memfree()," < ",2*$limit);
+		$job->resume();
+	    }
+	} else {
+	    for my $job (@{$jobs_of{$sshlogin}}) {
+		if($job->suspended()) {
+		    $job->resume();
+		    ::debug("mem","\nResume ",$#{$jobs_of{$sshlogin}}+1, " jobs. Seq ",
+			    $job->seq(), " resumed ",
+			    $sshlogin->memfree()," > ",2*$limit);
+		    last;
+		}
+	    }
+	}
+    }
+}
+
 sub kill_youngster_if_not_enough_mem() {
     # Check each $sshlogin if there is enough mem.
     # If less than 50% enough free mem: kill off the youngest child
@@ -5977,11 +6038,7 @@ sub kill_youngster_if_not_enough_mem() {
 			 @{$jobs_of{$sshlogin}}));
 		::debug("mem","\n", $job->seq(), "killed ",
 			$sshlogin->memfree()," < ",$limit);
-		if($opt::memsuspend) {
+		$job->kill();
-		    $job->suspend();
-		} else {
-		    $job->kill();
-		}
 		$sshlogin->memfree_recompute();
 	    } else {
 		last;
@@ -8684,8 +8741,6 @@ sub suspend($) {
     my @pgrps = map { -$_ } $self->pid();
     kill "STOP", @pgrps;
     $self->set_suspended(1);
-    # push job onto start stack
-    $Global::JobQueue->unget($self);
 }
 
 sub set_suspended($$) {
@@ -9723,11 +9778,6 @@ sub start($) {
     }
 
     my $job = shift;
-    if($job->suspended()) {
-	# Job is kill -STOP'ped: Restart it.
-	$job->resume();
-	return $job;
-    }
     # Get the shell command to be executed (possibly with ssh infront).
     my $command = $job->wrapped();
     my $pid;
@@ -12379,6 +12429,11 @@ sub pQ($) {
     return $ret;
 }
 
+sub hash($) {
+    $Global::use{"DBI"} ||= eval "use B; 1;";
+    B::hash(@_);
+}
+
 sub total_jobs() {
     return $Global::JobQueue->total_jobs();
 }
@@ -12402,6 +12457,24 @@ sub total_jobs() {
 	# Do not quote this arg
 	$Global::unquote_arg = 1;
     }
+    sub yyyy_mm_dd_hh_mm_ss() {
+	::strftime("%Y-%m-%dT%H:%M:%S", localtime(time()));
+    }
+    sub yyyy_mm_dd_hh_mm() {
+	::strftime("%Y-%m-%dT%H:%M", localtime(time()));
+    }
+    sub yyyy_mm_dd() {
+	::strftime("%Y-%m-%d", localtime(time()));
+    }
+    sub yyyymmddhhmmss() {
+	::strftime("%Y%m%d%H%M%S", localtime(time()));
+    }
+    sub yyyymmddhhmm() {
+	::strftime("%Y%m%d%H%M", localtime(time()));
+    }
+    sub yyyymmdd() {
+	::strftime("%Y%m%d", localtime(time()));
+    }
 
     sub replace($$$$) {
 	# Calculates the corresponding value for a given perl expression
@@ -13448,8 +13521,8 @@ sub main() {
 	while(start_more_jobs()) {}
 	spreadstdin();
     } else {
-	# Reap one - start one
+	# Reap the finished jobs - start more
-	while(reaper() + start_more_jobs()) {}
+	while(reapers() + start_more_jobs()) {}
     }
     ::debug("init", "Start draining\n");
     drain_job_queue(@command);

src/parallel_alternatives.pod

@@ -2877,7 +2877,8 @@ with replacement strings. Such as:
 
 that can be used like:
 
-  parallel --header : --tmpl my.tmpl {#}.t myprog {#}.t ::: x 1 2 3 ::: y 1 2 3
+  parallel --header : --tmpl my.tmpl {#}.t myprog {#}.t \
+    ::: x 1 2 3 ::: y 1 2 3
 
 Filtering may also be supported as:
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -941,6 +941,20 @@ par_json() {
 	perl -pe 's/\d/0/g'
 }
 
+par_hash_and_time_functions() {
+    echo '### Functions for replacement string'
+    parallel echo '{= $_=join(" ",
+                              yyyy_mm_dd_hh_mm_ss(),
+                              yyyy_mm_dd_hh_mm(),
+                              yyyy_mm_dd(),
+                              yyyymmddhhmmss(),
+			      yyyymmddhhmm(),
+			      yyyymmdd()) =}' ::: 1 |
+	perl -pe 's/\d/9/g'
+    parallel echo '{= $_=hash($_) =}' ::: 1 |
+	perl -pe 's/[a-f0-9]/X/g'
+}
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | LC_ALL=C sort |
     parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |

testsuite/tests-to-run/parallel-local-3s.sh

@@ -31,11 +31,6 @@ par_10000_m_X() {
     seq 10000 | perl -pe 's/$/.gif/' |
         parallel -j1 -km echo a{}b{.}c{.} |
         parallel -k --pipe --tee ::: wc md5sum
-    seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
-    seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l
-    seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | wc -l
-    seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c | wc -l
-    seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b | wc -l
 }
 
 par_10000_5_rpl_X() {
@@ -49,7 +44,6 @@ par_10000_5_rpl_X() {
 
 par_X_I_meta() {
     echo '### Test -X -I with shell meta chars'
-
     seq 10000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l
     seq 10000 | parallel -j1 -I '<>' -X echo 'a<>b<>c<>' | wc -l
     seq 10000 | parallel -j1 -I '<' -X echo 'a<b<c<' | wc -l

testsuite/tests-to-run/parallel-tutorial.sh

@@ -74,8 +74,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             # Timings are often off
             s/^(\d)$/9/;
             s/^(\d\d)$/99/;
-	    # Sometime these vars are not present
+	    # Remove variable names - they vary
-            s/^PAM_KWALLET5*_LOGIN$//;
+	    s/^[A-Z][A-Z0-9_]*\s$//;
 	    # Fails often due to race
 	    s/cat: input_file: No such file or directory\n//;
 	    s{rsync: link_stat ".*/home/parallel/input_file.out" .*\n}{};
@@ -93,11 +93,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
 	    s/doi.*=.*//;
 	    s/url.*= .*doi.org.*//;
 	    s/.Feel free to use .nocite.*//;
-	    ' |
+	    ' | uniq
-  perl -ne '/GTK2_RC_FILES/ and next;
-    /GTK_RC_FILES/ and next;
-    print' |
-  uniq
 # 3+3 .par files (from --files), 1 .tms-file from tmux attach
 find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l
 find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | parallel rm

testsuite/wanted-results/parallel-local-0.3s

@@ -193,12 +193,13 @@ par_halt_one_job	1
 par_halt_one_job	2
 par_halt_one_job	parallel: This job failed:
 par_halt_one_job	echo 2;exit 1
-par_halt_one_job	parallel: Starting no more jobs. Waiting for 0 jobs to finish.
 par_halt_one_job	should run 1 = job 1
 par_halt_one_job	1
 par_halt_one_job	parallel: This job failed:
 par_halt_one_job	echo 1;exit 1
-par_halt_one_job	parallel: Starting no more jobs. Waiting for 0 jobs to finish.
+par_hash_and_time_functions	### Functions for replacement string
+par_hash_and_time_functions	9999-99-99T99:99:99 9999-99-99T99:99 9999-99-99 99999999999999 999999999999 99999999
+par_hash_and_time_functions	XxXXXXXXXX
 par_help	### Test --help and -h: Help output (just check we get the same amount of lines)
 par_help	Output from -h and --help
 par_help	37

testsuite/wanted-results/parallel-local-3s

@@ -7,11 +7,6 @@ par_10000_5_rpl_X	1
 par_10000_m_X	### Test -m with 10000 args
 par_10000_m_X	      2   29996  186684
 par_10000_m_X	c606aec1723ee5cc15f2a1b95d83d3cf  -
-par_10000_m_X	3
-par_10000_m_X	2
-par_10000_m_X	2
-par_10000_m_X	2
-par_10000_m_X	1
 par_X_I_meta	### Test -X -I with shell meta chars
 par_X_I_meta	2
 par_X_I_meta	2

testsuite/wanted-results/parallel-local-ssh4

@@ -3,6 +3,7 @@ echo '### --env _'
   fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
 OK FUBAR test
   fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
+OK FUBAR test
 echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
 ### --env _ with explicit mentioning of normally ignored var $DEBEMAIL
   fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test

testsuite/wanted-results/parallel-tutorial

@@ -28,17 +28,8 @@ sleep .3
 /usr/bin/bash: -c: line 0: `   perl -e 'for(1..10){print "$_\n"}') > num_%header'
   perl -e 'print "HHHHAAABBBCCC"' > fixedlen
   parallel echo ::: A B C
-A
-B
-C
   parallel -a abc-file echo
-A
-B
-C
   cat abc-file | parallel echo
-A
-B
-C
   parallel echo ::: A B C ::: D E F
 A D
 A E
@@ -143,16 +134,8 @@ C D
 C E
 C F
   parallel -d _ echo :::: abc_-file
-A
-B
-C
   parallel -d '\0' echo :::: abc0-file
-A
-B
-C
   parallel -E stop echo ::: A B stop C D
-A
-B
   (echo 1; echo; echo 2) | parallel --no-run-if-empty echo
 9
   parallel ::: ls 'echo foo' pwd
@@ -185,9 +168,7 @@ A/B
   parallel echo {/} ::: A/B.C
 B.C
   parallel echo {//} ::: A/B.C
-A
   parallel echo {/.} ::: A/B.C
-B
   parallel echo {#} ::: A B C
 9
   parallel -j 2 echo {%} ::: A B C
@@ -199,9 +180,7 @@ A/B
   parallel --basenamereplace ,, echo ,, ::: A/B.C
 B.C
   parallel --dirnamereplace ,, echo ,, ::: A/B.C
-A
   parallel --basenameextensionreplace ,, echo ,, ::: A/B.C
-B
   parallel --seqreplace ,, echo ,, ::: A B C
 9
   parallel -j2 --slotreplace ,, echo ,, ::: A B C
@@ -365,7 +344,6 @@ G H
   parallel -N0 echo foo ::: 1 2 3
 foo
   perl -e 'print "@ARGV\n"' A
-A
   parallel perl -e 'print "@ARGV\n"' ::: This wont work
   Warning: Input is read from the terminal. You either know what you
   Warning: are doing (in which case: YOU ARE AWESOME!) or you forgot
@@ -420,9 +398,6 @@ echo A
 echo B
 echo C
   parallel --verbose echo {} ::: A B C
-A
-B
-C
 echo A
 echo B
 echo C
@@ -468,9 +443,6 @@ echo C
   parallel --tmpdir /var/tmp --files echo ::: A B C
 /var/tmp/parXXXXX.par
   parallel --results outdir echo ::: A B C
-A
-B
-C
   outdir/1/A/seq
   outdir/1/A/stderr
   outdir/1/A/stdout
@@ -804,87 +776,6 @@ in my_func baz
   cat ~/.parallel/ignored_vars|sort
 BASH_FUNC_run_once%%
 BASH_FUNC_run_test%%
-COLORFGBG
-COLORTERM
-DBUS_SESSION_BUS_ADDRESS
-DEBEMAIL
-DEBFULLNAME
-DESKTOP_SESSION
-DISPLAY
-EDITOR
-GOPATH
-GPG_AGENT_INFO
-HISTCONTROL
-HOME
-KDE_FULL_SESSION
-KDE_SESSION_UID
-KDE_SESSION_VERSION
-KONSOLE_DBUS_SERVICE
-KONSOLE_DBUS_SESSION
-KONSOLE_DBUS_WINDOW
-KONSOLE_VERSION
-LANG
-LANGUAGE
-LC_ALL
-LESS
-LESSCLOSE
-LESSOPEN
-LOGNAME
-LS_COLORS
-MAKEFLAGS
-MAKELEVEL
-MAKE_TERMERR
-MAKE_TERMOUT
-MFLAGS
-OLDPWD
-ORACLE_HOME
-ORACLE_SID
-PARALLEL
-PARALLEL_HOME
-PARALLEL_JOBSLOT
-PARALLEL_PID
-PARALLEL_RSYNC_OPTS
-PARALLEL_SEQ
-PARALLEL_SSHHOST
-PARALLEL_SSHLOGIN
-PARALLEL_TMP
-PATH
-PERL_MB_OPT
-PERL_MM_OPT
-PROFILEHOME
-PWD
-QT_ACCESSIBILITY
-QT_AUTO_SCREEN_SCALE_FACTOR
-SERVER1
-SERVER2
-SESSION_MANAGER
-SHELL
-SHELL_SESSION_ID
-SHLVL
-SSH_AGENT_PID
-SSH_AUTH_SOCK
-TERM
-TIMEOUT
-TMPDIR
-TRIES
-USER
-VISUAL
-WINDOWID
-XAUTHORITY
-XCURSOR_SIZE
-XCURSOR_THEME
-XDG_CONFIG_DIRS
-XDG_CURRENT_DESKTOP
-XDG_DATA_DIRS
-XDG_RUNTIME_DIR
-XDG_SEAT
-XDG_SEAT_PATH
-XDG_SESSION_CLASS
-XDG_SESSION_DESKTOP
-XDG_SESSION_ID
-XDG_SESSION_PATH
-XDG_SESSION_TYPE
-XDG_VTNR
 _
 mysqlrootpass
   # The function is only copied if using Bash
@@ -1133,61 +1024,35 @@ parallel: Error: echo is not a valid DBURL
   cat fixedlen | parallel --pipe --header .{4} --block 3 --recend '' \
     'echo start; cat; echo'
 start
-HHHHAAA
-start
-HHHHBBB
-start
-HHHHCCC
   echo /foo, bar/, /baz, qux/, | \
     parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
-JOB1
 /foo, END
-JOB2
 bar/, END
-JOB3
 /baz, END
-JOB4
 qux/,
-END
   echo /foo, bar/, /baz, qux/, | \
     parallel -kN1 --recstart / --pipe echo JOB{#}\;cat\;echo END
-JOB1
 /foo, barEND
-JOB2
 /, END
-JOB3
 /baz, quxEND
-JOB4
 /,
-END
   echo /foo, bar/, /baz, qux/, | \
     parallel -kN1 --recend ', ' --recstart / --pipe \
     echo JOB{#}\;cat\;echo END
-JOB1
 /foo, bar/, END
-JOB2
 /baz, qux/,
-END
   echo foo,bar,_baz,__qux, | \
     parallel -kN1 --regexp --recend ,_+ --pipe \
     echo JOB{#}\;cat\;echo END
-JOB1
 foo,bar,_END
-JOB2
 baz,__END
-JOB3
 qux,
-END
   echo foo,bar,_baz,__qux, | \
     parallel -kN1 --rrs --regexp --recend ,_+ --pipe \
     echo JOB{#}\;cat\;echo END
-JOB1
 foo,barEND
-JOB2
 bazEND
-JOB3
 qux,
-END
   cat num_%header | \
     parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat
 cat: num_%header: No such file or directory
@@ -1384,7 +1249,6 @@ When using programs that use GNU Parallel to process data for publication
 please cite as described in 'parallel --citation'.
   parallel --minversion VERSION && \
     echo Your version is at least VERSION.
-VERSION
 Your version is at least VERSION.
   parallel --citation
 Academic tradition requires you to cite works you base your article on.
@@ -1416,9 +1280,6 @@ mentioned in the release notes of next version of GNU Parallel.
   echo '--nice 17' > ~/.parallel/nicetimeout
   echo '--timeout 300%' >> ~/.parallel/nicetimeout
   parallel --profile nicetimeout echo ::: A B C
-A
-B
-C
   echo '-vv --dry-run' > ~/.parallel/dryverbose
   parallel --profile dryverbose --profile nicetimeout echo ::: A B C
 echo A