parallel: Make --slf reload every second.

This commit is contained in:
Ole Tange 2022-03-26 13:11:17 +01:00
parent 58d1437470
commit 88f2312390
17 changed files with 50 additions and 23 deletions

2
NEWS
View file

@ -4,7 +4,7 @@ New in this release:
* --sshlogin user:password@host is now supported by using sshpass. * --sshlogin user:password@host is now supported by using sshpass.
* Bug fixes and man page updates: Many options now have a 'See also' section. * Bug fixes and man page updates.
News about GNU Parallel: News about GNU Parallel:

View file

@ -4,10 +4,12 @@
Quote of the month: Quote of the month:
wow gnu parallel fucking rules
-- senior polycule reliability engineer @ilianathewitch@twitter
아 parallel 너무 좋지요. 아 parallel 너무 좋지요.
-- 어엉부엉@d_ijk_stra -- 어엉부엉@d_ijk_stra
It's amazing how fast you can get with bash pipelines and GNU Parallel. It's amazing how fast you can get with bash pipelines and GNU Parallel.
-- Eric Pauley @EricPauley_ -- Eric Pauley @EricPauley_

View file

@ -254,28 +254,25 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20220322 ('Маріу́поль') released Subject: GNU Parallel 20220422 ('albright<<>>') released [stable]
GNU Parallel 20220322 ('Маріу́поль') has been released. It is available for download at: lbry://@GnuParallel:4 GNU Parallel 20220422 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
No new functionality was introduced so this is a good candidate for a stable release. <<No new functionality was introduced so this is a good candidate for a stable release.>>
Quote of the month: Quote of the month:
My favorite software, ever. Keep the good work. <<>>
-- Federico Alves @federicoalves@twitter
New in this release: New in this release:
* --sshlogin user:password@host is now supported by using sshpass. <<>>
* Bug fixes and man page updates: Many options now have a 'See also' section. * Bug fixes and man page updates: Many options now have a 'See also' section.
News about GNU Parallel: News about GNU Parallel:
* Bash: GNU Parallel with Curl https://gist.github.com/CMCDragonkai/5914e02df62137e47f32?permalink_comment_id=2617456 <<>>
* The Pipe Operator Explained https://medium.com/geekculture/the-pipe-operator-explained-cbd41e23775a
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

View file

@ -385,7 +385,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -387,7 +387,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -385,7 +385,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -363,7 +363,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -365,7 +365,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -390,7 +390,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -355,7 +355,7 @@ _parset_main() {
return 255 return 255
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
echo "parset 20220322 (GNU parallel `parallel --minversion 1`)" echo "parset 20220323 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2022 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -26,7 +26,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20220322; $Global::version = 20220323;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

View file

@ -2313,7 +2313,7 @@ sub check_invalid_option_combinations() {
sub init_globals() { sub init_globals() {
# Defaults: # Defaults:
$Global::version = 20220322; $Global::version = 20220323;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$::name = "GNU Parallel"; $::name = "GNU Parallel";
$Global::infinity = 2**31; $Global::infinity = 2**31;
@ -3700,6 +3700,7 @@ sub drain_job_queue(@) {
} }
my $last_header = ""; my $last_header = "";
my $sleep = 0.2; my $sleep = 0.2;
my $sleepsum = 0;
do { do {
while($Global::total_running > 0) { while($Global::total_running > 0) {
debug("init",$Global::total_running, "==", scalar debug("init",$Global::total_running, "==", scalar
@ -3730,6 +3731,14 @@ sub drain_job_queue(@) {
} }
# Exponential back-off sleeping # Exponential back-off sleeping
$sleep = ::reap_usleep($sleep); $sleep = ::reap_usleep($sleep);
$sleepsum += $sleep;
if($sleepsum >= 1000) {
# At most do this every second
$sleepsum = 0;
changed_procs_file();
changed_sshloginfile();
start_more_jobs();
}
} }
if(not $Global::JobQueue->empty()) { if(not $Global::JobQueue->empty()) {
# These jobs may not be started: # These jobs may not be started:

View file

@ -2287,7 +2287,7 @@ will generate the files:
my_foo/stderr my_foo/stderr
my_foo/stdout my_foo/stdout
See also: B<--files> B<--tag> B<--header> B<--joblog> See also: B<--output-as-files> B<--tag> B<--header> B<--joblog>
=item B<--resume> =item B<--resume>

View file

@ -122,7 +122,7 @@ GetOptions(
"help" => \$opt::dummy, "help" => \$opt::dummy,
) || exit(255); ) || exit(255);
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
$Global::version = 20220322; $Global::version = 20220323;
if($opt::version) { version(); exit 0; } if($opt::version) { version(); exit 0; }
@Global::sortoptions = grep { ! /^-D$/ } @Global::sortoptions = grep { ! /^-D$/ }
shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]);

View file

@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20220322; $Global::version = 20220323;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

View file

@ -8,6 +8,23 @@
# Each should be taking 10-30s and be possible to run in parallel # Each should be taking 10-30s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_reload_slf_every_second() {
echo "### --slf should reload every second"
tmp=$(mktemp)
echo 5/lo >"$tmp"
(
sleep 3
(echo 5/nlv.pi.dk
echo 5/localhost
echo 5/127.0.0.1) >>"$tmp"
) &
# This used to take 20 seconds
seq 20 |
stdout /usr/bin/time -f %e parallel --slf "$tmp" 'true {};sleep 10' |
perl -ne '$_ < 20 and print "OK\n"'
rm "$tmp"
}
par_load_blocks() { par_load_blocks() {
echo "### Test if --load blocks. Bug."; echo "### Test if --load blocks. Bug.";
export PARALLEL="--load 300%" export PARALLEL="--load 300%"

View file

@ -719,6 +719,8 @@ par_pipe_line_buffer 200 400 1202
par_pipe_line_buffer These must diff: 1 par_pipe_line_buffer These must diff: 1
par_pipe_line_buffer_compress ### --pipe --line-buffer --compress par_pipe_line_buffer_compress ### --pipe --line-buffer --compress
par_pipe_line_buffer_compress 200 400 1202 par_pipe_line_buffer_compress 200 400 1202
par_reload_slf_every_second ### --slf should reload every second
par_reload_slf_every_second OK
par_results_csv bug #: --results csv par_results_csv bug #: --results csv
par_results_csv --header : --tag --files --compress Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,H2,H1,Stdout,Stderr par_results_csv --header : --tag --files --compress Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,Signal,Command,H2,H1,Stdout,Stderr
par_results_csv --header : --tag --files --compress 1,:,999.999,999.999,0,15,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile", par_results_csv --header : --tag --files --compress 1,:,999.999,999.999,0,15,0,0,"echo 22 11",22,11,"22 11 /tmp/parallel-local-10s-tmpdir/tmpfile",