parallel: Time functions can now take secs since epoch as argument.

This commit is contained in:
Ole Tange 2023-07-15 15:59:40 +02:00
parent 24fb1b9153
commit d8c419dbeb
5 changed files with 139 additions and 63 deletions

View file

@ -11695,38 +11695,37 @@ sub print_files($) {
if($opt::latestline) { $self->print_latest_line($out_fh); } if($opt::latestline) { $self->print_latest_line($out_fh); }
} }
if(defined $self->{'exitstatus'}) { if(defined $self->{'exitstatus'}) {
if($opt::latestline) {
# Force re-computing color if --colorfailed
if($opt::colorfailed) { delete $self->{'color'}; }
if($self->{$out_fh,'latestline'} ne "") {
$self->print_latest_line($out_fh);
}
if(@{$self->{'halfline'}{$fdno}}) {
my $l = join('', @{$self->{'halfline'}{$fdno}});
if($l ne "") {
$self->{$out_fh,'latestline'} = $l;
}
} else {
$self->{$out_fh,'latestline'} = undef;
}
# Print latest line from jobs that are already done
while($print_later{$minvisible}) {
$print_later{$minvisible}->print_latest_line($out_fh);
delete $print_later{$minvisible};
$minvisible++;
}
# Print latest line from jobs that are on screen now
for(my $row = $minvisible;
$row < $minvisible -1 + ::terminal_rows();
$row++) {
$print_later{$row} and
$print_later{$row}->print_latest_line($out_fh);
}
}
if($Global::files or ($opt::results and not $Global::csvsep)) { if($Global::files or ($opt::results and not $Global::csvsep)) {
$self->add_returnsize(-s $self->fh($fdno,"name")); $self->add_returnsize(-s $self->fh($fdno,"name"));
} else { } else {
if(not $opt::latestline) { if($opt::latestline) {
# Force re-computing color if --colorfailed
if($opt::colorfailed) { delete $self->{'color'}; }
if($self->{$out_fh,'latestline'} ne "") {
$self->print_latest_line($out_fh);
}
if(@{$self->{'halfline'}{$fdno}}) {
my $l = join('', @{$self->{'halfline'}{$fdno}});
if($l ne "") {
$self->{$out_fh,'latestline'} = $l;
}
} else {
$self->{$out_fh,'latestline'} = undef;
}
# Print latest line from jobs that are already done
while($print_later{$minvisible}) {
$print_later{$minvisible}->print_latest_line($out_fh);
delete $print_later{$minvisible};
$minvisible++;
}
# Print latest line from jobs that are on screen now
for(my $row = $minvisible;
$row < $minvisible -1 + ::terminal_rows();
$row++) {
$print_later{$row} and
$print_later{$row}->print_latest_line($out_fh);
}
} else {
# If the job is dead: print the remaining partial line # If the job is dead: print the remaining partial line
# read remaining (already done for $opt::latestline) # read remaining (already done for $opt::latestline)
my $halfline_ref = $self->{'halfline'}{$fdno}; my $halfline_ref = $self->{'halfline'}{$fdno};
@ -14185,45 +14184,45 @@ sub total_jobs() {
# Do not quote this arg # Do not quote this arg
$Global::unquote_arg = 1; $Global::unquote_arg = 1;
} }
sub yyyy_mm_dd_hh_mm_ss() { sub yyyy_mm_dd_hh_mm_ss(@) {
# ISO8601 2038-01-19T03:14:08 # ISO8601 2038-01-19T03:14:08
::strftime("%Y-%m-%dT%H:%M:%S", localtime(time())); ::strftime("%Y-%m-%dT%H:%M:%S", localtime(shift || time()));
} }
sub yyyy_mm_dd_hh_mm() { sub yyyy_mm_dd_hh_mm(@) {
# ISO8601 2038-01-19T03:14 # ISO8601 2038-01-19T03:14
::strftime("%Y-%m-%dT%H:%M", localtime(time())); ::strftime("%Y-%m-%dT%H:%M", localtime(shift || time()));
} }
sub yyyy_mm_dd() { sub yyyy_mm_dd(@) {
# ISO8601 2038-01-19 # ISO8601 2038-01-19
::strftime("%Y-%m-%d", localtime(time())); ::strftime("%Y-%m-%d", localtime(shift || time()));
} }
sub hh_mm_ss() { sub hh_mm_ss(@) {
# ISO8601 03:14:08 # ISO8601 03:14:08
::strftime("%H:%M:%S", localtime(time())); ::strftime("%H:%M:%S", localtime(shift || time()));
} }
sub hh_mm() { sub hh_mm(@) {
# ISO8601 03:14 # ISO8601 03:14
::strftime("%H:%M", localtime(time())); ::strftime("%H:%M", localtime(shift || time()));
} }
sub yyyymmddhhmmss() { sub yyyymmddhhmmss(@) {
# ISO8601 20380119 + ISO8601 031408 # ISO8601 20380119 + ISO8601 031408
::strftime("%Y%m%d%H%M%S", localtime(time())); ::strftime("%Y%m%d%H%M%S", localtime(shift || time()));
} }
sub yyyymmddhhmm() { sub yyyymmddhhmm(@) {
# ISO8601 20380119 + ISO8601 0314 # ISO8601 20380119 + ISO8601 0314
::strftime("%Y%m%d%H%M", localtime(time())); ::strftime("%Y%m%d%H%M", localtime(shift || time()));
} }
sub yyyymmdd() { sub yyyymmdd(@) {
# ISO8601 20380119 # ISO8601 20380119
::strftime("%Y%m%d", localtime(time())); ::strftime("%Y%m%d", localtime(shift || time()));
} }
sub hhmmss() { sub hhmmss(@) {
# ISO8601 031408 # ISO8601 031408
::strftime("%H%M%S", localtime(time())); ::strftime("%H%M%S", localtime(shift || time()));
} }
sub hhmm() { sub hhmm(@) {
# ISO8601 0314 # ISO8601 0314
::strftime("%H%M", localtime(time())); ::strftime("%H%M", localtime(shift || time()));
} }
sub replace($$$$) { sub replace($$$$) {

View file

@ -374,27 +374,28 @@ the arguments
skip this job (see also B<--filter>) skip this job (see also B<--filter>)
=item Z<> B<yyyy_mm_dd_hh_mm_ss()> =item Z<> B<yyyy_mm_dd_hh_mm_ss(sec)> (alpha testing)
=item Z<> B<yyyy_mm_dd_hh_mm()> =item Z<> B<yyyy_mm_dd_hh_mm(sec)> (alpha testing)
=item Z<> B<yyyy_mm_dd()> =item Z<> B<yyyy_mm_dd(sec)> (alpha testing)
=item Z<> B<hh_mm_ss()> =item Z<> B<hh_mm_ss(sec)> (alpha testing)
=item Z<> B<hh_mm()> =item Z<> B<hh_mm(sec)> (alpha testing)
=item Z<> B<yyyymmddhhmmss()> =item Z<> B<yyyymmddhhmmss(sec)> (alpha testing)
=item Z<> B<yyyymmddhhmm()> =item Z<> B<yyyymmddhhmm(sec)> (alpha testing)
=item Z<> B<yyyymmdd()> =item Z<> B<yyyymmdd(sec)> (alpha testing)
=item Z<> B<hhmmss()> =item Z<> B<hhmmss(sec)> (alpha testing)
=item Z<> B<hhmm()> =item Z<> B<hhmm(sec)> (alpha testing)
time functions time functions. I<sec> is number of seconds since epoch. If left out
it will use current local time.
=back =back
@ -405,6 +406,7 @@ Example:
seq 50 | parallel echo job {#} of {= '$_=total_jobs()' =} seq 50 | parallel echo job {#} of {= '$_=total_jobs()' =}
See also: B<--rpl> B<--parens> B<{}> B<{=>I<n> I<perl expression>B<=}> See also: B<--rpl> B<--parens> B<{}> B<{=>I<n> I<perl expression>B<=}>
B<--filter>
=item B<{=>I<n> I<perl expression>B<=}> =item B<{=>I<n> I<perl expression>B<=}>

View file

@ -4084,6 +4084,62 @@ https://github.com/binpash/pash
(Last checked: 2023-05) (Last checked: 2023-05)
=head2 DIFFERENCES BETWEEN korovkin-parallel AND GNU Parallel
Summary (see legend above):
=over
=item I1 - - - - - -
=item M1 - - - - M6
=item - - O3 - - - - N/A N/A -
=item E1 - - - - - -
=item R1 - - - - R6 N/A N/A -
=item - -
=back
B<korovkin-parallel> prepends all lines with some info.
The output is colored with 6 color combinations, so job 1 and 7 will
get the same color.
You can get similar output with:
(echo ...) |
parallel --color -j 10 --lb --tagstring \
'[l:{#}:{=$_=sprintf("%7.03f",::now()-$^T)=} {=$_=hh_mm_ss($^T)=} {%}]'
Lines longer than 8192 chars are broken into lines shorter than
8192. B<korovkin-parallel> loses the last char for lines exactly 8193
chars long.
Short lines from different jobs do not mix, but long lines do:
fun() {
perl -e '$a="'$1'"x1000000; for(1..'$2') { print $a };';
}
export -f fun
(echo fun a 100;echo fun b 100) | ./parallel | tr -s abcdef
There should be only one line of a's and one line of b's.
Just like GNU B<parallel> B<korovkin-parallel> offers a master/slave
model, so workers on other servers can do some of the tasks. But
contrary to GNU B<parallel> you must manually start workers on these
servers. The communication is neither authenticated nor encrypted.
It caches output in RAM: a 1GB line uses ~2.5GB RAM
https://github.com/korovkin/parallel
(Last checked: 2023-07)
=head2 Todo =head2 Todo
https://www.npmjs.com/package/concurrently https://www.npmjs.com/package/concurrently
@ -4136,7 +4192,7 @@ This test stresses whether output mixes.
#!/bin/bash #!/bin/bash
paralleltool="parallel -j0" paralleltool="parallel -j 30"
cat <<-EOF > mycommand cat <<-EOF > mycommand
#!/bin/bash #!/bin/bash

View file

@ -137,6 +137,24 @@ setup_databases() {
} }
add_server_to_hosts() { add_server_to_hosts() {
add_ssh_key_to_authorized() {
(cat vagrant/authorized_keys; cat ~/.ssh/*.pub) |
uniq > vagrant/authorized_keys.$$
mv vagrant/authorized_keys.$$ vagrant/authorized_keys
# Fix:
# could not settle on kex algorithm
# Server kex preferences: diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1
# Client kex preferences: ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1>
debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,sntrup761x25519-sha512@openssh.com,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group1-sha1,ext-info-c
debug2: host key algorithms: ssh-dss,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
debug2: ciphers ctos: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc
debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc
debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
}
insert_in_etc_hosts() { insert_in_etc_hosts() {
ip=$1 ip=$1
host=$2 host=$2
@ -159,8 +177,9 @@ add_server_to_hosts() {
} }
export -f add_single_vagrant_to_etc_hosts export -f add_single_vagrant_to_etc_hosts
add_ssh_key_to_authorized
insert_in_etc_hosts 127.1.2.3 server insert_in_etc_hosts 127.1.2.3 server
parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 rhel8 centos3 parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 rhel8 centos3 centos39-oracle817
} }
shellsplus() { shellsplus() {

View file

@ -32,7 +32,7 @@ Vagrant.configure("2") do |config|
# Create a private network, which allows host-only access to the machine # Create a private network, which allows host-only access to the machine
# using a specific IP. # using a specific IP.
config.vm.network "private_network", ip: "172.27.27.3" config.vm.network "private_network", ip: "172.27.27.33"
# Create a public network, which generally matched to bridged network. # Create a public network, which generally matched to bridged network.
# Bridged networks make the machine appear as another physical device on # Bridged networks make the machine appear as another physical device on