From d8c419dbeb564361fe8d608c8333046551564738 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 15 Jul 2023 15:59:40 +0200 Subject: [PATCH] parallel: Time functions can now take secs since epoch as argument. --- src/parallel | 97 +++++++++---------- src/parallel.pod | 24 ++--- src/parallel_alternatives.pod | 58 ++++++++++- testsuite/REQUIREMENTS | 21 +++- .../centos39-oracle817/Vagrantfile | 2 +- 5 files changed, 139 insertions(+), 63 deletions(-) diff --git a/src/parallel b/src/parallel index 1904dd21..52746044 100755 --- a/src/parallel +++ b/src/parallel @@ -11695,38 +11695,37 @@ sub print_files($) { if($opt::latestline) { $self->print_latest_line($out_fh); } } if(defined $self->{'exitstatus'}) { - if($opt::latestline) { - # Force re-computing color if --colorfailed - if($opt::colorfailed) { delete $self->{'color'}; } - if($self->{$out_fh,'latestline'} ne "") { - $self->print_latest_line($out_fh); - } - if(@{$self->{'halfline'}{$fdno}}) { - my $l = join('', @{$self->{'halfline'}{$fdno}}); - if($l ne "") { - $self->{$out_fh,'latestline'} = $l; - } - } else { - $self->{$out_fh,'latestline'} = undef; - } - # Print latest line from jobs that are already done - while($print_later{$minvisible}) { - $print_later{$minvisible}->print_latest_line($out_fh); - delete $print_later{$minvisible}; - $minvisible++; - } - # Print latest line from jobs that are on screen now - for(my $row = $minvisible; - $row < $minvisible -1 + ::terminal_rows(); - $row++) { - $print_later{$row} and - $print_later{$row}->print_latest_line($out_fh); - } - } if($Global::files or ($opt::results and not $Global::csvsep)) { $self->add_returnsize(-s $self->fh($fdno,"name")); } else { - if(not $opt::latestline) { + if($opt::latestline) { + # Force re-computing color if --colorfailed + if($opt::colorfailed) { delete $self->{'color'}; } + if($self->{$out_fh,'latestline'} ne "") { + $self->print_latest_line($out_fh); + } + if(@{$self->{'halfline'}{$fdno}}) { + my $l = join('', @{$self->{'halfline'}{$fdno}}); + if($l ne "") { + $self->{$out_fh,'latestline'} = $l; + } + } else { + $self->{$out_fh,'latestline'} = undef; + } + # Print latest line from jobs that are already done + while($print_later{$minvisible}) { + $print_later{$minvisible}->print_latest_line($out_fh); + delete $print_later{$minvisible}; + $minvisible++; + } + # Print latest line from jobs that are on screen now + for(my $row = $minvisible; + $row < $minvisible -1 + ::terminal_rows(); + $row++) { + $print_later{$row} and + $print_later{$row}->print_latest_line($out_fh); + } + } else { # If the job is dead: print the remaining partial line # read remaining (already done for $opt::latestline) my $halfline_ref = $self->{'halfline'}{$fdno}; @@ -14185,45 +14184,45 @@ sub total_jobs() { # Do not quote this arg $Global::unquote_arg = 1; } - sub yyyy_mm_dd_hh_mm_ss() { + sub yyyy_mm_dd_hh_mm_ss(@) { # ISO8601 2038-01-19T03:14:08 - ::strftime("%Y-%m-%dT%H:%M:%S", localtime(time())); + ::strftime("%Y-%m-%dT%H:%M:%S", localtime(shift || time())); } - sub yyyy_mm_dd_hh_mm() { + sub yyyy_mm_dd_hh_mm(@) { # ISO8601 2038-01-19T03:14 - ::strftime("%Y-%m-%dT%H:%M", localtime(time())); + ::strftime("%Y-%m-%dT%H:%M", localtime(shift || time())); } - sub yyyy_mm_dd() { + sub yyyy_mm_dd(@) { # ISO8601 2038-01-19 - ::strftime("%Y-%m-%d", localtime(time())); + ::strftime("%Y-%m-%d", localtime(shift || time())); } - sub hh_mm_ss() { + sub hh_mm_ss(@) { # ISO8601 03:14:08 - ::strftime("%H:%M:%S", localtime(time())); + ::strftime("%H:%M:%S", localtime(shift || time())); } - sub hh_mm() { + sub hh_mm(@) { # ISO8601 03:14 - ::strftime("%H:%M", localtime(time())); + ::strftime("%H:%M", localtime(shift || time())); } - sub yyyymmddhhmmss() { + sub yyyymmddhhmmss(@) { # ISO8601 20380119 + ISO8601 031408 - ::strftime("%Y%m%d%H%M%S", localtime(time())); + ::strftime("%Y%m%d%H%M%S", localtime(shift || time())); } - sub yyyymmddhhmm() { + sub yyyymmddhhmm(@) { # ISO8601 20380119 + ISO8601 0314 - ::strftime("%Y%m%d%H%M", localtime(time())); + ::strftime("%Y%m%d%H%M", localtime(shift || time())); } - sub yyyymmdd() { + sub yyyymmdd(@) { # ISO8601 20380119 - ::strftime("%Y%m%d", localtime(time())); + ::strftime("%Y%m%d", localtime(shift || time())); } - sub hhmmss() { + sub hhmmss(@) { # ISO8601 031408 - ::strftime("%H%M%S", localtime(time())); + ::strftime("%H%M%S", localtime(shift || time())); } - sub hhmm() { + sub hhmm(@) { # ISO8601 0314 - ::strftime("%H%M", localtime(time())); + ::strftime("%H%M", localtime(shift || time())); } sub replace($$$$) { diff --git a/src/parallel.pod b/src/parallel.pod index 9674bb00..e286c649 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -374,27 +374,28 @@ the arguments skip this job (see also B<--filter>) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -=item Z<> B +=item Z<> B (alpha testing) -time functions +time functions. I is number of seconds since epoch. If left out +it will use current local time. =back @@ -405,6 +406,7 @@ Example: seq 50 | parallel echo job {#} of {= '$_=total_jobs()' =} See also: B<--rpl> B<--parens> B<{}> B<{=>I IB<=}> +B<--filter> =item B<{=>I IB<=}> diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index b2c104c4..4b459b3e 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -4084,6 +4084,62 @@ https://github.com/binpash/pash (Last checked: 2023-05) +=head2 DIFFERENCES BETWEEN korovkin-parallel AND GNU Parallel + +Summary (see legend above): + +=over + +=item I1 - - - - - - + +=item M1 - - - - M6 + +=item - - O3 - - - - N/A N/A - + +=item E1 - - - - - - + +=item R1 - - - - R6 N/A N/A - + +=item - - + +=back + +B prepends all lines with some info. + +The output is colored with 6 color combinations, so job 1 and 7 will +get the same color. + +You can get similar output with: + + (echo ...) | + parallel --color -j 10 --lb --tagstring \ + '[l:{#}:{=$_=sprintf("%7.03f",::now()-$^T)=} {=$_=hh_mm_ss($^T)=} {%}]' + +Lines longer than 8192 chars are broken into lines shorter than +8192. B loses the last char for lines exactly 8193 +chars long. + +Short lines from different jobs do not mix, but long lines do: + + fun() { + perl -e '$a="'$1'"x1000000; for(1..'$2') { print $a };'; + } + export -f fun + (echo fun a 100;echo fun b 100) | ./parallel | tr -s abcdef + +There should be only one line of a's and one line of b's. + +Just like GNU B B offers a master/slave +model, so workers on other servers can do some of the tasks. But +contrary to GNU B you must manually start workers on these +servers. The communication is neither authenticated nor encrypted. + +It caches output in RAM: a 1GB line uses ~2.5GB RAM + +https://github.com/korovkin/parallel +(Last checked: 2023-07) + + =head2 Todo https://www.npmjs.com/package/concurrently @@ -4136,7 +4192,7 @@ This test stresses whether output mixes. #!/bin/bash - paralleltool="parallel -j0" + paralleltool="parallel -j 30" cat <<-EOF > mycommand #!/bin/bash diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index 19f7b02d..df27508a 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -137,6 +137,24 @@ setup_databases() { } add_server_to_hosts() { + add_ssh_key_to_authorized() { + (cat vagrant/authorized_keys; cat ~/.ssh/*.pub) | + uniq > vagrant/authorized_keys.$$ + mv vagrant/authorized_keys.$$ vagrant/authorized_keys + + # Fix: + # could not settle on kex algorithm + # Server kex preferences: diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1 + # Client kex preferences: ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1> +debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,sntrup761x25519-sha512@openssh.com,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group1-sha1,ext-info-c +debug2: host key algorithms: ssh-dss,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 +debug2: ciphers ctos: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc +debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc +debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 +debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 + + } + insert_in_etc_hosts() { ip=$1 host=$2 @@ -159,8 +177,9 @@ add_server_to_hosts() { } export -f add_single_vagrant_to_etc_hosts + add_ssh_key_to_authorized insert_in_etc_hosts 127.1.2.3 server - parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 rhel8 centos3 + parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 rhel8 centos3 centos39-oracle817 } shellsplus() { diff --git a/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile b/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile index cc131690..605430ee 100644 --- a/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile +++ b/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile @@ -32,7 +32,7 @@ Vagrant.configure("2") do |config| # Create a private network, which allows host-only access to the machine # using a specific IP. - config.vm.network "private_network", ip: "172.27.27.3" + config.vm.network "private_network", ip: "172.27.27.33" # Create a public network, which generally matched to bridged network. # Bridged networks make the machine appear as another physical device on