Ole Tange 2020-08-11 22:55:31 +02:00
parent 79b56269bb
commit 72dd439f20
7 changed files with 404 additions and 80 deletions

View file

@ -6249,36 +6249,48 @@ sub memfree_recompute() {
# Buffers: 19876 kB
# Cached: 431192 kB
# SwapCached: 0 kB
"linux" =>
q[ print 1024 * qx{ ].
q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ].
q[ { sum += \$2} END { print sum }' ].
q[ /proc/meminfo } ],
"linux" => (
q{
print 1024 * qx{
awk '/^((Swap)?Cached|MemFree|Buffers):/
{ sum += \$2} END { print sum }'
/proc/meminfo }
}),
# Android uses same code as GNU/Linux
"android" =>
q[ print 1024 * qx{ ].
q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ].
q[ { sum += \$2} END { print sum }' ].
q[ /proc/meminfo } ],
"android" => (
q{
print 1024 * qx{
awk '/^((Swap)?Cached|MemFree|Buffers):/
{ sum += \$2} END { print sum }'
/proc/meminfo }
}),
# $ vmstat 1 1
# procs memory page faults cpu
# r b w avm free re at pi po fr de sr in sy cs us sy id
# 1 0 0 242793 389737 5 1 0 0 0 0 0 107 978 60 1 1 99
"hpux" =>
q[ print (((reverse `vmstat 1 1`)[0] ].
q[ =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ],
# procs memory page faults cpu
# r b w avm free re at pi po fr de sr in sy cs us sy id
# 1 0 0 242793 389737 5 1 0 0 0 0 0 107 978 60 1 1 99
"hpux" => (
q{
print (((reverse `vmstat 1 1`)[0]
=~ /(?:\d+\D+){4}(\d+)/)[0]*1024)
}),
# $ vmstat 1 2
# kthr memory page disk faults cpu
# r b w swap free re mf pi po fr de sr s3 s4 -- -- in sy cs us sy id
# 0 0 0 6496720 5170320 68 260 8 2 1 0 0 -0 3 0 0 309 1371 255 1 2 97
# 0 0 0 6434088 5072656 7 15 8 0 0 0 0 0 261 0 0 1889 1899 3222 0 8 92
# 0 0 0 6434088 5072656 7 15 8 0 0 0 0 0 261 0 0 1889 1899 3222 0 8 92
#
# The second free value is correct
"solaris" =>
q[ print (((reverse `vmstat 1 2`)[0] ].
q[ =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ],
"freebsd" => q{
"solaris" => (
q{
print (((reverse `vmstat 1 2`)[0]
=~ /(?:\d+\D+){4}(\d+)/)[0]*1024)
}),
# hw.pagesize: 4096
# vm.stats.vm.v_cache_count: 0
# vm.stats.vm.v_inactive_count: 79574
# vm.stats.vm.v_free_count: 4507
"freebsd" => (
q{
for(qx{/sbin/sysctl -a}) {
if (/^([^:]+):\s+(.+)\s*$/s) {
$sysctl->{$1} = $2;
@ -6288,7 +6300,7 @@ sub memfree_recompute() {
($sysctl->{"vm.stats.vm.v_cache_count"}
+ $sysctl->{"vm.stats.vm.v_inactive_count"}
+ $sysctl->{"vm.stats.vm.v_free_count"});
},
}),
# Mach Virtual Memory Statistics: (page size of 4096 bytes)
# Pages free: 198061.
# Pages active: 159701.
@ -6302,12 +6314,13 @@ sub memfree_recompute() {
# Pageins: 1798068.
# Pageouts: 257.
# Object cache: 6603 hits of 1713223 lookups (0% hit rate)
'darwin' =>
q[ $vm = `vm_stat`;
print (($vm =~ /page size of (\d+)/)[0] *
(($vm =~ /Pages free:\s+(\d+)/)[0] +
($vm =~ /Pages inactive:\s+(\d+)/)[0]));
],
'darwin' => (
q{
$vm = `vm_stat`;
print (($vm =~ /page size of (\d+)/)[0] *
(($vm =~ /Pages free:\s+(\d+)/)[0] +
($vm =~ /Pages inactive:\s+(\d+)/)[0]));
}),
);
my $perlscript = "";
# Make a perl script that detects the OS ($^O) and runs
@ -6315,8 +6328,7 @@ sub memfree_recompute() {
for my $os (keys %script_of) {
$perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}';
}
$perlscript =~ s/[\t\n ]+/ /g;
$script = "perl -e " . ::Q($perlscript);
$script = "perl -e " . ::Q(::spacefree(1,$perlscript));
}
return $script;
}
@ -10469,6 +10481,7 @@ sub slot($) {
{
my $already_spread;
my $env_size;
sub populate($) {
# Add arguments from arg_queue until the number of arguments or
@ -10486,7 +10499,25 @@ sub slot($) {
my $next_arg;
my $max_len = $Global::minimal_command_line_length
|| Limits::Command::max_length();
if($^O eq "darwin") {
# On Mac bash 3.2.48(1)-release the length also depends on
# number of functions, variables and the size of those.
# 21 is found by trial and error and seems to be a safe value.
# test_with() {
# gen500k() {
# seq -f %f 1000000000000000 1000000000050000 |
# head -c 131000;
# }
# for a in `seq 5100`; do eval "export a$a=1" ; done;
# for a in `seq 5100`; do eval "a$a() { 1; }" ; done;
# for a in `seq 5100`; do eval export -f a$a ; done;
# gen500k | stdout parallel -Xj1 'echo {} {} {} {} | wc' |
# perl -pe 's/\d{10,} //g'
# }
$env_size ||= (keys %ENV)*21
+ length(join'',keys %ENV) + length(join'',%ENV);
$max_len -= $env_size;
}
if($opt::cat or $opt::fifo) {
# Get the empty arg added by --pipepart (if any)
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();

View file

@ -2616,6 +2616,85 @@ https://pypi.org/project/papply/ (Last checked: 2020-04)
=head2 Todo
test_many_var() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 11000`; do eval "export a$a=1" ; done
gen500k | stdout parallel --timeout 5 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
test_many_var_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 5100`; do eval "export a$a=1" ; done
for a in `seq 5100`; do eval "a$a() { 1; }" ; done
for a in `seq 5100`; do eval export -f a$a ; done
gen500k | stdout parallel --timeout 21 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
test_many_var_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 8000`; do eval "a$a() { 1; }" ; done
for a in `seq 8000`; do eval export -f a$a ; done
gen500k | stdout parallel --timeout 6 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
test_big_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`seq 1000`
for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
for a in `seq 50`; do eval export -f a$a ; done
gen500k | stdout parallel --timeout 4 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
test_many_var_big_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`seq 1000`
for a in `seq 5100`; do eval "export a$a=1" ; done
for a in `seq 20`; do eval "a$a() { '$big'; }" ; done
for a in `seq 20`; do eval export -f a$a ; done
gen500k | stdout parallel --timeout 6 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
test_big_func_name() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`perl -e print\"x\"x10000`
for a in `seq 20`; do eval "export a$big$a=1" ; done
gen500k | stdout parallel --timeout 8 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
test_big_var_func_name() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`perl -e print\"x\"x10000`
for a in `seq 2`; do eval "export a$big$a=1" ; done
for a in `seq 2`; do eval "a$big$a() { '$big'; }" ; done
for a in `seq 2`; do eval export -f a$big$a ; done
gen500k | stdout parallel --timeout 1000 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
}
tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1 'echo {} {} | wc' | perl -pe 's/\d{3,5} //g'
tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1 'echo {} {} | wc' | perl -pe 's/\d{3,5} //g'
tange@macosx:~$ for a in `seq 100`; do eval export -f a$a ; done
seq 100000 | stdout parallel -Xj1 'echo {} {} | wc'
export a=`seq 10000`
seq 100000 | stdout parallel -Xj1 'echo {} {} | wc'
my $already_spread;
my $env_size;
if($^O eq "darwin") {
$env_size ||= 500+length(join'',%ENV);
$max_len -= $env_size;
}
PASH: Light-touch Data-Parallel Shell Processing
https://arxiv.org/pdf/2007.09436.pdf
https://gitlab.com/netikras/bthread
https://github.com/JeiKeiLim/simple_distribute_job

View file

@ -6,26 +6,33 @@ install_packages() {
INSTALL=${INSTALL:-"sudo aptitude -y install"}
# The testsuite depends on this:
test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl pod2pdf gawk"
test_pkgs="$test_pkgs lua5.3 clisp php-cli nodejs mono-csharp-shell libevent-dev"
test_pkgs="$test_pkgs tcl libtext-csv-perl xterm libc6-i386 libcrypt1:i386"
test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl"
test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs"
test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl"
test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386"
test_pkgs="$test_pkgs libtest-nowarnings-perl"
# DEBIAN package
packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15 python3-m2crypto"
packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15"
packaging_pkgs="$packaging_pkgs python3-m2crypto alien"
# SHEBANG TOOLS
shebang_pkgs="gnuplot octave ruby r-base-core"
# SQL TOOLS
sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl libdbd-mysql-perl rlwrap"
sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
# Compression
compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2 lzma xz-utils lzip bzip2 lbzip2 lrzip"
compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2"
compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip"
compression_pkgs_missing="pxz"
# Shells
# (csh = bsd-csh that is broken)
shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash tcsh yash zsh"
shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash"
shell_pkgs="$shell_pkgs tcsh yash zsh"
# Databases
database_pkgs="postgresql mysql-server sqlite"
sudo dpkg --add-architecture i386
if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs $compression_pkgs $shell_pkgs $database_pkgs; then
sudo dpkg --add-architecture i386; sudo apt update
if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \
$compression_pkgs $shell_pkgs $database_pkgs; then
# OK
true
else
@ -35,6 +42,53 @@ install_packages() {
sudo apt update &
}
install_oracle_client() {
# https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian
# sql oracle:// && return 0
(cd /tmp
get_rpm_install_deb() {
glob="$1"
url="$oracleurl/$2"
testglob="$3"
if [ ! -e `echo $glob*rpm` ] ; then
wget $url
fi
if [ ! -e `echo $glob*deb` ] ; then
echo Convert `echo $glob*rpm` to deb
fakeroot alien `echo $glob*rpm`
fi
if [ ! -e `echo $testglob` ] ; then
echo Install `echo $glob*deb`
sudo dpkg -i `echo $glob*deb`
fi
}
oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600
client=oracle-instantclient19.6
ver=19.6.0.0.0-1.x86_64.rpm
get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright
get_rpm_install_deb 'oracle-instantclient*basic' $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README
get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus
)
echo Add this to .bashrc:
ORACLE_HOME=`echo /usr/lib/oracle/*/client*`
echo ORACLE_HOME=$ORACLE_HOME
if grep -q ORACLE_HOME=/ ~/.bashrc; then
perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc
else
echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc
echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc
echo 'export ORACLE_HOME' >> ~/.bashrc
echo 'export ORACLE_SID=XE' >> ~/.bashrc
fi
perl -MCPAN -e 'install DBD::Oracle'
# TODO set up vagrant oracle server
# TODO set up default passwd
# test it works: sql oracle://
}
setup_databases() {
# DATABASES
echo '# Create PostgreSQL'
@ -46,15 +100,15 @@ setup_databases() {
sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\""
mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E}
dburl=mysql://root:"$mysqlrootpass"@/mysql
echo '# Create MySQL'
sudo su mysql mysqladmin create `whoami`
# Default Debian way of getting "root" access
sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';"
# Drop database and user if needed
sudo sql mysql://root:"$mysqlrootpass"@/mysql "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
sudo sql mysql://root:"$mysqlrootpass"@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
sudo sql "$dburl" "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
}
add_server_to_hosts() {
@ -69,7 +123,8 @@ add_server_to_hosts() {
}
shellsplus() {
shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh posh rc sash yash nopathbash nopathcsh"
shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh"
shells="$shells posh rc sash yash nopathbash nopathcsh"
shellsplus="parallel $shells"
parallel -k echo ::: $shellsplus
}
@ -93,7 +148,9 @@ create_shell_logins() {
}
SSHPASS=`goodpasswd`
export SSHPASS
append-if-not-exists /etc/shells $(which $shell || which ${shell#"nopath"})
append-if-not-exists /etc/shells $(which $shell ||
which ${shell#"nopath"})
sudo killall -u $shell
sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM
sudo groupdel $shell
if echo $shell | grep -q parallel; then
@ -112,7 +169,8 @@ create_shell_logins() {
echo '# (Re-)create user'
# Racecondition: if multiple adds a group it will the same group ID
shellsplus | parallel --timeout 15 --retries 5 --tag -j1 del_add_user ||
shellsplus |
parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user ||
(echo Creation failed: $?; false)
}
@ -120,15 +178,19 @@ copy_ssh_keys() {
make_sshkey() {
shell="$1"
echo Add server keys for lo and server &&
ssh $shell@lo 'rm -f .ssh/id_rsa; ssh-keyscan lo >>.ssh/known_hosts; ssh-keyscan server >> .ssh/known_hosts' &&
ssh $shell@lo 'rm -f .ssh/id_rsa;
ssh-keyscan lo >>.ssh/known_hosts;
ssh-keyscan server >> .ssh/known_hosts' &&
echo Do ssh-keygen &&
echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa &&
echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' &&
ssh $shell@lo 'cat .ssh/id_rsa.pub | tee -a .ssh/authorized_keys' |
ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" &&
echo Do env_parallel --install &&
ssh $shell@lo 'mkdir -p .parallel; touch .parallel/will-cite; env_parallel --install' &&
ssh $shell@lo 'mkdir -p .parallel;
touch .parallel/will-cite;
env_parallel --install' &&
echo OK make_sshkey $shell &&
echo >&2 &&
echo OK make_sshkey $shell >&2 &&
@ -155,9 +217,11 @@ copy_ssh_keys() {
export -f ssh_copy_id
echo '# copy id from any X to any Y'
parallel -u --bar -j3 --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
parallel -u --bar -j3 --timeout 3 --retries 10 --tag \
ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
echo '# Test the copying went well'
parallel --bar -j2 --timeout 9 --retries 10 --tag ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
parallel --bar -j2 --timeout 9 --retries 10 --tag \
ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
echo '# change paths to no path'
(
@ -183,16 +247,21 @@ lsh_setup() {
cd
mkdir -p .lsh
lsh-make-seed -o ".lsh/yarrow-seed-file"
lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls lo echo Added host-auth
lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls localhost echo Added host-auth
lsh -c aes256-ctr --sloppy-host-authentication \
--capture-to ~/.lsh/host-acls lo echo Added host-auth
lsh -c aes256-ctr --sloppy-host-authentication \
--capture-to ~/.lsh/host-acls localhost echo Added host-auth
lsh-keygen | lsh-writekey -c none
lsh-export-key --openssh < ~/.lsh/identity.pub | lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys'
lsh-export-key --openssh < ~/.lsh/identity.pub | ssh csh@lo 'cat >>.ssh/authorized_keys'
lsh-export-key --openssh < ~/.lsh/identity.pub |
lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys'
lsh-export-key --openssh < ~/.lsh/identity.pub |
ssh csh@lo 'cat >>.ssh/authorized_keys'
}
add_freebsd() {
echo "# Add public key to freebsd7.t"
ssh freebsd7.t cat .ssh/id_rsa.pub | ssh parallel@localhost 'cat >>.ssh/authorized_keys'
ssh freebsd7.t cat .ssh/id_rsa.pub |
ssh parallel@localhost 'cat >>.ssh/authorized_keys'
echo Add:
echo HostkeyAlgorithms +ssh-dss
@ -204,7 +273,8 @@ add_freebsd() {
echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1
echo
echo If you get:
echo Unable to negotiate with server port 22: no matching cipher found. Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
echo Unable to negotiate with server port 22: no matching cipher found.
echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
echo add this to .ssh/config
echo Ciphers +aes256-cbc
}
@ -247,7 +317,8 @@ bash_versions() {
export -f make_one
echo '# Building bash'
(cd bash; git tag | grep -v -- '-.*-') |
stdout parallel --lb --tag '/usr/local/bin/{} --version || make_one {}'
stdout parallel --lb --tag \
'/usr/local/bin/{} --version || make_one {}'
(cd bash; git tag | grep -v -- '-.*-') |
parallel -k -v --tag '/usr/local/bin/{} --version'
)
@ -263,6 +334,7 @@ misc() {
run() {
install_packages &&
install_oracle_client &&
setup_databases &&
add_server_to_hosts &&
create_shell_logins &&

View file

@ -214,7 +214,8 @@ par_compress_prg_fails() {
echo $?) | tail -n1
}
export -f doit
parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files
stdout parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files |
grep -v -- -dc
}
par_pxz_complains() {

View file

@ -0,0 +1,74 @@
#!/bin/bash
. `which env_parallel.bash`
env_parallel --session
par_many_var() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 6000`; do eval "export a$a=1" ; done
gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_var_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 5000`; do eval "export a$a=1" ; done
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
for a in `seq 5000`; do eval export -f a$a ; done
gen500k | stdout parallel --load 21 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
for a in `seq 5000`; do eval export -f a$a ; done
gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_big_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`seq 1000`
for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
for a in `seq 50`; do eval export -f a$a ; done
gen500k | stdout parallel --load 3 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_var_big_func() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`seq 1000`
for a in `seq 5000`; do eval "export a$a=1" ; done
for a in `seq 10`; do eval "a$a() { '$big'; }" ; done
for a in `seq 10`; do eval export -f a$a ; done
gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_big_func_name() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`perl -e print\"x\"x10000`
for a in `seq 10`; do eval "export a$big$a=1" ; done
gen500k | stdout parallel --load 7 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_big_var_func_name() {
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
big=`perl -e print\"x\"x10000`
for a in `seq 10`; do eval "export a$big$a=1" ; done
for a in `seq 10`; do eval "a$big$a() { 1; }" ; done
for a in `seq 10`; do eval export -f a$big$a ; done
gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
scp /usr/local/bin/parallel macosx.p:bin/
export -f $(compgen -A function | grep par_)
#compgen -A function |
compgen -A function |
grep par_ |
LC_ALL=C sort |
env_parallel --timeout 3000% --tag -k -S macosx.p

View file

@ -31,26 +31,31 @@ export -f destroy_one
server_list() {
cat <<SSHOK
generic/arch.98
generic/centos6.6
generic/centos7.7
generic/centos8.8
generic/debian8.18
generic/debian9.19
generic/debian10.30
generic/devuan3.43
generic/freebsd11.71
generic/freebsd12.72
generic/gentoo.99
generic/netbsd9.89
generic/oracle7.127
generic/rhel6.106
generic/rhel7.107
generic/rhel8.108
generic/ubuntu1604.216
generic/ubuntu1804.218
generic/ubuntu2004.220
grep -v '#' <<SSHOK
#generic/arch.98
hfm4/centos4.4
hfm4/centos5.5
#generic/centos6.6
#generic/centos7.7
#generic/centos8.8
MarcinOrlowski/debian4-i386.14
twolfman/debian6-lamp-drush.16
puphpet/debian75-x64.17
#generic/debian8.18
#generic/debian9.19
#generic/debian10.30
#generic/devuan3.43
#generic/freebsd11.71
#generic/freebsd12.72
#generic/gentoo.99
#generic/netbsd9.89
#generic/oracle7.127
#generic/rhel6.106
#generic/rhel7.107
#generic/rhel8.108
#generic/ubuntu1604.216
#generic/ubuntu1804.218
#generic/ubuntu2004.220
SSHOK
# Ignore for now

View file

@ -0,0 +1,62 @@
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 1100 26340
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2380 57060
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2412 57828
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 1580 37860
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 1760 42180
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 1228 29412
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 1580 37860