mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
This commit is contained in:
Ole Tange
parent
79b56269bb
commit
72dd439f20
81
src/parallel
81
src/parallel
|
|
@ -6249,25 +6249,30 @@ sub memfree_recompute() {
|
|||
# Buffers: 19876 kB
|
||||
# Cached: 431192 kB
|
||||
# SwapCached: 0 kB
|
||||
"linux" =>
|
||||
q[ print 1024 * qx{ ].
|
||||
q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ].
|
||||
q[ { sum += \$2} END { print sum }' ].
|
||||
q[ /proc/meminfo } ],
|
||||
"linux" => (
|
||||
q{
|
||||
print 1024 * qx{
|
||||
awk '/^((Swap)?Cached|MemFree|Buffers):/
|
||||
{ sum += \$2} END { print sum }'
|
||||
/proc/meminfo }
|
||||
}),
|
||||
# Android uses same code as GNU/Linux
|
||||
"android" =>
|
||||
q[ print 1024 * qx{ ].
|
||||
q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ].
|
||||
q[ { sum += \$2} END { print sum }' ].
|
||||
q[ /proc/meminfo } ],
|
||||
|
||||
"android" => (
|
||||
q{
|
||||
print 1024 * qx{
|
||||
awk '/^((Swap)?Cached|MemFree|Buffers):/
|
||||
{ sum += \$2} END { print sum }'
|
||||
/proc/meminfo }
|
||||
}),
|
||||
# $ vmstat 1 1
|
||||
# procs memory page faults cpu
|
||||
# r b w avm free re at pi po fr de sr in sy cs us sy id
|
||||
# 1 0 0 242793 389737 5 1 0 0 0 0 0 107 978 60 1 1 99
|
||||
"hpux" =>
|
||||
q[ print (((reverse `vmstat 1 1`)[0] ].
|
||||
q[ =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ],
|
||||
"hpux" => (
|
||||
q{
|
||||
print (((reverse `vmstat 1 1`)[0]
|
||||
=~ /(?:\d+\D+){4}(\d+)/)[0]*1024)
|
||||
}),
|
||||
# $ vmstat 1 2
|
||||
# kthr memory page disk faults cpu
|
||||
# r b w swap free re mf pi po fr de sr s3 s4 -- -- in sy cs us sy id
|
||||
|
|
@ -6275,10 +6280,17 @@ sub memfree_recompute() {
|
|||
# 0 0 0 6434088 5072656 7 15 8 0 0 0 0 0 261 0 0 1889 1899 3222 0 8 92
|
||||
#
|
||||
# The second free value is correct
|
||||
"solaris" =>
|
||||
q[ print (((reverse `vmstat 1 2`)[0] ].
|
||||
q[ =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ],
|
||||
"freebsd" => q{
|
||||
"solaris" => (
|
||||
q{
|
||||
print (((reverse `vmstat 1 2`)[0]
|
||||
=~ /(?:\d+\D+){4}(\d+)/)[0]*1024)
|
||||
}),
|
||||
# hw.pagesize: 4096
|
||||
# vm.stats.vm.v_cache_count: 0
|
||||
# vm.stats.vm.v_inactive_count: 79574
|
||||
# vm.stats.vm.v_free_count: 4507
|
||||
"freebsd" => (
|
||||
q{
|
||||
for(qx{/sbin/sysctl -a}) {
|
||||
if (/^([^:]+):\s+(.+)\s*$/s) {
|
||||
$sysctl->{$1} = $2;
|
||||
|
|
@ -6288,7 +6300,7 @@ sub memfree_recompute() {
|
|||
($sysctl->{"vm.stats.vm.v_cache_count"}
|
||||
+ $sysctl->{"vm.stats.vm.v_inactive_count"}
|
||||
+ $sysctl->{"vm.stats.vm.v_free_count"});
|
||||
},
|
||||
}),
|
||||
# Mach Virtual Memory Statistics: (page size of 4096 bytes)
|
||||
# Pages free: 198061.
|
||||
# Pages active: 159701.
|
||||
|
|
@ -6302,12 +6314,13 @@ sub memfree_recompute() {
|
|||
# Pageins: 1798068.
|
||||
# Pageouts: 257.
|
||||
# Object cache: 6603 hits of 1713223 lookups (0% hit rate)
|
||||
'darwin' =>
|
||||
q[ $vm = `vm_stat`;
|
||||
'darwin' => (
|
||||
q{
|
||||
$vm = `vm_stat`;
|
||||
print (($vm =~ /page size of (\d+)/)[0] *
|
||||
(($vm =~ /Pages free:\s+(\d+)/)[0] +
|
||||
($vm =~ /Pages inactive:\s+(\d+)/)[0]));
|
||||
],
|
||||
}),
|
||||
);
|
||||
my $perlscript = "";
|
||||
# Make a perl script that detects the OS ($^O) and runs
|
||||
|
|
@ -6315,8 +6328,7 @@ sub memfree_recompute() {
|
|||
for my $os (keys %script_of) {
|
||||
$perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}';
|
||||
}
|
||||
$perlscript =~ s/[\t\n ]+/ /g;
|
||||
$script = "perl -e " . ::Q($perlscript);
|
||||
$script = "perl -e " . ::Q(::spacefree(1,$perlscript));
|
||||
}
|
||||
return $script;
|
||||
}
|
||||
|
|
@ -10469,6 +10481,7 @@ sub slot($) {
|
|||
|
||||
{
|
||||
my $already_spread;
|
||||
my $env_size;
|
||||
|
||||
sub populate($) {
|
||||
# Add arguments from arg_queue until the number of arguments or
|
||||
|
|
@ -10486,7 +10499,25 @@ sub slot($) {
|
|||
my $next_arg;
|
||||
my $max_len = $Global::minimal_command_line_length
|
||||
|| Limits::Command::max_length();
|
||||
|
||||
if($^O eq "darwin") {
|
||||
# On Mac bash 3.2.48(1)-release the length also depends on
|
||||
# number of functions, variables and the size of those.
|
||||
# 21 is found by trial and error and seems to be a safe value.
|
||||
# test_with() {
|
||||
# gen500k() {
|
||||
# seq -f %f 1000000000000000 1000000000050000 |
|
||||
# head -c 131000;
|
||||
# }
|
||||
# for a in `seq 5100`; do eval "export a$a=1" ; done;
|
||||
# for a in `seq 5100`; do eval "a$a() { 1; }" ; done;
|
||||
# for a in `seq 5100`; do eval export -f a$a ; done;
|
||||
# gen500k | stdout parallel -Xj1 'echo {} {} {} {} | wc' |
|
||||
# perl -pe 's/\d{10,} //g'
|
||||
# }
|
||||
$env_size ||= (keys %ENV)*21
|
||||
+ length(join'',keys %ENV) + length(join'',%ENV);
|
||||
$max_len -= $env_size;
|
||||
}
|
||||
if($opt::cat or $opt::fifo) {
|
||||
# Get the empty arg added by --pipepart (if any)
|
||||
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();
|
||||
|
|
|
|||
|
|
@ -2616,6 +2616,85 @@ https://pypi.org/project/papply/ (Last checked: 2020-04)
|
|||
|
||||
=head2 Todo
|
||||
|
||||
test_many_var() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
for a in `seq 11000`; do eval "export a$a=1" ; done
|
||||
gen500k | stdout parallel --timeout 5 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
test_many_var_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
for a in `seq 5100`; do eval "export a$a=1" ; done
|
||||
for a in `seq 5100`; do eval "a$a() { 1; }" ; done
|
||||
for a in `seq 5100`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --timeout 21 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
test_many_var_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
for a in `seq 8000`; do eval "a$a() { 1; }" ; done
|
||||
for a in `seq 8000`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --timeout 6 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
test_big_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`seq 1000`
|
||||
for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
|
||||
for a in `seq 50`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --timeout 4 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
test_many_var_big_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`seq 1000`
|
||||
for a in `seq 5100`; do eval "export a$a=1" ; done
|
||||
for a in `seq 20`; do eval "a$a() { '$big'; }" ; done
|
||||
for a in `seq 20`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --timeout 6 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
test_big_func_name() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`perl -e print\"x\"x10000`
|
||||
for a in `seq 20`; do eval "export a$big$a=1" ; done
|
||||
gen500k | stdout parallel --timeout 8 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
test_big_var_func_name() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`perl -e print\"x\"x10000`
|
||||
for a in `seq 2`; do eval "export a$big$a=1" ; done
|
||||
for a in `seq 2`; do eval "a$big$a() { '$big'; }" ; done
|
||||
for a in `seq 2`; do eval export -f a$big$a ; done
|
||||
gen500k | stdout parallel --timeout 1000 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
}
|
||||
|
||||
|
||||
|
||||
tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1 'echo {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1 'echo {} {} | wc' | perl -pe 's/\d{3,5} //g'
|
||||
tange@macosx:~$ for a in `seq 100`; do eval export -f a$a ; done
|
||||
|
||||
|
||||
seq 100000 | stdout parallel -Xj1 'echo {} {} | wc'
|
||||
export a=`seq 10000`
|
||||
seq 100000 | stdout parallel -Xj1 'echo {} {} | wc'
|
||||
|
||||
|
||||
|
||||
my $already_spread;
|
||||
my $env_size;
|
||||
|
||||
if($^O eq "darwin") {
|
||||
$env_size ||= 500+length(join'',%ENV);
|
||||
$max_len -= $env_size;
|
||||
}
|
||||
|
||||
|
||||
PASH: Light-touch Data-Parallel Shell Processing
|
||||
https://arxiv.org/pdf/2007.09436.pdf
|
||||
|
||||
https://gitlab.com/netikras/bthread
|
||||
|
||||
https://github.com/JeiKeiLim/simple_distribute_job
|
||||
|
|
|
|||
|
|
@ -6,26 +6,33 @@ install_packages() {
|
|||
INSTALL=${INSTALL:-"sudo aptitude -y install"}
|
||||
|
||||
# The testsuite depends on this:
|
||||
test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl pod2pdf gawk"
|
||||
test_pkgs="$test_pkgs lua5.3 clisp php-cli nodejs mono-csharp-shell libevent-dev"
|
||||
test_pkgs="$test_pkgs tcl libtext-csv-perl xterm libc6-i386 libcrypt1:i386"
|
||||
test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl"
|
||||
test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs"
|
||||
test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl"
|
||||
test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386"
|
||||
test_pkgs="$test_pkgs libtest-nowarnings-perl"
|
||||
|
||||
# DEBIAN package
|
||||
packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15 python3-m2crypto"
|
||||
packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15"
|
||||
packaging_pkgs="$packaging_pkgs python3-m2crypto alien"
|
||||
# SHEBANG TOOLS
|
||||
shebang_pkgs="gnuplot octave ruby r-base-core"
|
||||
# SQL TOOLS
|
||||
sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl libdbd-mysql-perl rlwrap"
|
||||
sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
|
||||
sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
|
||||
# Compression
|
||||
compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2 lzma xz-utils lzip bzip2 lbzip2 lrzip"
|
||||
compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2"
|
||||
compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip"
|
||||
compression_pkgs_missing="pxz"
|
||||
# Shells
|
||||
# (csh = bsd-csh that is broken)
|
||||
shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash tcsh yash zsh"
|
||||
shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash"
|
||||
shell_pkgs="$shell_pkgs tcsh yash zsh"
|
||||
# Databases
|
||||
database_pkgs="postgresql mysql-server sqlite"
|
||||
sudo dpkg --add-architecture i386
|
||||
if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs $compression_pkgs $shell_pkgs $database_pkgs; then
|
||||
sudo dpkg --add-architecture i386; sudo apt update
|
||||
if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \
|
||||
$compression_pkgs $shell_pkgs $database_pkgs; then
|
||||
# OK
|
||||
true
|
||||
else
|
||||
|
|
@ -35,6 +42,53 @@ install_packages() {
|
|||
sudo apt update &
|
||||
}
|
||||
|
||||
install_oracle_client() {
|
||||
# https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian
|
||||
# sql oracle:// && return 0
|
||||
(cd /tmp
|
||||
|
||||
get_rpm_install_deb() {
|
||||
glob="$1"
|
||||
url="$oracleurl/$2"
|
||||
testglob="$3"
|
||||
if [ ! -e `echo $glob*rpm` ] ; then
|
||||
wget $url
|
||||
fi
|
||||
if [ ! -e `echo $glob*deb` ] ; then
|
||||
echo Convert `echo $glob*rpm` to deb
|
||||
fakeroot alien `echo $glob*rpm`
|
||||
fi
|
||||
if [ ! -e `echo $testglob` ] ; then
|
||||
echo Install `echo $glob*deb`
|
||||
sudo dpkg -i `echo $glob*deb`
|
||||
fi
|
||||
}
|
||||
|
||||
oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600
|
||||
client=oracle-instantclient19.6
|
||||
ver=19.6.0.0.0-1.x86_64.rpm
|
||||
get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright
|
||||
get_rpm_install_deb 'oracle-instantclient*basic' $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README
|
||||
get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus
|
||||
)
|
||||
|
||||
echo Add this to .bashrc:
|
||||
ORACLE_HOME=`echo /usr/lib/oracle/*/client*`
|
||||
echo ORACLE_HOME=$ORACLE_HOME
|
||||
if grep -q ORACLE_HOME=/ ~/.bashrc; then
|
||||
perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc
|
||||
else
|
||||
echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc
|
||||
echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc
|
||||
echo 'export ORACLE_HOME' >> ~/.bashrc
|
||||
echo 'export ORACLE_SID=XE' >> ~/.bashrc
|
||||
fi
|
||||
perl -MCPAN -e 'install DBD::Oracle'
|
||||
# TODO set up vagrant oracle server
|
||||
# TODO set up default passwd
|
||||
# test it works: sql oracle://
|
||||
}
|
||||
|
||||
setup_databases() {
|
||||
# DATABASES
|
||||
echo '# Create PostgreSQL'
|
||||
|
|
@ -46,15 +100,15 @@ setup_databases() {
|
|||
sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\""
|
||||
|
||||
mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E}
|
||||
|
||||
dburl=mysql://root:"$mysqlrootpass"@/mysql
|
||||
echo '# Create MySQL'
|
||||
sudo su mysql mysqladmin create `whoami`
|
||||
# Default Debian way of getting "root" access
|
||||
sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';"
|
||||
|
||||
# Drop database and user if needed
|
||||
sudo sql mysql://root:"$mysqlrootpass"@/mysql "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
|
||||
sudo sql mysql://root:"$mysqlrootpass"@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
|
||||
sudo sql "$dburl" "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
|
||||
sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
|
||||
}
|
||||
|
||||
add_server_to_hosts() {
|
||||
|
|
@ -69,7 +123,8 @@ add_server_to_hosts() {
|
|||
}
|
||||
|
||||
shellsplus() {
|
||||
shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh posh rc sash yash nopathbash nopathcsh"
|
||||
shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh"
|
||||
shells="$shells posh rc sash yash nopathbash nopathcsh"
|
||||
shellsplus="parallel $shells"
|
||||
parallel -k echo ::: $shellsplus
|
||||
}
|
||||
|
|
@ -93,7 +148,9 @@ create_shell_logins() {
|
|||
}
|
||||
SSHPASS=`goodpasswd`
|
||||
export SSHPASS
|
||||
append-if-not-exists /etc/shells $(which $shell || which ${shell#"nopath"})
|
||||
append-if-not-exists /etc/shells $(which $shell ||
|
||||
which ${shell#"nopath"})
|
||||
sudo killall -u $shell
|
||||
sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM
|
||||
sudo groupdel $shell
|
||||
if echo $shell | grep -q parallel; then
|
||||
|
|
@ -112,7 +169,8 @@ create_shell_logins() {
|
|||
|
||||
echo '# (Re-)create user'
|
||||
# Racecondition: if multiple adds a group it will the same group ID
|
||||
shellsplus | parallel --timeout 15 --retries 5 --tag -j1 del_add_user ||
|
||||
shellsplus |
|
||||
parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user ||
|
||||
(echo Creation failed: $?; false)
|
||||
}
|
||||
|
||||
|
|
@ -120,7 +178,9 @@ copy_ssh_keys() {
|
|||
make_sshkey() {
|
||||
shell="$1"
|
||||
echo Add server keys for lo and server &&
|
||||
ssh $shell@lo 'rm -f .ssh/id_rsa; ssh-keyscan lo >>.ssh/known_hosts; ssh-keyscan server >> .ssh/known_hosts' &&
|
||||
ssh $shell@lo 'rm -f .ssh/id_rsa;
|
||||
ssh-keyscan lo >>.ssh/known_hosts;
|
||||
ssh-keyscan server >> .ssh/known_hosts' &&
|
||||
echo Do ssh-keygen &&
|
||||
echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa &&
|
||||
echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' &&
|
||||
|
|
@ -128,7 +188,9 @@ copy_ssh_keys() {
|
|||
ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
|
||||
ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" &&
|
||||
echo Do env_parallel --install &&
|
||||
ssh $shell@lo 'mkdir -p .parallel; touch .parallel/will-cite; env_parallel --install' &&
|
||||
ssh $shell@lo 'mkdir -p .parallel;
|
||||
touch .parallel/will-cite;
|
||||
env_parallel --install' &&
|
||||
echo OK make_sshkey $shell &&
|
||||
echo >&2 &&
|
||||
echo OK make_sshkey $shell >&2 &&
|
||||
|
|
@ -155,9 +217,11 @@ copy_ssh_keys() {
|
|||
export -f ssh_copy_id
|
||||
|
||||
echo '# copy id from any X to any Y'
|
||||
parallel -u --bar -j3 --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
|
||||
parallel -u --bar -j3 --timeout 3 --retries 10 --tag \
|
||||
ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
|
||||
echo '# Test the copying went well'
|
||||
parallel --bar -j2 --timeout 9 --retries 10 --tag ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
|
||||
parallel --bar -j2 --timeout 9 --retries 10 --tag \
|
||||
ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
|
||||
|
||||
echo '# change paths to no path'
|
||||
(
|
||||
|
|
@ -183,16 +247,21 @@ lsh_setup() {
|
|||
cd
|
||||
mkdir -p .lsh
|
||||
lsh-make-seed -o ".lsh/yarrow-seed-file"
|
||||
lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls lo echo Added host-auth
|
||||
lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls localhost echo Added host-auth
|
||||
lsh -c aes256-ctr --sloppy-host-authentication \
|
||||
--capture-to ~/.lsh/host-acls lo echo Added host-auth
|
||||
lsh -c aes256-ctr --sloppy-host-authentication \
|
||||
--capture-to ~/.lsh/host-acls localhost echo Added host-auth
|
||||
lsh-keygen | lsh-writekey -c none
|
||||
lsh-export-key --openssh < ~/.lsh/identity.pub | lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys'
|
||||
lsh-export-key --openssh < ~/.lsh/identity.pub | ssh csh@lo 'cat >>.ssh/authorized_keys'
|
||||
lsh-export-key --openssh < ~/.lsh/identity.pub |
|
||||
lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys'
|
||||
lsh-export-key --openssh < ~/.lsh/identity.pub |
|
||||
ssh csh@lo 'cat >>.ssh/authorized_keys'
|
||||
}
|
||||
|
||||
add_freebsd() {
|
||||
echo "# Add public key to freebsd7.t"
|
||||
ssh freebsd7.t cat .ssh/id_rsa.pub | ssh parallel@localhost 'cat >>.ssh/authorized_keys'
|
||||
ssh freebsd7.t cat .ssh/id_rsa.pub |
|
||||
ssh parallel@localhost 'cat >>.ssh/authorized_keys'
|
||||
|
||||
echo Add:
|
||||
echo HostkeyAlgorithms +ssh-dss
|
||||
|
|
@ -204,7 +273,8 @@ add_freebsd() {
|
|||
echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1
|
||||
echo
|
||||
echo If you get:
|
||||
echo Unable to negotiate with server port 22: no matching cipher found. Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
|
||||
echo Unable to negotiate with server port 22: no matching cipher found.
|
||||
echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
|
||||
echo add this to .ssh/config
|
||||
echo Ciphers +aes256-cbc
|
||||
}
|
||||
|
|
@ -247,7 +317,8 @@ bash_versions() {
|
|||
export -f make_one
|
||||
echo '# Building bash'
|
||||
(cd bash; git tag | grep -v -- '-.*-') |
|
||||
stdout parallel --lb --tag '/usr/local/bin/{} --version || make_one {}'
|
||||
stdout parallel --lb --tag \
|
||||
'/usr/local/bin/{} --version || make_one {}'
|
||||
(cd bash; git tag | grep -v -- '-.*-') |
|
||||
parallel -k -v --tag '/usr/local/bin/{} --version'
|
||||
)
|
||||
|
|
@ -263,6 +334,7 @@ misc() {
|
|||
|
||||
run() {
|
||||
install_packages &&
|
||||
install_oracle_client &&
|
||||
setup_databases &&
|
||||
add_server_to_hosts &&
|
||||
create_shell_logins &&
|
||||
|
|
|
|||
|
|
@ -214,7 +214,8 @@ par_compress_prg_fails() {
|
|||
echo $?) | tail -n1
|
||||
}
|
||||
export -f doit
|
||||
parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files
|
||||
stdout parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files |
|
||||
grep -v -- -dc
|
||||
}
|
||||
|
||||
par_pxz_complains() {
|
||||
|
|
|
|||
74
testsuite/tests-to-run/parallel-macos.sh
Normal file
74
testsuite/tests-to-run/parallel-macos.sh
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#!/bin/bash
|
||||
|
||||
. `which env_parallel.bash`
|
||||
env_parallel --session
|
||||
|
||||
par_many_var() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
for a in `seq 6000`; do eval "export a$a=1" ; done
|
||||
gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
par_many_var_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
for a in `seq 5000`; do eval "export a$a=1" ; done
|
||||
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
|
||||
for a in `seq 5000`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --load 21 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
par_many_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
|
||||
for a in `seq 5000`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
par_big_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`seq 1000`
|
||||
for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
|
||||
for a in `seq 50`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --load 3 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
par_many_var_big_func() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`seq 1000`
|
||||
for a in `seq 5000`; do eval "export a$a=1" ; done
|
||||
for a in `seq 10`; do eval "a$a() { '$big'; }" ; done
|
||||
for a in `seq 10`; do eval export -f a$a ; done
|
||||
gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
par_big_func_name() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`perl -e print\"x\"x10000`
|
||||
for a in `seq 10`; do eval "export a$big$a=1" ; done
|
||||
gen500k | stdout parallel --load 7 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
par_big_var_func_name() {
|
||||
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
|
||||
big=`perl -e print\"x\"x10000`
|
||||
for a in `seq 10`; do eval "export a$big$a=1" ; done
|
||||
for a in `seq 10`; do eval "a$big$a() { 1; }" ; done
|
||||
for a in `seq 10`; do eval export -f a$big$a ; done
|
||||
gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' |
|
||||
perl -pe 's/\d{10,}.\d+ //g'
|
||||
}
|
||||
|
||||
scp /usr/local/bin/parallel macosx.p:bin/
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
#compgen -A function |
|
||||
compgen -A function |
|
||||
grep par_ |
|
||||
LC_ALL=C sort |
|
||||
env_parallel --timeout 3000% --tag -k -S macosx.p
|
||||
|
|
@ -31,26 +31,31 @@ export -f destroy_one
|
|||
|
||||
|
||||
server_list() {
|
||||
cat <<SSHOK
|
||||
generic/arch.98
|
||||
generic/centos6.6
|
||||
generic/centos7.7
|
||||
generic/centos8.8
|
||||
generic/debian8.18
|
||||
generic/debian9.19
|
||||
generic/debian10.30
|
||||
generic/devuan3.43
|
||||
generic/freebsd11.71
|
||||
generic/freebsd12.72
|
||||
generic/gentoo.99
|
||||
generic/netbsd9.89
|
||||
generic/oracle7.127
|
||||
generic/rhel6.106
|
||||
generic/rhel7.107
|
||||
generic/rhel8.108
|
||||
generic/ubuntu1604.216
|
||||
generic/ubuntu1804.218
|
||||
generic/ubuntu2004.220
|
||||
grep -v '#' <<SSHOK
|
||||
#generic/arch.98
|
||||
hfm4/centos4.4
|
||||
hfm4/centos5.5
|
||||
#generic/centos6.6
|
||||
#generic/centos7.7
|
||||
#generic/centos8.8
|
||||
MarcinOrlowski/debian4-i386.14
|
||||
twolfman/debian6-lamp-drush.16
|
||||
puphpet/debian75-x64.17
|
||||
#generic/debian8.18
|
||||
#generic/debian9.19
|
||||
#generic/debian10.30
|
||||
#generic/devuan3.43
|
||||
#generic/freebsd11.71
|
||||
#generic/freebsd12.72
|
||||
#generic/gentoo.99
|
||||
#generic/netbsd9.89
|
||||
#generic/oracle7.127
|
||||
#generic/rhel6.106
|
||||
#generic/rhel7.107
|
||||
#generic/rhel8.108
|
||||
#generic/ubuntu1604.216
|
||||
#generic/ubuntu1804.218
|
||||
#generic/ubuntu2004.220
|
||||
SSHOK
|
||||
|
||||
# Ignore for now
|
||||
|
|
|
|||
62
testsuite/wanted-results/parallel-macos
Normal file
62
testsuite/wanted-results/parallel-macos
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 2592 62208
|
||||
par_big_func 1 1100 26340
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2432 58368
|
||||
par_big_func_name 1 2380 57060
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2428 58272
|
||||
par_big_var_func_name 1 2412 57828
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 2532 60768
|
||||
par_many_func 1 1580 37860
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 2868 68832
|
||||
par_many_var 1 1760 42180
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 2576 61824
|
||||
par_many_var_big_func 1 1228 29412
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 2532 60768
|
||||
par_many_var_func 1 1580 37860
|
||||
Loading…
Reference in a new issue