#!/bin/bash # SPDX-FileCopyrightText: 2002-2023 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. # # SPDX-License-Identifier: GPL-3.0-or-later install_packages() { # After first run, set this: # export INSTALL=echo INSTALL=${INSTALL:-"sudo apt -y install"} # The testsuite depends on this: test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl" test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs" test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl" test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386 vagrant virtualbox" test_pkgs="$test_pkgs libtest-nowarnings-perl xemacs21 pv zenity" # Debian package packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake" packaging_pkgs="$packaging_pkgs python3-m2crypto alien python3-pip" # Shebang Tools shebang_pkgs="gnuplot octave ruby r-base-core" # SQL Tools sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl" sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap" # Compression compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2" compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip" compression_pkgs_missing="pxz" # Shells # (csh = bsd-csh that is broken) # (ksh93 gone) shell_pkgs="ash dash fdclone fish fizsh ksh ksh93u+m mksh posh rc rush sash" shell_pkgs="$shell_pkgs tcsh yash zsh busybox-static" # Databases database_pkgs="postgresql mysql-server sqlite influxdb influxdb-client" # Build Tools build_pkgs="bison libxxhash-dev libzstd-dev liblz4-dev libssl-dev" build_pkgs="$build_pkgs python3-cmarkgfm libpod-pom-view-restructured-perl" build_pkgs="$build_pkgs graphviz" sudo dpkg --add-architecture i386; sudo apt update if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \ $compression_pkgs $shell_pkgs $database_pkgs $build_pkgs ; then # OK true else echo Packages missing return 1 fi pip install sphinx-rtd-theme sudo apt update & } install_oracle_client() { # https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian # sql oracle:// && return 0 (cd /tmp get_rpm_install_deb() { glob="$1" url="$oracleurl/$2" testglob="$3" if [ ! -e `echo $glob*rpm` ] ; then wget $url fi if [ ! -e `echo $glob*deb` ] ; then echo Convert `echo $glob*rpm` to deb fakeroot alien `echo $glob*rpm` fi if [ ! -e `echo $testglob` ] ; then echo Install `echo $glob*deb` sudo dpkg -i `echo $glob*deb` fi } oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600 client=oracle-instantclient19.6 ver=19.6.0.0.0-1.x86_64.rpm get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright get_rpm_install_deb 'oracle-instantclient*basic' $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus ) echo Add this to .bashrc: ORACLE_HOME=`echo /usr/lib/oracle/*/client*` echo ORACLE_HOME=$ORACLE_HOME if grep -q ORACLE_HOME=/ ~/.bashrc; then perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc else echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc echo 'export ORACLE_HOME' >> ~/.bashrc echo 'export ORACLE_SID=XE' >> ~/.bashrc fi perl -MCPAN -e 'install DBD::Oracle' # TODO set up vagrant oracle server git clone https://github.com/oracle/vagrant-projects.git # TODO set up default passwd echo sudo su - oracle -c "'/home/oracle/setPassword.sh $oracle_password'" | vagrant ssh # test it works: sql oracle:// } setup_virtual_box_network() { echo "Adding IP-range to virtualbox" sudo mkdir -p /etc/vbox (cat; echo '* 172.27.27.0/24') < /etc/vbox/networks.conf | uniq | sudo tee /etc/vbox/networks.conf } setup_databases() { # DATABASES echo '# Create PostgreSQL' sudo em /etc/postgresql/*/main/postgresql.conf # listen_addresses = '*' sudo service postgresql restart # Drop database and user if needed sudo su - postgres -c 'dropdb '`whoami` sudo su - postgres -c 'dropuser '`whoami` sudo su - postgres -c 'createdb '`whoami` sudo su - postgres -c 'createuser '`whoami` sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\"" mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E} dburl=mysql://root:"$mysqlrootpass"@/mysql echo '# Create MySQL' sudo su mysql mysqladmin create `whoami` # Default Debian way of getting "root" access sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';" # Drop database and user if needed sudo sql "$dburl" "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';" sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';" } add_server_to_hosts() { add_ssh_key_to_authorized() { (cat vagrant/authorized_keys; cat ~/.ssh/*.pub) | uniq > vagrant/authorized_keys.$$ mv vagrant/authorized_keys.$$ vagrant/authorized_keys # Fix: # could not settle on kex algorithm # Server kex preferences: diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1 # Client kex preferences: ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1> #debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,sntrup761x25519-sha512@openssh.com,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group1-sha1,ext-info-c #debug2: host key algorithms: ssh-dss,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 #debug2: ciphers ctos: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc #debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc #debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 #debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 # } insert_in_etc_hosts() { ip=$1 host=$2 if grep -q $1 /etc/hosts && grep -q $2 /etc/hosts ; then # skip true else printf "$1\t$2\n" | sudo tee -a /etc/hosts fi } export -f insert_in_etc_hosts add_single_vagrant_to_etc_hosts() { vagrant_ip() { cd vagrant/*/$1 # .vagrant can contain dirt rm -rf .vagrant grep 172 Vagrantfile | grep -v '#' | perl -pe 's/.*(172.27[\.0-9]+).*/$1/'; } insert_in_etc_hosts $(vagrant_ip $1) $1 } export -f add_single_vagrant_to_etc_hosts add_ssh_key_to_authorized insert_in_etc_hosts 127.1.2.3 server insert_in_etc_hosts 127.0.0.2 lo parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817 # TODO alias # 172.27.27.8 centos8 parallel-server1 server1 # 172.27.27.72 freebsd12 parallel-server3 server3 # 172.27.27.108 rhel8 parallel-server4 server4 # 172.27.27.73 freebsd13 parallel-server2 server2 # TODO # $ cat /etc/vbox/networks.conf # * 172.27.0.0/16 # TODO sshaddvagrant() { cat ~/.ssh/*.pub | (cd vagrant/*/$1 && vagrant ssh -c 'cat >>.ssh/authorized_keys') sshpass -p vagrant ssh-copy-id -o StrictHostKeyChecking=accept-new vagrant@$1 ssh vagrant@$1 echo vagrant@$1 OK } export -f sshaddvagrant parallel sshaddvagrant ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817 parallel ssh -o StrictHostKeyChecking=accept-new vagrant@{} ::: parallel-server{1..4} centos3 # TODO find ipv6 ipv6= parallel ssh -o StrictHostKeyChecking=accept-new {} ::: lo localhost 172.27.27.1 127.0.0.1 ::1 $ipv6 # parallel sshpass -p vagrant ssh-copy-id vagrant@{} ::: parallel-server{1..4} centos3 # ssh-copy-id vagrant@parallel-server1..4 centos3 } shellsplus() { shells="bash sh csh ash dash tcsh zsh ksh ksh93 fish fizsh mksh" shells="$shells posh rc sash yash nopathbash nopathcsh" shellsplus="parallel $shells" parallel -k echo ::: $shellsplus } create_shell_logins() { # SHELLS echo '# Create shell logins' touch ~/.zshrc del_add_user() { shell="$1" append-if-not-exists() { file="$1" shift string="$@" if grep -qF "$string" "$file"; then # It is there already echo "$string is already in $file" else echo "$string" | sudo tee -a "$file" fi } SSHPASS=`goodpasswd` export SSHPASS append-if-not-exists /etc/shells $(which $shell || which ${shell#"nopath"}) sudo killall -u $shell sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM sudo groupdel $shell if echo $shell | grep -qE "parallel|withpassword"; then # User parallel+withpassword should have /bin/bash loginshell=/bin/bash else # Other users should have `which $shell` with nopath removed loginshell=$(which $shell || which ${shell#"nopath"}) fi sudo adduser --shell $loginshell --disabled-password --gecos "$shell for parallel,,," $shell && echo "$shell:$SSHPASS" | sudo chpasswd && sshpass -e ssh-copy-id $shell@lo && echo "$shell created" } export -f del_add_user echo '# (Re-)create user' # Racecondition: if multiple adds a group it will the same group ID (shellsplus; echo withpassword) | parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user || (echo Creation failed: $?; false) } copy_ssh_keys() { make_sshkey() { shell="$1" echo Add server keys for lo and server && ssh $shell@lo 'rm -f .ssh/id_rsa; ssh-keyscan lo >>.ssh/known_hosts; ssh-keyscan server >> .ssh/known_hosts' && echo Do ssh-keygen && echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa && echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' && ssh $shell@lo 'cat .ssh/id_rsa.pub | tee -a .ssh/authorized_keys' | ssh parallel@lo 'cat >> .ssh/authorized_keys' && ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" && echo Do env_parallel --install && ssh $shell@lo 'mkdir -p .parallel; touch .parallel/will-cite; env_parallel --install' && echo OK make_sshkey $shell && echo >&2 && echo OK make_sshkey $shell >&2 && echo } export -f make_sshkey echo '# Make and copy sshkey' shellsplus | parallel --timeout 20 --retries 5 --tag -j5 make_sshkey ssh_a_to_b() { ssh $1@lo ssh $2@lo echo OK || (echo failed && false) } export -f ssh_a_to_b echo "# copy id from X to X" shellsplus | parallel --timeout 10 --tag ssh_a_to_b {} {} ssh_copy_id() { from="$1" to="$2" ssh $from cat .ssh/id_rsa.pub | ssh $to 'cat >> .ssh/authorized_keys' } export -f ssh_copy_id echo '# copy id from any X to any Y' parallel -u --bar -j3 --timeout 3 --retries 10 --tag \ ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus) echo '# Test the copying went well' parallel --bar -j2 --timeout 9 --retries 10 --tag \ ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus) echo '# change paths to no path' ( ssh nopathbash@lo 'echo > .bashrc PATH=/bin:/usr/bin' ssh nopathbash@lo 'echo > .profile PATH=/bin:/usr/bin' # Remove env_parallel from .profile ssh nopathbash@lo 'perl -i.bak -pe s/.*env_parallel.*// .profile .bashrc' ssh nopathcsh@lo 'echo >> .cshrc setenv PATH /bin:/usr/bin' ssh nopathcsh@lo 'echo >> .login setenv PATH /bin:/usr/bin' ssh nopathbash@lo 'echo $PATH; echo 1 | parallel echo' || echo Great - this should FAIL: parallel should not be found ssh nopathcsh@lo 'echo $PATH; echo 1 | parallel echo' || echo Great - this should FAIL: parallel should not be found ) } lsh_setup() { ssh-keyscan localhost >>~/.ssh/known_hosts ssh-copy-id localhost # SHELLS: lsh-client against openssh server $INSTALL lsh-client cd mkdir -p .lsh lsh-make-seed -o ".lsh/yarrow-seed-file" lsh -c aes256-ctr --sloppy-host-authentication \ --capture-to ~/.lsh/host-acls lo echo Added host-auth lsh -c aes256-ctr --sloppy-host-authentication \ --capture-to ~/.lsh/host-acls localhost echo Added host-auth lsh-keygen | lsh-writekey -c none export_key_to_local_users() { lsh-export-key --openssh < ~/.lsh/identity.pub | ssh -l $1 lo 'cat >>.ssh/authorized_keys' } export -f export_key_to_local_users # lsh: Protocol error: No common key exchange method. # It seems # $ lsh --list-algorithms # Supported hostkey algorithms: ssh-dss, spki, none # # $ nmap --script ssh2-enum-algos -sV -p 22 lo # | server_host_key_algorithms: (4) # | rsa-sha2-512 # | rsa-sha2-256 # | ecdsa-sha2-nistp256 # | ssh-ed25519 # | # # There is no longer an overlap: LSH is unsupported until there is # a common algorithm again shellsplus | parallel --bar --timeout 5 export_key_to_local_users shellsplus | parallel --bar --timeout 5 'lsh -l {} lo true || export_key_to_local_users {}' shellsplus | parallel --bar --timeout 5 'lsh -l {} lo true || echo Fail {}' } add_freebsd() { echo "# Add public key to freebsd7.t" ssh freebsd7.t cat .ssh/id_rsa.pub | ssh parallel@localhost 'cat >>.ssh/authorized_keys' echo Add: echo HostkeyAlgorithms +ssh-dss echo to .ssh/config if you get echo no matching host key type found. Their offer: ssh-dss echo echo Add to /etc/ssh/sshd_config echo Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 echo echo If you get: echo Unable to negotiate with server port 22: no matching cipher found. echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se echo add this to .ssh/config echo Ciphers +aes256-cbc } tmux_versions() { (cd /tmp git clone https://github.com/tmux/tmux.git listtags() { (cd tmux; git tag) } make_one() { rsync -a --delete tmux/ tmux$1/ cd tmux$1 git reset --hard git checkout $1 ./autogen.sh && ./configure && make -j2 && sudo cp tmux /usr/local/bin/tmux-$1 } export -f make_one echo '# Building tmux' listtags | stdout parallel --lb --tag '/usr/local/bin/tmux-{} -V || make_one {}' listtags | parallel -k -v /usr/local/bin/tmux-{} -V ) } bash_versions() { echo You may have to kill conftest (cd /tmp git clone https://git.savannah.gnu.org/git/bash.git listtags() { (cd bash; git tag | grep -v -- '-.*-' | grep -v bash-4.1.11) } make_one() { rsync -a --delete bash/ $1/ cd $1 git reset --hard git checkout $1 LDFLAGS=-static ./configure && make -j2 && sudo cp bash /usr/local/bin/$1 } export -f make_one echo '# Building bash' listtags | stdout parallel --lb --tag \ '/usr/local/bin/{} --version || make_one {}' listtags | parallel -k -v --tag '/usr/local/bin/{} --version' ) } rsync_versions() { (cd /tmp git clone https://github.com/WayneD/rsync listtags() { (cd rsync; git tag | grep -v -- '-.*-' | grep -v pre) } make_one() { rsync -a --delete rsync/ rsync-$1/ cd rsync-$1 git reset --hard git checkout $1 perl -i -pe 's/AC_DEFINE_UNQUOTED.HAVE_REMSH, .HAVE_REMSH./AC_DEFINE_UNQUOTED(HAVE_REMSH, \$HAVE_REMSH,[dummy])/; s/AC_DEFINE.HAVE_ERRNO_DECL.,/AC_DEFINE(HAVE_ERRNO_DECL,[1],[dummy]),/; s/AC_DEFINE.HAVE_FNMATCH.,/AC_DEFINE(HAVE_FNMATCH,[1],[dummy]),/;' configure.in autoreconf --install -W gnu # Make "lib/addrinfo.h" ? touch lib/addrinfo.h LDFLAGS=-static ./configure && (make proto; make -j2) && sudo cp rsync /usr/local/bin/rsync-$1 } export -f make_one echo '# Building rsync' listtags | stdout parallel --lb --tag \ '/usr/local/bin/rsync-{} --version || make_one {}' listtags | parallel -k -v --tag '/usr/local/bin/rsync-{} --version' ) } install_tangetools() { ( git clone https://codeberg.org/tange/tangetools cd tangetools && make && sudo make install ) } misc() { parallel --record-env # Build locale for LC_ALL=zh_HK.big5hkscs perl -e 1 sudo locale-gen zh_HK } run() { install_packages && install_tangetools && install_oracle_client && setup_databases && add_server_to_hosts && create_shell_logins && copy_ssh_keys && lsh_setup && add_freebsd && tmux_versions && bash_versions && rsync_versions && misc }