testsuite: Make it easier to run test on other machines.

doc/release_new_version

@@ -273,26 +273,27 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20231222 ('Sundhnúkagígur') released
+Subject: GNU Parallel 20240122 ('Margrethe') released
 
-GNU Parallel 20231222 ('Sundhnúkagígur') has been released. It is available for download at: lbry://@GnuParallel:4
+GNU Parallel 20240122 ('Margrethe') has been released. It is available for download at: lbry://@GnuParallel:4
 
 Quote of the month:
 
-  Parallel is so damn good! You’ve got to use it.
+  <<>>
-    -- @ThePrimeTimeagen@youtube.com
  
 New in this release:
 
-* --combine-exec combines GNU Parallel with any executable.
+* --sshlogin supports ranges: server[01-12,15] 10.0.[1-10].[2-254]
+
+* --plus enables {slot-1} and {seq-1} = {%}-1 and {#}-1 to count from 0.
+
+* env_parallel.{sh,ash,dash,bash,ksh,zsh} is now the same script.
 
 * Bug fixes and man page updates.
 
 News about GNU Parallel:
 
-* The first video that seems to be completely AI generated: https://www.youtube.com/watch?v=Ie8eAhlEHOg
+* <<>>
-
-* GNU Parallel and “fail immediately if any fails” problem https://antiarchitect.medium.com/gnu-parallel-and-fail-immediately-if-any-fails-problem-481ddea690ba
 
 
 GNU Parallel - For people who live life in the parallel lane.

src/env_parallel

@@ -79,9 +79,11 @@ done
 
 
 cat <<'_EOS'
+You have called the dummy script "env_parallel".
+
 env_parallel only works if it is a function.
 
-Do this and restart your shell:
+You need to do this and restart your shell:
 
 bash:  Put this in $HOME/.bashrc:  . env_parallel.bash
        E.g. by doing:  echo '. env_parallel.bash' >> $HOME/.bashrc

src/env_parallel.pod

@@ -25,24 +25,15 @@ environment to GNU B<parallel>.
 If the shell function is not loaded, a dummy script will be run
 instead that explains how to install the function.
 
-B<env_parallel> is 100 ms slower at startup than pure GNU
+B<env_parallel> is 100 ms slower at startup than pure GNU B<parallel>,
-B<parallel>, and takes up to 30% longer to start a job (typically 15 ms).
+and takes up to 30% longer to start a job (typically 15 ms).
 
-Due to the problem with environment space (see below) the recommended
+Due to the problem with environment space (see below) you are
-usage is either:
+recommended only to transfer the environment that you need.
 
-  # Do --record-env into $PARALLEL_IGNORED_NAMES
+To help you do that, you can mark names that should not be
-  env_parallel --session
+transferred. This can be done with either B<--session> or
-
+B<--record-env>.
-  # Define whatever you want to use
-  myfunc() { myalias and functions $myvar work. $1.; }
-  alias myalias='echo Aliases'
-  myvar='and variables'
-
-  # env_parallel will not export names in $PARALLEL_IGNORED_NAMES
-  env_parallel -S localhost myfunc ::: Hooray
-
-Or:
 
   # Record the "clean" environment (this only needs to be run once)
   env_parallel --record-env
@@ -57,6 +48,22 @@ Or:
   # Use --env _ to only transfer the names not in the "empty" environment
   env_parallel --env _ -S localhost myfunc ::: Hooray
 
+Or:
+
+  # Do --record-env into $PARALLEL_IGNORED_NAMES
+  env_parallel --session
+
+  # Define whatever you want to use
+  myfunc() { myalias and functions $myvar work. $1.; }
+  alias myalias='echo Aliases'
+  myvar='and variables'
+
+  # env_parallel will not export names in $PARALLEL_IGNORED_NAMES
+  env_parallel -S localhost myfunc ::: Hooray
+
+  # Optionally
+  env_parallel --end-session
+
 In B<csh> B<--session> is not supported:
 
   # Record the "clean" environment - this only needs to be run once

testsuite/REQUIREMENTS

@@ -44,8 +44,7 @@ install_packages() {
     # Build Tools
     build_pkgs="bison libxxhash-dev libzstd-dev liblz4-dev libssl-dev"
     build_pkgs="$build_pkgs python3-cmarkgfm libpod-pom-view-restructured-perl"
-    build_pkgs="$build_pkgs graphviz"
+    build_pkgs="$build_pkgs graphviz yodl libreoffice"
-
     sudo dpkg --add-architecture i386; sudo apt update
     if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \
 		$compression_pkgs $shell_pkgs $database_pkgs $build_pkgs ; then
@@ -109,13 +108,6 @@ install_oracle_client() {
     # test it works: sql oracle://
 }
 
-setup_virtual_box_network() {
-    echo "Adding IP-range to virtualbox"
-    sudo mkdir -p /etc/vbox
-    (cat; echo '* 172.27.27.0/24') < /etc/vbox/networks.conf | uniq |
-	sudo tee /etc/vbox/networks.conf
-}
-
 setup_databases() {
     # DATABASES
     echo '# Create PostgreSQL'
@@ -142,6 +134,20 @@ setup_databases() {
     sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
 }
 
+setup_virtual_box_network() {
+    echo "Adding IP-range to virtualbox"
+    sudo mkdir -p /etc/vbox
+    sudo touch /etc/vbox/networks.conf
+    (cat; echo '* 172.27.27.0/24') < /etc/vbox/networks.conf | uniq |
+	sudo tee /etc/vbox/networks.conf
+}
+
+start_vagrant_servers() {
+    cd testsuite/vagrant/
+    ssh-add -L >> authorized_keys
+    parallel 'cd {//}; vagrant up' ::: testsuite/vagrant/*/*/Vagrantfile
+}
+
 add_server_to_hosts() {
     add_ssh_key_to_authorized() {
 	(cat vagrant/authorized_keys; cat ~/.ssh/*.pub) |
@@ -159,6 +165,13 @@ add_server_to_hosts() {
 #debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
 #debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
 	#
+
+	# Fix: .ssh/config
+ 	#Host centos3 127.0.0.1 
+ 	#  KexAlgorithms +diffie-hellman-group1-sha1 
+ 	#  Ciphers +3des-cbc,aes128-cbc,aes192-cbc,aes256-cbc 
+ 	#  HostKeyAlgorithms +ssh-rsa,ssh-dss
+ 	# sshpass -p vagrant ssh -v -p 50136 vagrant@127.0.0.1
     }
     
     insert_in_etc_hosts() {
@@ -168,7 +181,7 @@ add_server_to_hosts() {
 	    # skip
 	    true
 	else
-	    printf "$1\t$2\n" |
+	    printf "$@\n" |
 		sudo tee -a /etc/hosts
 	fi
     }
@@ -181,7 +194,7 @@ add_server_to_hosts() {
 	    grep 172 Vagrantfile | grep -v '#' |
 		perl -pe 's/.*(172.27[\.0-9]+).*/$1/';
 	}
-	insert_in_etc_hosts $(vagrant_ip $1) $1
+	insert_in_etc_hosts $(vagrant_ip $1) $@
     }
     export -f add_single_vagrant_to_etc_hosts
     
@@ -189,7 +202,8 @@ add_server_to_hosts() {
     insert_in_etc_hosts 127.1.2.3 server
     
     insert_in_etc_hosts 127.0.0.2 lo
-    parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817
+    parallel add_single_vagrant_to_etc_hosts {} parallel-server{#} ::: centos8 freebsd12 freebsd13 rhel8 centos3
+    #centos39-oracle817
     # TODO alias
     # 172.27.27.8     centos8 parallel-server1 server1
     # 172.27.27.72    freebsd12 parallel-server3 server3
@@ -199,13 +213,16 @@ add_server_to_hosts() {
     # $ cat /etc/vbox/networks.conf
     # * 172.27.0.0/16
     # TODO
+    parallel --ll --tag 'cd {};vagrant up' ::: vagrant/*/*
+    parallel w4it-for-port-open {} 22 ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3
+    
     sshaddvagrant() {
 	cat ~/.ssh/*.pub | (cd vagrant/*/$1 && vagrant ssh -c 'cat >>.ssh/authorized_keys')
 	sshpass -p vagrant ssh-copy-id -o StrictHostKeyChecking=accept-new vagrant@$1
 	ssh vagrant@$1 echo vagrant@$1 OK
     }
     export -f sshaddvagrant
-    parallel sshaddvagrant ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817
+    parallel sshaddvagrant ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3
     parallel ssh -o StrictHostKeyChecking=accept-new vagrant@{} ::: parallel-server{1..4} centos3
     # TODO find ipv6
     ipv6=
@@ -463,7 +480,7 @@ rsync_versions() {
 	 # Make "lib/addrinfo.h" ?
 	 touch lib/addrinfo.h
 	 LDFLAGS=-static ./configure &&
-	     (make proto; make -j2) &&
+	     (make proto; make -j2 || make reconfigure && make -j2; make) &&
 	     sudo cp rsync /usr/local/bin/rsync-$1
      }
      export -f make_one
@@ -497,6 +514,8 @@ run() {
 	install_tangetools &&
 	install_oracle_client &&
 	setup_databases &&
+	setup_virtual_box_network &&
+	start_vagrant_servers &&
 	add_server_to_hosts &&
 	create_shell_logins &&
 	copy_ssh_keys &&

testsuite/tests-to-run/parallel-local-1s.sh

@@ -680,6 +680,17 @@ par_pipe_compress_blocks() {
 par_too_long_line_X() {
     echo 'bug #54869: Long lines break'
     seq 3000 | parallel -Xj1 'echo {} {} {} {} {} {} {} {} {} {} {} {} {} {} | wc'
+}
+
+par__test_cpu_detection_topology() {
+    pack() { zstd -19 | mmencode; }
+    unpack() { mmencode -u | zstd -d; }
+    PARALLEL_CPUPREFIX=
+    # ssh server tar cvf - /sys/devices/system/cpu | pack
+
+
+    
+    
 }
 
 par__test_cpu_detection_cpuinfo() {

testsuite/vagrant/generic/centos8/Vagrantfile

@@ -0,0 +1,76 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "generic/centos8"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "172.27.27.8"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+#    vb.memory = "512"
+    vb.memory = "300"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell" do |s|
+    ssh_pub_key = File.readlines("../../authorized_keys").first.strip
+    s.inline = <<-SHELL
+      mkdir /root/.ssh
+      echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
+      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
+      sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync
+    SHELL
+  end
+end

testsuite/vagrant/generic/freebsd12/Vagrantfile

@@ -0,0 +1,76 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "generic/freebsd12"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "172.27.27.72"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+#    vb.memory = "512"
+    vb.memory = "350"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell" do |s|
+    ssh_pub_key = File.readlines("../../authorized_keys").first.strip
+    s.inline = <<-SHELL
+      mkdir /root/.ssh
+      echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
+      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
+      sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync
+    SHELL
+  end
+end

testsuite/vagrant/generic/rhel8/Vagrantfile

@@ -0,0 +1,79 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# SPDX-FileCopyrightText: 2023 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "generic/rhel8"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "172.27.27.108"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+    vb.memory = "300"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell" do |s|
+    ssh_pub_key = File.readlines("../../authorized_keys").first.strip
+    s.inline = <<-SHELL
+      mkdir /root/.ssh
+      echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
+      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
+      sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync
+    SHELL
+  end
+end