diff --git a/doc/release_new_version b/doc/release_new_version index 3a955c69..32003e14 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -273,26 +273,27 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20231222 ('Sundhnúkagígur') released +Subject: GNU Parallel 20240122 ('Margrethe') released -GNU Parallel 20231222 ('Sundhnúkagígur') has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20240122 ('Margrethe') has been released. It is available for download at: lbry://@GnuParallel:4 Quote of the month: - Parallel is so damn good! You’ve got to use it. - -- @ThePrimeTimeagen@youtube.com + <<>> New in this release: -* --combine-exec combines GNU Parallel with any executable. +* --sshlogin supports ranges: server[01-12,15] 10.0.[1-10].[2-254] + +* --plus enables {slot-1} and {seq-1} = {%}-1 and {#}-1 to count from 0. + +* env_parallel.{sh,ash,dash,bash,ksh,zsh} is now the same script. * Bug fixes and man page updates. News about GNU Parallel: -* The first video that seems to be completely AI generated: https://www.youtube.com/watch?v=Ie8eAhlEHOg - -* GNU Parallel and “fail immediately if any fails” problem https://antiarchitect.medium.com/gnu-parallel-and-fail-immediately-if-any-fails-problem-481ddea690ba +* <<>> GNU Parallel - For people who live life in the parallel lane. diff --git a/src/env_parallel b/src/env_parallel index 83ed027b..cab0ef26 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -79,9 +79,11 @@ done cat <<'_EOS' +You have called the dummy script "env_parallel". + env_parallel only works if it is a function. -Do this and restart your shell: +You need to do this and restart your shell: bash: Put this in $HOME/.bashrc: . env_parallel.bash E.g. by doing: echo '. env_parallel.bash' >> $HOME/.bashrc diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 263a6717..bc34841e 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -25,24 +25,15 @@ environment to GNU B. If the shell function is not loaded, a dummy script will be run instead that explains how to install the function. -B is 100 ms slower at startup than pure GNU -B, and takes up to 30% longer to start a job (typically 15 ms). +B is 100 ms slower at startup than pure GNU B, +and takes up to 30% longer to start a job (typically 15 ms). -Due to the problem with environment space (see below) the recommended -usage is either: +Due to the problem with environment space (see below) you are +recommended only to transfer the environment that you need. - # Do --record-env into $PARALLEL_IGNORED_NAMES - env_parallel --session - - # Define whatever you want to use - myfunc() { myalias and functions $myvar work. $1.; } - alias myalias='echo Aliases' - myvar='and variables' - - # env_parallel will not export names in $PARALLEL_IGNORED_NAMES - env_parallel -S localhost myfunc ::: Hooray - -Or: +To help you do that, you can mark names that should not be +transferred. This can be done with either B<--session> or +B<--record-env>. # Record the "clean" environment (this only needs to be run once) env_parallel --record-env @@ -57,6 +48,22 @@ Or: # Use --env _ to only transfer the names not in the "empty" environment env_parallel --env _ -S localhost myfunc ::: Hooray +Or: + + # Do --record-env into $PARALLEL_IGNORED_NAMES + env_parallel --session + + # Define whatever you want to use + myfunc() { myalias and functions $myvar work. $1.; } + alias myalias='echo Aliases' + myvar='and variables' + + # env_parallel will not export names in $PARALLEL_IGNORED_NAMES + env_parallel -S localhost myfunc ::: Hooray + + # Optionally + env_parallel --end-session + In B B<--session> is not supported: # Record the "clean" environment - this only needs to be run once diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index a67813a6..c5227d14 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -44,8 +44,7 @@ install_packages() { # Build Tools build_pkgs="bison libxxhash-dev libzstd-dev liblz4-dev libssl-dev" build_pkgs="$build_pkgs python3-cmarkgfm libpod-pom-view-restructured-perl" - build_pkgs="$build_pkgs graphviz" - + build_pkgs="$build_pkgs graphviz yodl libreoffice" sudo dpkg --add-architecture i386; sudo apt update if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \ $compression_pkgs $shell_pkgs $database_pkgs $build_pkgs ; then @@ -109,13 +108,6 @@ install_oracle_client() { # test it works: sql oracle:// } -setup_virtual_box_network() { - echo "Adding IP-range to virtualbox" - sudo mkdir -p /etc/vbox - (cat; echo '* 172.27.27.0/24') < /etc/vbox/networks.conf | uniq | - sudo tee /etc/vbox/networks.conf -} - setup_databases() { # DATABASES echo '# Create PostgreSQL' @@ -142,6 +134,20 @@ setup_databases() { sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';" } +setup_virtual_box_network() { + echo "Adding IP-range to virtualbox" + sudo mkdir -p /etc/vbox + sudo touch /etc/vbox/networks.conf + (cat; echo '* 172.27.27.0/24') < /etc/vbox/networks.conf | uniq | + sudo tee /etc/vbox/networks.conf +} + +start_vagrant_servers() { + cd testsuite/vagrant/ + ssh-add -L >> authorized_keys + parallel 'cd {//}; vagrant up' ::: testsuite/vagrant/*/*/Vagrantfile +} + add_server_to_hosts() { add_ssh_key_to_authorized() { (cat vagrant/authorized_keys; cat ~/.ssh/*.pub) | @@ -158,7 +164,14 @@ add_server_to_hosts() { #debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc #debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 #debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 -# + # + + # Fix: .ssh/config + #Host centos3 127.0.0.1 + # KexAlgorithms +diffie-hellman-group1-sha1 + # Ciphers +3des-cbc,aes128-cbc,aes192-cbc,aes256-cbc + # HostKeyAlgorithms +ssh-rsa,ssh-dss + # sshpass -p vagrant ssh -v -p 50136 vagrant@127.0.0.1 } insert_in_etc_hosts() { @@ -168,7 +181,7 @@ add_server_to_hosts() { # skip true else - printf "$1\t$2\n" | + printf "$@\n" | sudo tee -a /etc/hosts fi } @@ -181,7 +194,7 @@ add_server_to_hosts() { grep 172 Vagrantfile | grep -v '#' | perl -pe 's/.*(172.27[\.0-9]+).*/$1/'; } - insert_in_etc_hosts $(vagrant_ip $1) $1 + insert_in_etc_hosts $(vagrant_ip $1) $@ } export -f add_single_vagrant_to_etc_hosts @@ -189,7 +202,8 @@ add_server_to_hosts() { insert_in_etc_hosts 127.1.2.3 server insert_in_etc_hosts 127.0.0.2 lo - parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817 + parallel add_single_vagrant_to_etc_hosts {} parallel-server{#} ::: centos8 freebsd12 freebsd13 rhel8 centos3 + #centos39-oracle817 # TODO alias # 172.27.27.8 centos8 parallel-server1 server1 # 172.27.27.72 freebsd12 parallel-server3 server3 @@ -199,13 +213,16 @@ add_server_to_hosts() { # $ cat /etc/vbox/networks.conf # * 172.27.0.0/16 # TODO + parallel --ll --tag 'cd {};vagrant up' ::: vagrant/*/* + parallel w4it-for-port-open {} 22 ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 + sshaddvagrant() { cat ~/.ssh/*.pub | (cd vagrant/*/$1 && vagrant ssh -c 'cat >>.ssh/authorized_keys') sshpass -p vagrant ssh-copy-id -o StrictHostKeyChecking=accept-new vagrant@$1 ssh vagrant@$1 echo vagrant@$1 OK } export -f sshaddvagrant - parallel sshaddvagrant ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817 + parallel sshaddvagrant ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 parallel ssh -o StrictHostKeyChecking=accept-new vagrant@{} ::: parallel-server{1..4} centos3 # TODO find ipv6 ipv6= @@ -463,7 +480,7 @@ rsync_versions() { # Make "lib/addrinfo.h" ? touch lib/addrinfo.h LDFLAGS=-static ./configure && - (make proto; make -j2) && + (make proto; make -j2 || make reconfigure && make -j2; make) && sudo cp rsync /usr/local/bin/rsync-$1 } export -f make_one @@ -497,6 +514,8 @@ run() { install_tangetools && install_oracle_client && setup_databases && + setup_virtual_box_network && + start_vagrant_servers && add_server_to_hosts && create_shell_logins && copy_ssh_keys && diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index cfe997ef..9df89e80 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -680,6 +680,17 @@ par_pipe_compress_blocks() { par_too_long_line_X() { echo 'bug #54869: Long lines break' seq 3000 | parallel -Xj1 'echo {} {} {} {} {} {} {} {} {} {} {} {} {} {} | wc' +} + +par__test_cpu_detection_topology() { + pack() { zstd -19 | mmencode; } + unpack() { mmencode -u | zstd -d; } + PARALLEL_CPUPREFIX= + # ssh server tar cvf - /sys/devices/system/cpu | pack + + + + } par__test_cpu_detection_cpuinfo() { diff --git a/testsuite/vagrant/generic/centos8/Vagrantfile b/testsuite/vagrant/generic/centos8/Vagrantfile new file mode 100644 index 00000000..72c9a4d5 --- /dev/null +++ b/testsuite/vagrant/generic/centos8/Vagrantfile @@ -0,0 +1,76 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "generic/centos8" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "172.27.27.8" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: +# vb.memory = "512" + vb.memory = "300" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell" do |s| + ssh_pub_key = File.readlines("../../authorized_keys").first.strip + s.inline = <<-SHELL + mkdir /root/.ssh + echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys + echo #{ssh_pub_key} >> /root/.ssh/authorized_keys + sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync + SHELL + end +end diff --git a/testsuite/vagrant/generic/freebsd12/Vagrantfile b/testsuite/vagrant/generic/freebsd12/Vagrantfile new file mode 100644 index 00000000..76c13b71 --- /dev/null +++ b/testsuite/vagrant/generic/freebsd12/Vagrantfile @@ -0,0 +1,76 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "generic/freebsd12" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "172.27.27.72" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: +# vb.memory = "512" + vb.memory = "350" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell" do |s| + ssh_pub_key = File.readlines("../../authorized_keys").first.strip + s.inline = <<-SHELL + mkdir /root/.ssh + echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys + echo #{ssh_pub_key} >> /root/.ssh/authorized_keys + sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync + SHELL + end +end diff --git a/testsuite/vagrant/generic/rhel8/Vagrantfile b/testsuite/vagrant/generic/rhel8/Vagrantfile new file mode 100644 index 00000000..f8b70693 --- /dev/null +++ b/testsuite/vagrant/generic/rhel8/Vagrantfile @@ -0,0 +1,79 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# SPDX-FileCopyrightText: 2023 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "generic/rhel8" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "172.27.27.108" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + vb.memory = "300" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell" do |s| + ssh_pub_key = File.readlines("../../authorized_keys").first.strip + s.inline = <<-SHELL + mkdir /root/.ssh + echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys + echo #{ssh_pub_key} >> /root/.ssh/authorized_keys + sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync + SHELL + end +end