diff --git a/NEWS b/NEWS index 868a55ef..ed82186d 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,18 @@ +20230822 + +New in this release: + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* GNU Parallel, where have you been all my life? + https://alexplescan.com/posts/2023/08/20/gnu-parallel/ + +* Parallel (multithreaded) music download from Youtube + https://hrna.moe/?p=parallel-multithread-music-download + + 20230722 New in this release: diff --git a/doc/release_new_version b/doc/release_new_version index 366c04ba..45b31a17 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -264,7 +264,7 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20230822 ('Chandrayaan') released [stable] +Subject: GNU Parallel 20230922 ('Derna/Korankrise/Barbie/Woman Life Freedom') released [stable] GNU Parallel 20230822 ('Chandrayaan') has been released. It is available for download at: lbry://@GnuParallel:4 diff --git a/src/asciinema-manuscript b/src/asciinema-manuscript index 183d8e9b..fbceb133 100755 --- a/src/asciinema-manuscript +++ b/src/asciinema-manuscript @@ -26,7 +26,8 @@ # Run commands in tty2 to set clipboard to next command manuscript() { - # sleep this many seconds before executing command + # S ; sleep this many seconds before executing command + # T ; sleep this many seconds before pressing enter cat <<'_' ### # Definitions @@ -34,64 +35,66 @@ PS1='$ ' S() { sleep $@; } T() { echo -n '$ '; sleep $@; echo; } # Wait for asciinema to read all input -sleep 5 +sleep 1 ### S 0; # Make some files -S 3; seq 100 | parallel --bar seq {}0000 '>' file-{} -T 1; +S 1; seq 100 | parallel --bar seq {}0000 '>' file-{} +T 2; S 0; # gzip files with -1 .. -7 Note how all combinations are made -S 3; parallel --bar gzip {1} '<{2}' '>{2}{1}.gz' ::: -{1..7} ::: * +S 1; parallel --bar gzip {1} '<{2}' '>{2}{1}.gz' ::: -{1..7} ::: * T 1; S 0; # Count the bytes in some files. Each job gets its own color -S 3; parallel --color --tag wc {} ::: file-99* -T 1; +S 1; parallel --color --tag wc {} ::: file-99* +T 2; S 0; # Remove .gz-files -S 3; rm *.gz -T 1; +S 1; rm *.gz +T 3; S 0; # Make a big file -S 3; seq 10000000 > bigfile -T 1; +S 1; seq 10000000 > bigfile +T 3; S 0; # Chop bigfile into parts with 100000 lines (slow) -S 3; cat bigfile | parallel --pipe -n100000 cat '>lines-{#}' -S 3; ls lines-* +S 0; cat bigfile | parallel --pipe -n100000 cat '>lines-{#}' T 1; +S 1; ls lines-* +T 3; S 0; # Chop bigfile into parts each ~10mb (faster) -S 3; cat bigfile | parallel --pipe --block 10m cat '>10mb-{#}' +S 0; cat bigfile | parallel --pipe --block 10m cat '>10mb-{#}' S 3; ls -l 10mb-* T 1; S 0; # Chop bigfile into one part per CPU thread (fastest) -S 3; parallel --pipepart -a bigfile --block -1 \ +S 0; parallel --pipepart -a bigfile --block -1 \ cat '>thread-part-{#}' S 3; ls -l thread-part-* T 1; S 0; # Chop bigfile into one part per CPU thread - chop at 000\n -S 3; parallel --pipepart -a bigfile --block -1 --recend '000\n' \ +S 0; parallel --pipepart -a bigfile --block -1 --recend '000\n' \ cat '>p000-{#}' -S 3; head -1 p000-* -T 1; +T 3; +S 1; head -1 p000-* +T 3; S 0; # Rename all files to UPPERCASE using a {= perl expr =} S 3; parallel mv {} {= tr/a-z/A-Z/ =} ::: * -S 3; ls -T 1; +S 1; ls +T 3; S 0; # Move files into dirs named after the last char in the name -S 3; parallel 'mkdir -p {= $_=chop =}; mv {} {= $_=chop =}' ::: * -S 3; ls -T 1; +S 1; parallel 'mkdir -p {= $_=chop =}; mv {} {= $_=chop =}' ::: * +S 1; ls +T 3; S 0; # Zip dirs into files -S 3; parallel --ll --color --tag zip -r '{= s:/$:: =}'.zip {} ::: */ -T 1; +S 1; parallel --ll --color --tag zip -r '{= s:/$:: =}'.zip {} ::: */ +T 3; S 0; # Remove the dirs -S 3; rm -r */ -T 1; +S 1; rm -r */ +T 3; S 0; # Unzip zip files into dirs -S 3; parallel --ll --tag 'mkdir {.}; cd {.}; unzip ../{}' ::: *.zip -T 1; +S 1; parallel --ll --tag 'mkdir {.}; cd {.}; unzip ../{}' ::: *.zip +T 3; S 0; # Remove zip and dirs -S 3; rm -r *.zip ? -T 1; +S 1; rm -r *.zip ? +T 3; S 0; # Paint output of the failing commands -S 3; parallel --colorfailed echo Arg:{} ';' exit {} ::: 0 0 1 2 0 -T 1; +S 1; parallel --colorfailed echo Arg:{} ';' exit {} ::: 0 0 1 2 0 +T 3; S 0; # Use a bash function with one argument doit() { echo Doing it for $1 @@ -99,8 +102,8 @@ doit() { echo Done with $1 } export -f doit -S 3; parallel doit ::: 1 2 3 -T 1; +S 1; parallel doit ::: 1 2 3 +T 3; S 0; # Use a bash function with two argument doubleit() { echo Doing it for $1 $2 @@ -108,11 +111,11 @@ doubleit() { echo Done with $1 $2 } export -f doubleit -S 3; parallel doubleit ::: 1 2 3 ::: a b -T 1; +S 1; parallel doubleit ::: 1 2 3 ::: a b +T 3; S 0; # Make some logfiles S 1; touch log -S 3; parallel seq {} '>' log.{} ::: {1..10} +S 1; parallel seq {} '>' log.{} ::: {1..10} S 3; ls -l T 1; S 0; # Simple log rotate @@ -123,20 +126,25 @@ T 1; S 0; # Monitor progress of multiple jobs in parallel slowseq() { seq "$@" | pv -qL 20; } export -f slowseq -S 3; parallel --ll --color --tag slowseq {} ::: 150 {1..60} +S 1; parallel --ll --color --tag slowseq {} ::: 150 {1..60} T 3; ### exit _ } +# $sleep = $sleep/2+100.001; +# $sleep = $sleep/2+100.001; +# $sleep = $sleep/2+100.001; +# return $ms/2+100.001; +# if($something_printed) { $ms = $ms/2+100.001; } + pwd=$(pwd) tmp=$(mktemp -d) cd "$tmp" && manuscript | - parallel -j1 echo '{= s/^(\d+)\s/sleep($1);""/e; =}' | - asciinema rec - | - perl -ne 's/[ST] \d;\s?//; + asciinema rec - | tee /tmp/parallel.c | + perl -ne 's/;?[ST] \d;\s?//;s/; [ST] \d//; /###/ and $mark++ and next; - ($mark == 0 or $mark == 5) and print; + ($mark == 0 or $mark == 4) and print; ' > "$pwd"/parallel.cast diff --git a/src/pod2rst-fix b/src/pod2rst-fix index 4cc97ad1..ab40f510 100755 --- a/src/pod2rst-fix +++ b/src/pod2rst-fix @@ -25,6 +25,8 @@ # Conversion errors: +# B + # Fixed: # ... B # bar @@ -32,6 +34,9 @@ # Fixed: # =item - - a +# Fixed: +# \\` => \` + # Not fixed (RST does not support Bold-Italic): # B> @@ -98,6 +103,8 @@ sub post { s/\\\\[*]/\\*/g; # - - s/^-(\s+)\001/-$1\\-/g; + # \\` => \` + s/\\\\`/\\`/g; print; } } diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index 7f8dd10b..a67813a6 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -7,18 +7,18 @@ install_packages() { # After first run, set this: # export INSTALL=echo - INSTALL=${INSTALL:-"sudo aptitude -y install"} + INSTALL=${INSTALL:-"sudo apt -y install"} # The testsuite depends on this: test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl" test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs" test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl" - test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386" - test_pkgs="$test_pkgs libtest-nowarnings-perl xemacs21-bin" + test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386 vagrant virtualbox" + test_pkgs="$test_pkgs libtest-nowarnings-perl xemacs21 pv zenity" # Debian package - packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15" - packaging_pkgs="$packaging_pkgs python3-m2crypto alien" + packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake" + packaging_pkgs="$packaging_pkgs python3-m2crypto alien python3-pip" # Shebang Tools shebang_pkgs="gnuplot octave ruby r-base-core" @@ -34,7 +34,8 @@ install_packages() { # Shells # (csh = bsd-csh that is broken) - shell_pkgs="ash dash fdclone fish fizsh ksh ksh93u+m ksh93 mksh posh rc rush sash" + # (ksh93 gone) + shell_pkgs="ash dash fdclone fish fizsh ksh ksh93u+m mksh posh rc rush sash" shell_pkgs="$shell_pkgs tcsh yash zsh busybox-static" # Databases @@ -42,10 +43,10 @@ install_packages() { # Build Tools build_pkgs="bison libxxhash-dev libzstd-dev liblz4-dev libssl-dev" - build_pkgs="$build_pkgs python3-cmarkgfm graphviz" + build_pkgs="$build_pkgs python3-cmarkgfm libpod-pom-view-restructured-perl" + build_pkgs="$build_pkgs graphviz" sudo dpkg --add-architecture i386; sudo apt update - pip install sphinx-rtd-theme if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \ $compression_pkgs $shell_pkgs $database_pkgs $build_pkgs ; then # OK @@ -54,6 +55,7 @@ install_packages() { echo Packages missing return 1 fi + pip install sphinx-rtd-theme sudo apt update & } @@ -117,6 +119,10 @@ setup_virtual_box_network() { setup_databases() { # DATABASES echo '# Create PostgreSQL' + sudo em /etc/postgresql/*/main/postgresql.conf + # listen_addresses = '*' + sudo service postgresql restart + # Drop database and user if needed sudo su - postgres -c 'dropdb '`whoami` sudo su - postgres -c 'dropuser '`whoami` @@ -146,13 +152,13 @@ add_server_to_hosts() { # could not settle on kex algorithm # Server kex preferences: diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1 # Client kex preferences: ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1> -debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,sntrup761x25519-sha512@openssh.com,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group1-sha1,ext-info-c -debug2: host key algorithms: ssh-dss,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 -debug2: ciphers ctos: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc -debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc -debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 -debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 - +#debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,sntrup761x25519-sha512@openssh.com,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group1-sha1,ext-info-c +#debug2: host key algorithms: ssh-dss,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 +#debug2: ciphers ctos: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc +#debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc +#debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 +#debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1 +# } insert_in_etc_hosts() { @@ -170,6 +176,8 @@ debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-25 add_single_vagrant_to_etc_hosts() { vagrant_ip() { cd vagrant/*/$1 + # .vagrant can contain dirt + rm -rf .vagrant grep 172 Vagrantfile | grep -v '#' | perl -pe 's/.*(172.27[\.0-9]+).*/$1/'; } @@ -179,11 +187,36 @@ debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-25 add_ssh_key_to_authorized insert_in_etc_hosts 127.1.2.3 server + + insert_in_etc_hosts 127.0.0.2 lo parallel add_single_vagrant_to_etc_hosts ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817 + # TODO alias + # 172.27.27.8 centos8 parallel-server1 server1 + # 172.27.27.72 freebsd12 parallel-server3 server3 + # 172.27.27.108 rhel8 parallel-server4 server4 + # 172.27.27.73 freebsd13 parallel-server2 server2 + # TODO + # $ cat /etc/vbox/networks.conf + # * 172.27.0.0/16 + # TODO + sshaddvagrant() { + cat ~/.ssh/*.pub | (cd vagrant/*/$1 && vagrant ssh -c 'cat >>.ssh/authorized_keys') + sshpass -p vagrant ssh-copy-id -o StrictHostKeyChecking=accept-new vagrant@$1 + ssh vagrant@$1 echo vagrant@$1 OK + } + export -f sshaddvagrant + parallel sshaddvagrant ::: centos8 freebsd11 freebsd12 freebsd13 rhel8 centos3 centos39-oracle817 + parallel ssh -o StrictHostKeyChecking=accept-new vagrant@{} ::: parallel-server{1..4} centos3 + # TODO find ipv6 + ipv6= + parallel ssh -o StrictHostKeyChecking=accept-new {} ::: lo localhost 172.27.27.1 127.0.0.1 ::1 $ipv6 + # parallel sshpass -p vagrant ssh-copy-id vagrant@{} ::: parallel-server{1..4} centos3 + + # ssh-copy-id vagrant@parallel-server1..4 centos3 } shellsplus() { - shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh" + shells="bash sh csh ash dash tcsh zsh ksh ksh93 fish fizsh mksh" shells="$shells posh rc sash yash nopathbash nopathcsh" shellsplus="parallel $shells" parallel -k echo ::: $shellsplus @@ -213,15 +246,15 @@ create_shell_logins() { sudo killall -u $shell sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM sudo groupdel $shell - if echo $shell | grep -q parallel; then - # User parallel should have /bin/bash + if echo $shell | grep -qE "parallel|withpassword"; then + # User parallel+withpassword should have /bin/bash loginshell=/bin/bash else # Other users should have `which $shell` with nopath removed loginshell=$(which $shell || which ${shell#"nopath"}) fi sudo adduser --shell $loginshell --disabled-password --gecos "$shell for parallel,,," $shell && - echo "$shell:$SSHPASS" | sudo chpasswd && + echo "$shell:$SSHPASS" | sudo chpasswd && sshpass -e ssh-copy-id $shell@lo && echo "$shell created" } @@ -444,10 +477,12 @@ rsync_versions() { } install_tangetools() { - git clone https://gitlab.com/ole.tange/tangetools.git - cd tangetools && - make && - sudo make install + ( + git clone https://codeberg.org/tange/tangetools + cd tangetools && + make && + sudo make install + ) } misc() { diff --git a/testsuite/tests-to-run/parallel-local21.sh b/testsuite/tests-to-run/parallel-local21.sh index d4ecf2cb..d2c7f304 100644 --- a/testsuite/tests-to-run/parallel-local21.sh +++ b/testsuite/tests-to-run/parallel-local21.sh @@ -216,6 +216,7 @@ EOF par_shebang_wrap_octave() { script="$TMPDIR"/shebang_wrap_octave + unset DISPLAY cat <<'EOF' > "$script" #!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/octave diff --git a/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile b/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile new file mode 100644 index 00000000..b7fe8097 --- /dev/null +++ b/testsuite/vagrant/FritsHoogland/centos39-oracle817/Vagrantfile @@ -0,0 +1,80 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "FritsHoogland/centos39-oracle817" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + config.vm.network :forwarded_port, guest: 22, host: 50136, id: 'ssh' + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "172.27.27.3" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: +# vb.memory = "512" + vb.memory = "300" + vb.customize ["modifyvm", :id, "--usb", "on"] + vb.customize ["modifyvm", :id, "--usbehci", "off"] + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell" do |s| + ssh_pub_key = File.readlines("../../authorized_keys").first.strip + s.inline = <<-SHELL + mkdir /root/.ssh + echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys + echo #{ssh_pub_key} >> /root/.ssh/authorized_keys + sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync + sudo apt install perl || sudo yum install -y perl || sudo pkg install -y perl + SHELL + end +end