From 3cfcbc6920b86541032d5f9b3eff86ed87a686d3 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 22 Apr 2021 18:20:41 +0200 Subject: [PATCH] Released as 20210422 ('Ever Given') --- .gitignore | 2 + CITATION | 4 + Makefile.am | 1 + Makefile.in | 1 + NEWS | 29 +++++ README | 27 +++-- bootstrap | 7 +- configure | 20 ++-- configure.ac | 2 +- doc/FUTURE_IDEAS | 108 ++---------------- doc/boxplot-overhead | 3 + doc/haikus | 11 +- doc/pod2savannah_publicinfo | 3 + doc/promo | 4 + doc/release_new_version | 72 ++++++++++-- packager/0install/check | 4 + packager/0install/update | 2 + packager/centos/install-parallel-centos-5.sh | 4 + packager/debian/Makefile | 4 + packager/debian/doc/index.html | 6 + packager/obs/home:tange/parallel/.osc/_files | 12 +- .../home:tange/parallel/.osc/parallel.spec | 2 +- packager/opensuse/parallel.spec | 72 ------------ packager/releasescripts/updateversion | 16 ++- src/Makefile.am | 4 + src/Makefile.in | 4 + src/env_parallel | 3 + src/env_parallel.ash | 2 +- src/env_parallel.bash | 18 +-- src/env_parallel.dash | 2 +- src/env_parallel.ksh | 2 +- src/env_parallel.mksh | 2 +- src/env_parallel.sh | 24 ++-- src/env_parallel.zsh | 2 +- src/niceload | 2 +- src/parallel | 75 ++++++------ src/parallel.pod | 88 +++++++++----- src/parallel_alternatives.pod | 18 +-- src/parallel_book.pod | 2 +- src/parallel_tutorial.pod | 12 +- src/parcat | 2 +- src/parsort | 2 +- src/sem.pod | 2 +- src/sql | 2 +- testsuite/Fixed.sh | 4 + testsuite/Makefile | 20 ++-- testsuite/Portable.sh | 4 + testsuite/REQUIREMENTS | 37 +++--- testsuite/Start.sh | 4 + testsuite/input-files/segfault/items-0.xi | Bin 63 -> 0 bytes testsuite/input-files/segfault/trysegfault | 16 --- testsuite/input-files/shebang/shebangwrap.gp | 4 - testsuite/input-files/shebang/shebangwrap.oct | 6 - testsuite/input-files/shebang/shebangwrap.pl | 3 - testsuite/input-files/shebang/shebangwrap.py | 5 - testsuite/input-files/shebang/shebangwrap.r | 4 - testsuite/input-files/shebang/shebangwrap.rb | 3 - testsuite/input-files/shebang/shebangwrap.sh | 3 - testsuite/input-files/test08/a | 3 - testsuite/input-files/test08/b | 3 - testsuite/input-files/test08/c | 3 - testsuite/input-files/test08/d | 4 - testsuite/tests-to-run/parallel-local-1s.sh | 11 +- testsuite/tests-to-run/parallel-local-30s.sh | 4 +- testsuite/tests-to-run/parallel-local-3s.sh | 103 ++++++++++++++++- testsuite/tests-to-run/parallel-local-ssh6.sh | 5 +- testsuite/tests-to-run/parallel-local12.sh | 27 +++++ testsuite/tests-to-run/parallel-local150.sh | 6 - testsuite/tests-to-run/parallel-local164.sh | 13 --- testsuite/tests-to-run/parallel-local23.sh | 3 - testsuite/tests-to-run/test15.sh | 35 ------ testsuite/vagrant/Vagrantfile.tmpl | 2 + testsuite/wanted-results/parallel-local-1s | 5 - testsuite/wanted-results/parallel-local-3s | 84 +++++++++++++- testsuite/wanted-results/parallel-local-ssh1 | 2 +- testsuite/wanted-results/parallel-local12 | 45 ++++++++ testsuite/wanted-results/parallel-local150 | 5 - testsuite/wanted-results/parallel-local164 | 5 - testsuite/wanted-results/parallel-local18 | 2 +- testsuite/wanted-results/parallel-local23 | 31 ----- testsuite/wanted-results/parallel-tutorial | 6 +- testsuite/wanted-results/test15 | 14 --- testsuite/wanted-results/test19 | 24 ++-- 83 files changed, 698 insertions(+), 544 deletions(-) delete mode 100644 packager/opensuse/parallel.spec delete mode 100644 testsuite/input-files/segfault/items-0.xi delete mode 100755 testsuite/input-files/segfault/trysegfault delete mode 100755 testsuite/input-files/shebang/shebangwrap.gp delete mode 100755 testsuite/input-files/shebang/shebangwrap.oct delete mode 100755 testsuite/input-files/shebang/shebangwrap.pl delete mode 100755 testsuite/input-files/shebang/shebangwrap.py delete mode 100755 testsuite/input-files/shebang/shebangwrap.r delete mode 100755 testsuite/input-files/shebang/shebangwrap.rb delete mode 100755 testsuite/input-files/shebang/shebangwrap.sh delete mode 100644 testsuite/input-files/test08/a delete mode 100644 testsuite/input-files/test08/b delete mode 100644 testsuite/input-files/test08/c delete mode 100644 testsuite/input-files/test08/d delete mode 100644 testsuite/tests-to-run/parallel-local164.sh delete mode 100755 testsuite/tests-to-run/test15.sh delete mode 100644 testsuite/wanted-results/parallel-local164 delete mode 100644 testsuite/wanted-results/test15 diff --git a/.gitignore b/.gitignore index 939636e4..beb99571 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later *~ *.pdf Makefile diff --git a/CITATION b/CITATION index aa79e248..2c9420c9 100644 --- a/CITATION +++ b/CITATION @@ -19,3 +19,7 @@ Or: DOI https://doi.org/10.5281/zenodo.11460 This helps funding further development. + +SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +SPDX-License-Identifier: GFDL-1.3-or-later +SPDX-License-Identifier: CC-BY-SA-4.0 diff --git a/Makefile.am b/Makefile.am index 7d970a00..745a1fcb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -130,6 +130,7 @@ reuse: (wd=`pwd`; \ cd /tmp/reuse; \ git clone "$$wd"; \ + cd parallel; \ reuse lint || \ (echo Use; \ echo ' reuse addheader --copyright="Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc." --license="GPL-3.0-or-later" source-files'; \ diff --git a/Makefile.in b/Makefile.in index f30c7d9e..8cd2bfb5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -881,6 +881,7 @@ reuse: (wd=`pwd`; \ cd /tmp/reuse; \ git clone "$$wd"; \ + cd parallel; \ reuse lint || \ (echo Use; \ echo ' reuse addheader --copyright="Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc." --license="GPL-3.0-or-later" source-files'; \ diff --git a/NEWS b/NEWS index d145d701..73da62e2 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,32 @@ +20210422 + +New in this release: + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* Bioinformatics tutorials - linux and shell advanced - parallel + https://www.youtube.com/watch?v=5leL8pyl0XA + +* GNU Parallel for quick gains + https://edbennett.github.io/high-performance-python/04-gnu-parallel/index.html + +* Processing Linux Commands in Parallel + https://www.baeldung.com/linux/processing-commands-in-parallel + +* GNU parallel + https://docs-research-it.berkeley.edu/services/high-performance-computing/user-guide/running-your-jobs/gnu-parallel/ + +* GNU Parallel 활용 가이드https://genoglobe.com/kribb/gnu_parallel + +* Parallel Grep and Awk + https://www.highonscience.com/blog/2021/03/21/parallel-grep/ + +* Getting things done with shell scripting + https://doma.dev/blog/get-things-done-with-bash/ + + 20210322 * Bug fixes and man page updates. diff --git a/README b/README index 26a01233..f3e95180 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20210322.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20210322.tar.bz2.sig - gpg parallel-20210322.tar.bz2.sig - bzip2 -dc parallel-20210322.tar.bz2 | tar xvf - - cd parallel-20210322 + wget https://ftpmirror.gnu.org/parallel/parallel-20210422.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20210422.tar.bz2.sig + gpg parallel-20210422.tar.bz2.sig + bzip2 -dc parallel-20210422.tar.bz2 | tar xvf - + cd parallel-20210422 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20210322.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20210322.tar.bz2.sig - gpg parallel-20210322.tar.bz2.sig - bzip2 -dc parallel-20210322.tar.bz2 | tar xvf - - cd parallel-20210322 + wget https://ftpmirror.gnu.org/parallel/parallel-20210422.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20210422.tar.bz2.sig + gpg parallel-20210422.tar.bz2.sig + bzip2 -dc parallel-20210422.tar.bz2 | tar xvf - + cd parallel-20210422 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2021, March 22). GNU Parallel 20210322 ('2002-01-06'). - Zenodo. https://doi.org/10.5281/zenodo.4628277 + Tange, O. (2021, April 22). GNU Parallel 20210422 ('Ever Given'). + Zenodo. https://doi.org/10.5281/zenodo.4710607 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ole Tange, http://ole.tange.dk and @@ -142,3 +142,6 @@ if you system has split up Perl into multiple packages then these are the important ones: opkg install perlbase-getopt perlbase-ipc procps-ng-ps perlbase-mime + +SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +SPDX-License-Identifier: GPL-3.0-or-later diff --git a/bootstrap b/bootstrap index c98fd759..2a40376f 100755 --- a/bootstrap +++ b/bootstrap @@ -1,5 +1,8 @@ #!/bin/bash -# I always forget how to make ./configure +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later -autoreconf --install -W gnu +autoreconf --install -W gnu && + ./configure && + make -j diff --git a/configure b/configure index 8f1a0070..5c48abf2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20210322. +# Generated by GNU Autoconf 2.69 for parallel 20210422. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20210322' -PACKAGE_STRING='parallel 20210322' +PACKAGE_VERSION='20210422' +PACKAGE_STRING='parallel 20210422' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20210322 to adapt to many kinds of systems. +\`configure' configures parallel 20210422 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20210322:";; + short | recursive ) echo "Configuration of parallel 20210422:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20210322 +parallel configure 20210422 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20210322, which was +It was created by parallel $as_me 20210422, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20210322' + VERSION='20210422' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20210322, which was +This file was extended by parallel $as_me 20210422, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20210322 +parallel config.status 20210422 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 823c2f0a..1398e857 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20210322], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20210422], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 88958adb..23adf8f4 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,8 +1,5 @@ -Luk filen ved EOF - lad være med bare at læse videre. - -> /tmp/ged; tail -f /tmp/ged| xargs -n1 -E eof & sleep 1; echo echo a >>/tmp/ged; echo eof >>/tmp/ged; seq 4 >>/tmp/ged; wait - -> /tmp/ged; tail -f /tmp/ged| parallel -n1 -E eof & sleep 1; echo echo a >>/tmp/ged; echo eof >>/tmp/ged; seq 4 >>/tmp/ged; wait +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later niceload seeks last column: @@ -10,17 +7,8 @@ iostat -x 1 2 niceload --start-condition -niceload should prioritize jobs and only unsuspend the highest -priority job. If the running job with lowest pri has run for 1 sec: -Consider unsuspending next pri job. - - Til QUOTING: -cat <<'_EOF' | parallel -v echo -awk -v FS="\",\"" '{print $1, $3, $4, $5, $9, $14}' | grep -v "#" | sed -e '1d' -e 's/\"//g' -e 's/\/\/\//\t/g' | cut -f1-6,11 | sed -e 's/\/\//\t/g' -e 's/ /\t/g -_EOF - FN="two spaces" echo 1 | parallel -q echo {} "$FN" # Prints 2 spaces between 'two' and 'spaces' @@ -49,16 +37,6 @@ echo 1 | parallel echo {} $FN4 \; echo $FN4 {} locate .gz | parallel -X find {} -size +1000 -size -2000 | parallel --workdir ... -S .. --trc {/}.bz2 'zcat {} | bzip2 > {/}.bz2' -== Compare == - -Unchanged since 2008 http://code.google.com/p/spawntool/ -Unchanged since 2011 http://code.google.com/p/push/ - -== Bug? == - -.parallel/config with --long-options - - == SQL == Example with %0a as newline @@ -66,12 +44,6 @@ sql :my_postgres?'\dt %0a SELECT * FROM users' cat ~/.sql/aliases | parallel --colsep '\s' sql {1} '"select 0.14+3;" | grep -q 3.14 || (echo dead: {1}; exit 1)' -== FEX == - -fex syntax for splitting fields -http://www.semicomplete.com/projects/fex/ -sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1} - --autocolsep: Læs alle linjer. Prøv fastlængde: Find tegn, som står i alle linjer på de samme pladser. Risiko for falske pos @@ -102,13 +74,17 @@ colsep = [sepchars]{no_of_sepchars} Til inspiration. -Hvis du stadig er ved at lave post- eller visitkort ting, så kunne du evt tilføje en QR code under frimærket. Med MECARD tagget kan flere tags gemmes i en og samme fil: +Hvis du stadig er ved at lave post- eller visitkort ting, så kunne du +evt tilføje en QR code under frimærket. Med MECARD tagget kan flere +tags gemmes i en og samme fil: -qrencode -l L -o x.png "MECARD:N:GNU Parallel;EMAIL:parallel@gnu.org;URL:gnu.org/software/parallel;" +qrencode -l L -o x.png "MECARD:N:GNU +Parallel;EMAIL:parallel@gnu.org;URL:gnu.org/software/parallel;" Den ser OK ud i en Androide tlf. -Husk at skrive indholdet under billedet, det er irreterende at skulle gætte. +Husk at skrive indholdet under billedet, det er irreterende at skulle +gætte. GNU parallel is a UNIX-tool for running commands in parallel. @@ -117,25 +93,6 @@ To gzip all files running one job per CPU write: Watch the intro video to learn more: http://pi.dk/1 Or read more about GNU parallel: http://gnu.org/s/parallel -job->start(): -$jobslot = Global::jobslot->$sshlogin - -sub get_jobslot { - my $sshlogin = shift; - my $jobslot_id = pop @Global::jobslots{$sshlogin}; - if not defined $jobslot_id { - $jobslot_id = ++$Global::max_jobslot_id; - } - return $jobslot_id; -} - -sub release_jobslot { - my $sshlogin = shift; - my $jobslot_id = shift; - push @Global::jobslots{$sshlogin}, $jobslot_id; -} - -Test sshlogins in parallel. Assume parallel is in path seq 1 10 | parallel -I {o} 'seq 1 255 | parallel echo ssh -oNoHostAuthenticationForLocalhost=true 127.0.{o}.{}' >/tmp/sshloginfile seq 1 1000 | parallel --sshloginfile /tmp/sshloginfile echo @@ -720,50 +677,3 @@ parallel -a table_file.tsv --colsep '\t' mycmd -o {2} {3} -i {1} # Run traceroute in parallel, but keep the output order the same parallel -k traceroute ::: foss.org.my debian.org freenetproject.org - -Test of signal passing through ssh -#!/bin/bash - -SERVER1=parallel-server3 -SERVER2=parallel-server2 - -export BG_PROC - -start_remote_sleep() { - parallel -D -u -S parallel@$SERVER2 sleep ::: 370 & - BG_PROC=$! - while ! ssh parallel@$SERVER2 ps -A -o cmd | grep -q '^sleep 370' ; do - sleep 0.3 - done -} - -stop_local_parallel() { - kill -9 $BG_PROC -} - -check_and_stop_remote_sleep() { - ssh parallel@$SERVER2 ps -A -o cmd | grep '^sleep 370' - ssh parallel@$SERVER2 killall sleep -} - -echo '### Test kill signals' -start_remote_sleep 2>/dev/null -kill -1 $BG_PROC -check_and_stop_remote_sleep - -sub propagate_signal { - my $signal = shift; -# $signal = "KILL"; - ::debug("Sending $signal to ",keys %Global::running); - kill $signal, keys %Global::running; - if(defined $Global::original_sig{$signal}) { - &{$Global::original_sig{$signal}}; - } -} - -my %do_not_propagate = map { $_ => 1 } qw(TTOU TTIN CONT TSTP __WARN__ __DIE__); -for (keys %SIG) { - $do_not_propagate{$_} and next; - $SIG{$_} = eval 'sub { propagate_signal("'.$_.'"); };'; -} - diff --git a/doc/boxplot-overhead b/doc/boxplot-overhead index 0240b25f..b95385fc 100644 --- a/doc/boxplot-overhead +++ b/doc/boxplot-overhead @@ -1,5 +1,8 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + # Create a boxplot of running 1000 jobs 10 times on all released # versions of GNU Parallel diff --git a/doc/haikus b/doc/haikus index 1a63c48d..990ecdb9 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,5 +1,10 @@ +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GFDL-1.3-or-later +# SPDX-License-Identifier: CC-BY-SA-4.0 + Quote of the month: + Parallel is amazing! -- fatboy93@reddit @@ -121,10 +126,14 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + GNU Parallel is your friend. + Can shorten that time by X cores. + -- iRODS @irods@twitter + GNU Parallel is my new favorite thing -- Will Tejeda @thewilltejeda -GNU Parallel is a priceless timesaver + GNU Parallel is a priceless timesaver -- June "Wakalix" @TWakalix@twitter I think many people would be surprised to learn that GNU parallel is diff --git a/doc/pod2savannah_publicinfo b/doc/pod2savannah_publicinfo index 395faf9d..fc1ef01e 100644 --- a/doc/pod2savannah_publicinfo +++ b/doc/pod2savannah_publicinfo @@ -1,5 +1,8 @@ #!/usr/bin/perl +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + $line=""; while(<>) { diff --git a/doc/promo b/doc/promo index b4a441a8..e4fd4a7c 100644 --- a/doc/promo +++ b/doc/promo @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GFDL-1.3-or-later +# SPDX-License-Identifier: CC-BY-SA-4.0 + =head1 GNU Parallel 10 year anniversery - 2020-04-22 Git log entry 2010-04-22: diff --git a/doc/release_new_version b/doc/release_new_version index 407e1b18..836bf299 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GFDL-1.3-or-later +# SPDX-License-Identifier: CC-BY-SA-4.0 + = Notes on how to release new version = == alpha/beta/production == @@ -40,7 +44,7 @@ gpl v3 10.5281/zenodo.4381888 DOINO=3956817 -TAG='Privacy Shield' +SPCTAG='Privacy Shield' # Update version . packager/releasescripts/updateversion @@ -50,6 +54,7 @@ TAG='Privacy Shield' === Reconfig autoconf/automake, build tar.bz2 and test it === +. .last-doitag.txt make reconf && make pack_unpack_and_test_build @@ -81,6 +86,41 @@ make https://build.opensuse.org/package/show/home:tange/parallel # Check that one .deb (Debian 5.0) and one .rpm (CentOS) end with 'succeeded' +== Upload LBRY == + +lbry://@GnuParallel#4/parallel-20210322#4 + +An easy way to support GNU Parallel is to tip on LBRY. + +: + +If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include a command that uses GNU Parallel if you feel like it. + +Thumbnail: https://www.gnu.org/software/parallel/logo-gray+black10000.png + +Tags: gnu parallel software + + +file_path="parallel-20210422.tar.bz2" +name="GNU Parallel $YYYYMMDD ('$SPCTAG')" +title="$name" +description="$description" + +lbrynet publish \ + --bid=0.01 \ + --file_path="$file_path" \ + --name="$name" \ + --title="$title" \ + --description="$description" \ + --author="$author" \ + $languages_opt \ + $tags_opt \ + --license="$license" \ + --thumbnail_url="$thumbnail_url" \ + --release_time="$release_time" \ + --channel_name="$channel_name" \ + + == Update website == http://www.gnu.org/software/parallel/ @@ -113,11 +153,11 @@ git diff . .last-doitag.txt echo "Released as $YYYYMMDD ('$TAG')." | grep MyTag && (STOP;STOP;STOP) echo "$TAG" | grep ' ' && (STOP;STOP;STOP) -echo "Released as $YYYYMMDD ('$TAG')." +echo "Released as $YYYYMMDD ('$SPCTAG')." -git commit -a -m "Released as $YYYYMMDD ('$TAG')" -git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $TAG -git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $YYYYMMDD +git commit -a -m "Released as $YYYYMMDD ('$SPCTAG')" +git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$SPCTAG')" $TAG +git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$SPCTAG')" $YYYYMMDD torsocks git push torsocks git push origin $TAG @@ -201,9 +241,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20210422 ('<<>>') released <<[stable]>> +Subject: GNU Parallel 20210422 ('Ever Given') released <<[stable]>> -GNU Parallel 20210322 ('<<>>') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20210322 ('Ever Given') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4 <> @@ -213,7 +253,9 @@ It does not have to be as detailed as Juan's. It is perfectly fine if you just s Quote of the month: - <<>> + GNU Parallel is your friend. + Can shorten that time by X cores. + -- iRODS @irods@twitter New in this release: @@ -221,7 +263,19 @@ New in this release: News about GNU Parallel: -<<>> +* Bioinformatics tutorials - linux and shell advanced - parallel https://www.youtube.com/watch?v=5leL8pyl0XA + +* GNU Parallel for quick gains https://edbennett.github.io/high-performance-python/04-gnu-parallel/index.html + +* Processing Linux Commands in Parallel https://www.baeldung.com/linux/processing-commands-in-parallel + +* GNU parallel https://docs-research-it.berkeley.edu/services/high-performance-computing/user-guide/running-your-jobs/gnu-parallel/ + +* GNU Parallel 활용 가이드https://genoglobe.com/kribb/gnu_parallel + +* Parallel Grep and Awk https://www.highonscience.com/blog/2021/03/21/parallel-grep/ + +* Getting things done with shell scripting https://doma.dev/blog/get-things-done-with-bash/ Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html diff --git a/packager/0install/check b/packager/0install/check index 4ffc1bd8..cea3ea62 100755 --- a/packager/0install/check +++ b/packager/0install/check @@ -1,2 +1,6 @@ #!/bin/sh + +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + 0launch http://0install.net/2007/interfaces/FeedLint.xml http://git.savannah.gnu.org/cgit/parallel.git/plain/packager/0install/parallel.xml diff --git a/packager/0install/update b/packager/0install/update index a44cdbfc..87126428 100755 --- a/packager/0install/update +++ b/packager/0install/update @@ -1,5 +1,7 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later version=$1 if [ -z "${version}" ]; then diff --git a/packager/centos/install-parallel-centos-5.sh b/packager/centos/install-parallel-centos-5.sh index a7692981..c86c1924 100755 --- a/packager/centos/install-parallel-centos-5.sh +++ b/packager/centos/install-parallel-centos-5.sh @@ -17,6 +17,10 @@ # along with this program; if not, see # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA +# +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later if [ "$EUID" -ne 0 ] then echo "Please run as root" diff --git a/packager/debian/Makefile b/packager/debian/Makefile index ef433ea7..e8294c33 100644 --- a/packager/debian/Makefile +++ b/packager/debian/Makefile @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + all: clean makedist unpack build unpack: diff --git a/packager/debian/doc/index.html b/packager/debian/doc/index.html index 0e7b9cb1..148bb14c 100644 --- a/packager/debian/doc/index.html +++ b/packager/debian/doc/index.html @@ -1,6 +1,12 @@ + + Debian parallel package - build and execute shell command lines from standard input in parallel diff --git a/packager/obs/home:tange/parallel/.osc/_files b/packager/obs/home:tange/parallel/.osc/_files index bc4aa489..e0db7813 100644 --- a/packager/obs/home:tange/parallel/.osc/_files +++ b/packager/obs/home:tange/parallel/.osc/_files @@ -1,7 +1,7 @@ - - - - - - + + + + + + diff --git a/packager/obs/home:tange/parallel/.osc/parallel.spec b/packager/obs/home:tange/parallel/.osc/parallel.spec index ce2e2ad9..0bc73c63 100644 --- a/packager/obs/home:tange/parallel/.osc/parallel.spec +++ b/packager/obs/home:tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20210322 +Version: 20210422 Release: 1.3 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/packager/opensuse/parallel.spec b/packager/opensuse/parallel.spec deleted file mode 100644 index 68406e4c..00000000 --- a/packager/opensuse/parallel.spec +++ /dev/null @@ -1,72 +0,0 @@ -Summary: Shell tool for executing jobs in parallel -Name: parallel -Version: 20100906 -Release: 1 -License: GPL -Group: Productivity/File utilities -URL: ftp://ftp.gnu.org/gnu/parallel -Source0: %{name}-%{version}.tar.bz2 -BuildArch: noarch -BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot -Requires: perl -Recommends: ssh rsync - -%description -GNU Parallel is a shell tool for executing jobs in parallel using one -or more machines. A job is typically a single command or a small -script that has to be run for each of the lines in the input. The -typical input is a list of files, a list of hosts, a list of users, or -a list of tables. - -If you use xargs today you will find GNU Parallel very easy to use. If -you write loops in shell, you will find GNU Parallel may be able to -replace most of the loops and make them run faster by running jobs in -parallel. If you use ppss or pexec you will find GNU Parallel will -often make the command easier to read. - -GNU Parallel also makes sure output from the commands is the same -output as you would get had you run the commands sequentially. This -makes it possible to use output from GNU Parallel as input for other -programs. - -%prep -if [ "${RPM_BUILD_ROOT}x" == "x" ]; then - echo "RPM_BUILD_ROOT empty, bad idea!" - exit 1 -fi -if [ "${RPM_BUILD_ROOT}" == "/" ]; then - echo "RPM_BUILD_ROOT is set to "/", bad idea!" - exit 1 -fi -%setup -q - -%build -./configure -make - -%install -rm -rf $RPM_BUILD_ROOT -make install prefix=$RPM_BUILD_ROOT%{_prefix} exec_prefix=$RPM_BUILD_ROOT%{_prefix} \ - datarootdir=$RPM_BUILD_ROOT%{_prefix} docdir=$RPM_BUILD_ROOT%{_docdir} \ - mandir=$RPM_BUILD_ROOT%{_mandir} - -rm $RPM_BUILD_ROOT%{_docdir}/parallel.html -rm $RPM_BUILD_ROOT%{_docdir}/sem.html -rm $RPM_BUILD_ROOT%{_docdir}/sql.html - -%clean -rm -rf $RPM_BUILD_ROOT - -%files -%defattr(-,root,root,-) -/usr/bin/* -/usr/share/man/man1/* -%doc README NEWS src/parallel.html src/sem.html src/sql.html - -%changelog -* Mon Sep 06 2010 Ole Tange -- Upgrade to current git-version of source. Tested on build.opensuse.org -* Fri Aug 27 2010 Ole Tange -- Untested upgrade to current git-version of source. -* Sat Aug 08 2010 Markus Ammer -- Initial package setup. diff --git a/packager/releasescripts/updateversion b/packager/releasescripts/updateversion index 50532af4..c2d1a827 100755 --- a/packager/releasescripts/updateversion +++ b/packager/releasescripts/updateversion @@ -1,8 +1,12 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + updater() { export DOINO export TAG + export SPCTAG export YYYYMMDD=`yyyymmdd` export DOI=10.5281/zenodo.$DOINO export YYYY=${YYYYMMDD:0:4} @@ -10,8 +14,9 @@ updater() { export YYYYMMDD=${YYYYMM}22 export MON=`date +%b` export MONTH=`date +%B` - echo Tag=$TAG Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI - export TITLE="GNU Parallel $YYYYMMDD ('$TAG')" + TAG=$(echo $SPCTAG | perl -pe 's/ //g') + echo Tag=$TAG = "$SPCTAG" Date:$YYYYMMDD Year:$YYYY Mon:$MON Month:$MONTH DOI:$DOI + export TITLE="GNU Parallel $YYYYMMDD ('$SPCTAG')" if [ -z "$DOINO" ] ; then echo '*** Set DOINO and try again ***' @@ -21,14 +26,15 @@ updater() { echo . .last-doitag.txt return fi - if [ -z "$TAG" ] ; then - echo '*** Set TAG and try again ***' + if [ -z "$SPCTAG" ] ; then + echo '*** Set SPCTAG and try again ***' cat .last-doitag.txt echo . .last-doitag.txt return fi echo "export DOINO=$DOINO" > .last-doitag.txt - echo "export TAG=$TAG" >> .last-doitag.txt + echo "export TAG='$TAG'" >> .last-doitag.txt + echo "export SPCTAG='$SPCTAG'" >> .last-doitag.txt echo "export YYYYMMDD=$YYYYMMDD" >> .last-doitag.txt echo 'echo "$DOINO $YYYYMMDD ($TAG)"' >> .last-doitag.txt diff --git a/src/Makefile.am b/src/Makefile.am index 839206a2..4202706f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2002-2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + bin_SCRIPTS = parallel sql niceload parcat parset parsort \ env_parallel env_parallel.ash env_parallel.bash \ env_parallel.csh env_parallel.dash env_parallel.fish \ diff --git a/src/Makefile.in b/src/Makefile.in index e3f6c034..66038a1e 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -14,6 +14,10 @@ @SET_MAKE@ +# SPDX-FileCopyrightText: 2002-2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/src/env_parallel b/src/env_parallel index e47ece77..71c24a43 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -17,6 +17,9 @@ # along with this program; if not, see # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA +# +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# SPDX-License-Identifier: GPL-3.0-or-later grepq() { # grep -q for systems without -q diff --git a/src/env_parallel.ash b/src/env_parallel.ash index c87fea7d..35dd4d04 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 664796e7..7ce8e4b2 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -37,8 +37,9 @@ env_parallel() { } _bodies_of_ALIASES() { local _i - for _i in $@; do - if [ $(alias $_i | wc -l) == 1 ] ; then + for _i in "$@"; do + # shellcheck disable=SC2046 + if [ $(alias "$_i" | wc -l) == 1 ] ; then true Alias is a single line. Good. else _warning_PAR "Alias '$_i' contains newline." @@ -65,6 +66,7 @@ env_parallel() { echo '(_|TIMEOUT|GROUPS|FUNCNAME|DIRSTACK|PIPESTATUS|USERNAME|BASHPID|BASH_[A-Z_]+)' } _ignore_READONLY() { + # shellcheck disable=SC1078,SC1079,SC2026 readonly | perl -e '@r = map { chomp; # sh on UnixWare: readonly TIMEOUT @@ -196,15 +198,16 @@ env_parallel() { END { exit not $exit }' } _warning_PAR() { - echo "env_parallel: Warning: $@" >&2 + echo "env_parallel: Warning: $*" >&2 } _error_PAR() { - echo "env_parallel: Error: $@" >&2 + echo "env_parallel: Error: $*" >&2 } # Bash is broken in version 3.2.25 and 4.2.39 # The crazy '[ "`...`" == "" ]' is needed for the same reason if [ "`_which_PAR parallel`" == "" ]; then + # shellcheck disable=SC2016 _error_PAR 'parallel must be in $PATH.' return 255 fi @@ -226,7 +229,7 @@ env_parallel() { (_names_of_ALIASES; _names_of_FUNCTIONS; _names_of_VARIABLES) | - cat > $HOME/.parallel/ignored_vars + cat > "$HOME"/.parallel/ignored_vars return 0 fi @@ -388,7 +391,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " @@ -420,12 +423,13 @@ _parset_main() { if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then # $_parset_NAME contains , or space # Split on , or space to get the names + # shellcheck disable=SC2016,SC2046 eval "$( # Compute results into files ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") | # var1=`cat tmpfile1; rm tmpfile1` # var2=`cat tmpfile2; rm tmpfile2` - parallel --plain -q echo {2}='`cat {1}; rm {1}`' :::: - :::+ $( + parallel --plain -q echo '{2}=`cat {1}; rm {1}`' :::: - :::+ $( echo "$_parset_NAME" | perl -pe 's/,/ /g' ) ); diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 6773956f..fb7d6187 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index c2aa241f..1b69f910 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -368,7 +368,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 0ae4627f..1c8d4c13 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -371,7 +371,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 55507356..61df0821 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -27,6 +27,7 @@ # # SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. # SPDX-License-Identifier: GPL-3.0-or-later +# shellcheck disable=SC2006 env_parallel() { # env_parallel.sh @@ -36,8 +37,8 @@ env_parallel() { for _i in `alias 2>/dev/null | perl -ne 's/^alias //;s/^(\S+)=.*/$1/ && print' 2>/dev/null`; do # Check if this name really is an alias # or just part of a multiline alias definition - if alias $_i >/dev/null 2>/dev/null; then - echo $_i + if alias "$_i" >/dev/null 2>/dev/null; then + echo "$_i" fi done } @@ -47,7 +48,7 @@ env_parallel() { # alias myalias='definition' (FreeBSD ash) # so remove 'alias ' from first line for _i in "$@"; do - echo 'alias '"`alias $_i | perl -pe '1..1 and s/^alias //'`" + echo 'alias '"`alias "$_i" | perl -pe '1..1 and s/^alias //'`" done } _names_of_maybe_FUNCTIONS() { @@ -55,6 +56,7 @@ env_parallel() { } _names_of_FUNCTIONS() { # myfunc is a function + # shellcheck disable=SC2046 LANG=C type `_names_of_maybe_FUNCTIONS` | perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' } @@ -70,7 +72,7 @@ env_parallel() { for _i in "$@" do perl -e 'print @ARGV' "$_i=" - eval echo \"\$$_i\" | perl -e '$/=undef; $a=<>; chop($a); print $a' | + eval echo "\"\$$_i\"" | perl -e '$/=undef; $a=<>; chop($a); print $a' | perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;"; echo done @@ -80,6 +82,7 @@ env_parallel() { echo '(_|TIMEOUT)' } _ignore_READONLY() { + # shellcheck disable=SC1078,SC1079,SC2026 readonly | perl -e '@r = map { chomp; # sh on UnixWare: readonly TIMEOUT @@ -203,15 +206,16 @@ env_parallel() { END { exit not $exit }' } _warning_PAR() { - echo "env_parallel: Warning: $@" >&2 + echo "env_parallel: Warning: $*" >&2 } _error_PAR() { - echo "env_parallel: Error: $@" >&2 + echo "env_parallel: Error: $*" >&2 } if _which_PAR parallel >/dev/null; then true parallel found in path else + # shellcheck disable=SC2016 _error_PAR 'parallel must be in $PATH.' return 255 fi @@ -231,7 +235,7 @@ env_parallel() { (_names_of_ALIASES; _names_of_FUNCTIONS; _names_of_VARIABLES) | - cat > $HOME/.parallel/ignored_vars + cat > "$HOME"/.parallel/ignored_vars return 0 fi @@ -310,6 +314,7 @@ env_parallel() { unset _remove_bad_NAMES _grep_REGEXP unset _prefix_PARALLEL_ENV # Test if environment is too big + # shellcheck disable=SC2092 if `_which_PAR true` >/dev/null 2>/dev/null ; then parallel "$@" _parallel_exit_CODE=$? @@ -385,7 +390,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " @@ -417,12 +422,13 @@ _parset_main() { if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then # $_parset_NAME contains , or space # Split on , or space to get the names + # shellcheck disable=SC2016,SC2046 eval "` # Compute results into files ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") | # var1= cat tmpfile1; rm tmpfile1 # var2= cat tmpfile2; rm tmpfile2 - parallel -q echo {2}='\`cat {1}; rm {1}\`' :::: - :::+ \` + parallel --plain -q echo '{2}=\`cat {1}; rm {1}\`' :::: - :::+ \` echo "$_parset_NAME" | perl -pe 's/,/ /g' \` diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 2838c7fa..fe801fad 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -362,7 +362,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210323 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210422 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index 90291e80..c656f80f 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20210323; +$Global::version = 20210422; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 10619f37..006a3867 100755 --- a/src/parallel +++ b/src/parallel @@ -14,7 +14,7 @@ # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see +# along with this program; if not, see # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA # @@ -1030,8 +1030,8 @@ sub spreadstdin() { # Buf does not start with $recstart => There is garbage. # Make a single record of the garbage if($buf =~ - /(?s)^(?-s)( - (?:(?:(?!$recend$recstart)(?s).(?-s))*?$recend) + /(?s:^)( + (?:(?:(?!$recend$recstart)(?s:.))*?$recend) ) # Followed by recstart (?=$recstart)/mox and length $1 > 0) { @@ -1047,11 +1047,11 @@ sub spreadstdin() { $Global::max_number_of_args * ($Global::max_lines || 1); # (?!negative lookahead) is needed to avoid backtracking # See: https://unix.stackexchange.com/questions/439356/ - # (?s).(?-s) = (.|[\n]) but faster + # (?s:.) = (.|[\n]) but faster while($buf =~ - /(?s)^(?-s)( + /(?s:^)( # n more times recstart.*recend - (?:$recstart(?:(?!$recend$recstart)(?s)(.)(?-s))*?$recend){$n_records} + (?:$recstart(?:(?!$recend$recstart)(?s:.))*?$recend){$n_records} ) # Followed by recstart (?=$recstart)/mox and length $1 > 0) { @@ -1065,8 +1065,8 @@ sub spreadstdin() { sub pass_regexp() { # Find the last recend-recstart in $buf $eof and return; - # (?s).(?-s) = (.|[\n]) but faster - if($buf =~ /^((?s).(?-s)*$recend)$recstart(?s).(?-s)*?$/mox) { + # (?s:.) = (.|[\n]) but faster + if($buf =~ /^((?s:.)*$recend)$recstart(?s:.)*?$/mox) { $anything_written += write_record_to_pipe($chunk_number++,\$header,\$buf, $recstart,$recend,length $1); @@ -1246,20 +1246,18 @@ sub recstartrecend() { $recend = $opt::recend; if($opt::regexp and $recend eq '') { # --regexp --recend '' - $recend = '(?s).(?-s)'; + $recend = '(?s:.)'; } } if($opt::regexp) { + # Do not allow /x comments - to avoid having to quote space + $recstart = "(?-x:".$recstart.")"; + $recend = "(?-x:".$recend.")"; # If $recstart/$recend contains '|' # the | should only apply to the regexp $recstart = "(?:".$recstart.")"; $recend = "(?:".$recend.")"; - # Quote # and space - $recstart =~ s/#/\\#/g; - $recend =~ s/#/\\#/g; - $recstart =~ s/ /\\ /g; - $recend =~ s/ /\\ /g; } else { # $recstart/$recend = printf strings (\n) $recstart =~ s/\\([0rnt\'\"\\])/"qq|\\$1|"/gee; @@ -2189,7 +2187,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20210323; + $Global::version = 20210422; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -2246,8 +2244,12 @@ sub init_globals() { '{:(\d+?):(\d+?)}' => '$_ = substr($_,$$1,$$2);', # Bash ${a#bc} '{#([^#}][^}]*?)}' => 's/^$$1//;', + # Bash ${a##bc} + '{##([^#}][^}]*?)}' => 's/^$$1//;', # Bash ${a%def} '{%([^}]+?)}' => 's/$$1$//;', + # Bash ${a%%def} + '{%%([^}]+?)}' => 's/$$1$//;', # Bash ${a/def/ghi} ${a/def/} '{/([^}]+?)/([^}]*?)}' => 's/$$1/$$2/;', # Bash ${a^a} @@ -4984,8 +4986,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2021, March 22). GNU Parallel 20210322 ('2002-01-06').", - " Zenodo. https://doi.org/10.5281/zenodo.4628277", + " Tange, O. (2021, April 22). GNU Parallel 20210422 ('Ever Given').", + " Zenodo. https://doi.org/10.5281/zenodo.4710607", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice @@ -5015,8 +5017,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2021, March 22). GNU Parallel 20210322 ('2002-01-06').", - " Zenodo. https://doi.org/10.5281/zenodo.4628277", + " Tange, O. (2021, April 22). GNU Parallel 20210422 ('Ever Given').", + " Zenodo. https://doi.org/10.5281/zenodo.4710607", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -5139,20 +5141,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2021_4628277,", + "\@software{tange_2021_4710607,", " author = {Tange, Ole},", - " title = {GNU Parallel 20210322 ('2002-01-06')},", - " month = Mar,", - " year = 2020,", + " title = {GNU Parallel 20210422 ('Ever Given')},", + " month = Apr,", + " year = 2021,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.4628277},", - " url = {https://doi.org/10.5281/zenodo.4628277}", + " doi = {10.5281/zenodo.4710607},", + " url = {https://doi.org/10.5281/zenodo.4710607}", "}", "", - "(Feel free to use \\nocite{tange_2021_4628277})", + "(Feel free to use \\nocite{tange_2021_4710607})", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -8020,15 +8022,15 @@ sub cleanup_cmd($$$) { $dir .= $_."/"; unshift @rmdir, ::shell_quote_file($dir); } - my $rmdir = @rmdir ? "sh -c ".::Q("rmdir @rmdir 2>/dev/null;") : ""; + my $rmdir = @rmdir ? "rmdir @rmdir 2>/dev/null;" : ""; if(defined $opt::workdir and $opt::workdir eq "...") { - $rmdir .= ::Q("rm -rf " . ::shell_quote_file($workdir).';'); + $rmdir .= "rm -rf " . ::shell_quote_file($workdir).';'; } - - $f = ::shell_quote_file($f); + my $rmf = "sh -c ". + ::Q("rm -f ".::shell_quote_file($f)." 2>/dev/null;".$rmdir); my $sshcmd = $self->sshcommand(); my $serverlogin = $self->serverlogin(); - return "$sshcmd $serverlogin -- ".::Q("rm -f $f; $rmdir"); + return "$sshcmd $serverlogin -- ".::Q("$rmf"); } { @@ -9263,12 +9265,13 @@ sub sshlogin_wrap($) { exec $shell, "-c", ($bashfunc."@ARGV"); die "exec: $!\n"; } + my $parent = getppid; do { - # Parent is not init (ppid=1), so sshd is alive + # Parent pid is not changed, so sshd is alive # Exponential sleep up to 1 sec $s = $s < 1 ? 0.001 + $s * 1.03 : $s; select(undef, undef, undef, $s); - } until ($done || getppid == 1); + } until ($done || getppid != $parent); if(not $done) { # Kill as per --termseq my @term_seq = split/,/,$termseq; @@ -12674,21 +12677,27 @@ sub total_jobs() { $Global::unquote_arg = 1; } sub yyyy_mm_dd_hh_mm_ss() { + # ISO8601 2038-01-19T03:14:08 ::strftime("%Y-%m-%dT%H:%M:%S", localtime(time())); } sub yyyy_mm_dd_hh_mm() { + # ISO8601 2038-01-19T03:14 ::strftime("%Y-%m-%dT%H:%M", localtime(time())); } sub yyyy_mm_dd() { + # ISO8601 2038-01-19 ::strftime("%Y-%m-%d", localtime(time())); } sub yyyymmddhhmmss() { + # ISO8601 20380119031408 ::strftime("%Y%m%d%H%M%S", localtime(time())); } sub yyyymmddhhmm() { + # ISO8601 203801190314 ::strftime("%Y%m%d%H%M", localtime(time())); } sub yyyymmdd() { + # ISO8601 20380119 ::strftime("%Y%m%d", localtime(time())); } diff --git a/src/parallel.pod b/src/parallel.pod index b57ddfe5..ee3e1c25 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -67,12 +67,12 @@ how-to, reference and explanation. =head3 Tutorial If you prefer reading a book buy B at -http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html +https://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html or download it at: https://doi.org/10.5281/zenodo.1146014 Read at least chapter 1+2. It should take you less than 20 minutes. Otherwise start by watching the intro videos for a quick introduction: -http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 +https://youtube.com/playlist?list=PL284C9FF2488BC6D1 If you want to dive deeper: spend a couple of hours walking through the tutorial (B). Your command line will love @@ -352,6 +352,10 @@ sequence number of job the arguments +=item Z<> B + +skip this job (see also B<--filter>) + =item Z<> B =item Z<> B @@ -668,12 +672,12 @@ files on the remote computer after processing is done. --return {.}.bz2 --cleanup "zcat {} | bzip -9 >{.}.bz2" With B<--transferfile {}> the file transferred to the remote computer -will be removed on the remote computer. Directories created will not -be removed - even if they are empty. +will be removed on the remote computer. Directories on the remote +computer containing the file will be removed if they are empty. With B<--return> the file transferred from the remote computer will be -removed on the remote computer. Directories created will not be -removed - even if they are empty. +removed on the remote computer. Directories on the remote +computer containing the file will be removed if they are empty. B<--cleanup> is ignored when not used with B<--transferfile> or B<--return>. @@ -697,7 +701,7 @@ B<--colsep> implies B<--trim rl>, which can be overridden with B<--trim n>. I is a Perl Regular Expression: -http://perldoc.perl.org/perlre.html +https://perldoc.perl.org/perlre.html =item B<--compress> @@ -2132,6 +2136,8 @@ B<--return> is often used with B<--transferfile> and B<--cleanup>. B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. +For details on transferring see B<--transferfile>. + =item B<--round-robin> @@ -2372,7 +2378,7 @@ On FreeBSD B is needed: https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2 There are many limitations of shebang (#!) depending on your operating -system. See details on http://www.in-ulm.de/~mascheck/various/shebang/ +system. See details on https://www.in-ulm.de/~mascheck/various/shebang/ =item B<--shebang-wrap> @@ -2789,29 +2795,41 @@ Transfer files to remote computers. Shorthand for: B<--transferfile {}>. B<--transferfile> is used with B<--sshlogin> to transfer files to the remote computers. The files will be transferred using B and -will be put relative to the default work dir. If the path contains /./ -the remaining path will be relative to the work dir. E.g. +will be put relative to the work dir (see B<--workdir>). + +The I will normally contain a replacement string. + +If the path contains /./ the remaining path will be relative to the +work dir (for details: see B). If the work dir is +B, the transferring will be as follows: + + /tmp/foo/bar => /tmp/foo/bar + tmp/foo/bar => /home/user/tmp/foo/bar + /tmp/./foo/bar => /home/user/foo/bar + tmp/./foo/bar => /home/user/foo/bar + +I + +This will transfer the file I to the computer +I to the file I<$HOME/foo/bar.txt> before running +B on I: echo foo/bar.txt | parallel --transferfile {} \ --sshlogin server.example.com wc -This will transfer the file I to the computer -I to the file I<$HOME/foo/bar.txt> before running -B on I. +This will transfer the file I to the computer +I to the file I before running +B on I: echo /tmp/foo/bar.txt | parallel --transferfile {} \ --sshlogin server.example.com wc This will transfer the file I to the computer -I to the file I before running -B on I. +I to the file I before running +B on I: echo /tmp/./foo/bar.txt | parallel --transferfile {} \ - --sshlogin server.example.com wc {= s:.*/./:./: =} - -This will transfer the file I to the computer -I to the file I before running -B on I. + --sshlogin server.example.com wc {= s:.*/\./:./: =} B<--transferfile> is often used with B<--return> and B<--cleanup>. A shorthand for B<--transferfile {}> is B<--transfer>. @@ -2950,7 +2968,8 @@ Print the version GNU B and exit. =item B<--wd> I -Jobs will be run in the dir I. +Jobs will be run in the dir I. The default is the current dir +for the local machine, and the login dir for remote computers. Files transferred using B<--transferfile> and B<--return> will be relative to I on remote computers. @@ -3335,7 +3354,7 @@ To remove a string anywhere you can use regular expressions with Let us assume a website stores images like: - http://www.example.com/path/to/YYYYMMDD_##.jpg + https://www.example.com/path/to/YYYYMMDD_##.jpg where YYYYMMDD is the date and ## is the number 01-24. This will download images for the past 30 days: @@ -3343,7 +3362,7 @@ download images for the past 30 days: getit() { date=$(date -d "today -$1 days" +%Y%m%d) num=$2 - echo wget http://www.example.com/path/to/${date}_${num}.jpg + echo wget https://www.example.com/path/to/${date}_${num}.jpg } export -f getit @@ -3871,13 +3890,13 @@ parallel. To download byte 10000000-19999999 you can use B: - curl -r 10000000-19999999 http://example.com/the/big/file >file.part + curl -r 10000000-19999999 https://example.com/the/big/file >file.part To download a 1 GB file we need 100 10MB chunks downloaded and combined in the correct order. seq 0 99 | parallel -k curl -r \ - {}0000000-{}9999999 http://example.com/the/big/file > file + {}0000000-{}9999999 https://example.com/the/big/file > file =head2 EXAMPLE: Parallel grep @@ -4146,6 +4165,15 @@ the special short hand I<-S ..> can be used: --trc {.}.bz2 "zcat {} | bzip2 -9 >{.}.bz2" +=head2 EXAMPLE: Advanced file transfer + +Assume you have files in in/*, want them processed on server, +and transferred back into /other/dir: + + parallel -S server --trc /other/dir/./{/}.out \ + cp {/} {/}.out ::: in/./* + + =head2 EXAMPLE: Distributing work to local and remote computers Convert *.mp3 to *.ogg running one process per CPU on local computer @@ -5347,7 +5375,7 @@ this. =item * The complete output of B. If you are not running -the latest released version (see http://ftp.gnu.org/gnu/parallel/) you +the latest released version (see https://ftp.gnu.org/gnu/parallel/) you should specify why you believe the problem is not fixed in that version. @@ -5384,7 +5412,7 @@ by others, the output might help them figure out the problem. =item * Whether you have watched the intro videos -(http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1), walked +(https://www.youtube.com/playlist?list=PL284C9FF2488BC6D1), walked through the tutorial (man parallel_tutorial), and read the EXAMPLE section in the man page (man parallel - search for EXAMPLE:). @@ -5393,8 +5421,8 @@ section in the man page (man parallel - search for EXAMPLE:). If you suspect the error is dependent on your environment or distribution, please see if you can reproduce the error on one of these VirtualBox images: -http://sourceforge.net/projects/virtualboximage/files/ -http://www.osboxes.org/virtualbox-images/ +https://sourceforge.net/projects/virtualboximage/files/ +https://www.osboxes.org/virtualbox-images/ Specifying the name of your distribution is not enough as you may have installed software that is not in the VirtualBox images. @@ -5467,7 +5495,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program. If not, see . +along with this program. If not, see . =head2 Documentation license I diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 691ba113..517d8406 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -575,7 +575,7 @@ using GNU B: echo unzip 1.zip >> /var/run/my_named_pipe; echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe -http://www.maier-komor.de/xjobs.html (Last checked: 2019-01) +https://www.maier-komor.de/xjobs.html (Last checked: 2019-01) =head2 DIFFERENCES BETWEEN prll AND GNU Parallel @@ -1096,9 +1096,9 @@ https://github.com/reconquest/orgalorg =head2 DIFFERENCES BETWEEN Rust parallel AND GNU Parallel -Rust parallel focuses on speed. It is almost as fast as B. It -implements a few features from GNU B, but lacks many -functions. All these fail: +Rust parallel focuses on speed. It is almost as fast as B, but +not as fast as B. It implements a few features from GNU +B, but lacks many functions. All these fail: # Read arguments from file parallel -a file echo @@ -3216,6 +3216,10 @@ https://github.com/themattrix/bash-concurrent =head2 Todo +http://code.google.com/p/spawntool/ + +http://code.google.com/p/push/ + https://github.com/mylanconnolly/parallel https://github.com/krashanoff/parallel @@ -3240,9 +3244,9 @@ https://github.com/flesler/parallel https://github.com/Julian/Verge -http://manpages.ubuntu.com/manpages/xenial/man1/tsp.1.html +https://manpages.ubuntu.com/manpages/xenial/man1/tsp.1.html -http://vicerveza.homeunix.net/~viric/soft/ts/ +https://vicerveza.homeunix.net/~viric/soft/ts/ https://github.com/chapmanjacobd/que @@ -3533,7 +3537,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program. If not, see . +along with this program. If not, see . =head2 Documentation license I diff --git a/src/parallel_book.pod b/src/parallel_book.pod index 74999e3f..2edd57b1 100644 --- a/src/parallel_book.pod +++ b/src/parallel_book.pod @@ -108,7 +108,7 @@ job slots with B<-j>. Give B<-j> the number of jobs to run in parallel: parallel -j50 \ - wget http://ftpmirror.gnu.org/parallel/parallel-{1}{2}22.tar.bz2 \ + wget https://ftpmirror.gnu.org/parallel/parallel-{1}{2}22.tar.bz2 \ ::: 2012 2013 2014 2015 2016 \ ::: 01 02 03 04 05 06 07 08 09 10 11 12 diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 410c7105..210084d5 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -14,11 +14,11 @@ real world. =head2 Reader's guide If you prefer reading a book buy B at -http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html +https://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html or download it at: https://doi.org/10.5281/zenodo.1146014 Otherwise start by watching the intro videos for a quick introduction: -http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 +https://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 Then browse through the Bs after the list of B in B (Use B). That will give @@ -3006,11 +3006,11 @@ Output: GNU parallel 20210122 Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. - License GPLv3+: GNU GPL version 3 or later + License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. - Web site: http://www.gnu.org/software/parallel + Web site: https://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication please cite as described in 'parallel --citation'. @@ -3045,7 +3045,7 @@ Output: month = {Feb}, number = {1}, volume = {36}, - url = {http://www.gnu.org/s/parallel}, + url = {https://www.gnu.org/s/parallel}, year = {2011}, pages = {42-47}, doi = {10.5281/zenodo.16303} @@ -3121,7 +3121,7 @@ If you like GNU B: =item * (Re-)walk through the tutorial if you have not done so in the past year -(http://www.gnu.org/software/parallel/parallel_tutorial.html) +(https://www.gnu.org/software/parallel/parallel_tutorial.html) =item * diff --git a/src/parcat b/src/parcat index 3042c975..13708ebe 100755 --- a/src/parcat +++ b/src/parcat @@ -14,7 +14,7 @@ # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see +# along with this program; if not, see # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA # diff --git a/src/parsort b/src/parsort index 4145c510..5e97c02e 100755 --- a/src/parsort +++ b/src/parsort @@ -121,7 +121,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20210323; +$Global::version = 20210422; if($opt::version) { version(); exit 0; } @Global::sortoptions = shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); diff --git a/src/sem.pod b/src/sem.pod index c61a09f3..0e7bfb95 100644 --- a/src/sem.pod +++ b/src/sem.pod @@ -265,7 +265,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program. If not, see . +along with this program. If not, see . =head2 Documentation license I diff --git a/src/sql b/src/sql index c19d6fc8..402e96c0 100755 --- a/src/sql +++ b/src/sql @@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20210323; + $Global::version = 20210422; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/Fixed.sh b/testsuite/Fixed.sh index e4b5ef25..a1eff6e5 100644 --- a/testsuite/Fixed.sh +++ b/testsuite/Fixed.sh @@ -1,3 +1,7 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + ls tests-to-run/test*.sh | perl -pe 's:(.*/(.*)).sh:cp actual-results/$2 wanted-results/$2:' | sh -x diff --git a/testsuite/Makefile b/testsuite/Makefile index fb4020b9..5bb5500f 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + ### Full test set (except mem) testsuite: 3 true @@ -40,17 +44,15 @@ polar: ../src/parallel tests-to-run/*polar* wanted-results/*polar* prereqlocal touch ~/.parallel/will-cite # *local* only -local: testlocal - true - -testlocal: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal installparallel +local: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal installparallel time bash Start.sh local mem -# short and local -short: testlocalshort - true +# short +short: ../src/parallel tests-to-run/* wanted-results/* prereqlocal installparallel + TRIES=1 time bash Start.sh '' 'polar|mem|100s|300s' -testlocalshort: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal installparallel +# short and local +localshort: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal installparallel TRIES=1 time bash Start.sh local 'mem|100s|300s' # sql @@ -71,7 +73,6 @@ REQUIRE=bash -c '\ export -f is_installed; \ parallel -n2 --tag is_installed ::: \ script "echo | script -c echo -q /dev/null" \ - lsh "lsh lo echo lsh is installed" \ "ssh sh@lo" "timeout 20 ssh sh@lo true" \ "ssh csh@lo" "timeout 5 ssh csh@lo true" \ niceload "niceload echo niceload is installed" \ @@ -86,6 +87,7 @@ REQUIRE=bash -c '\ pv "echo | pv -qL 10" \ timeout "timeout 1 echo timeout is installed" \ ' +# lsh "lsh lo echo lsh is installed" \ prereqlocal: installparallel startvm ${REQUIRE} diff --git a/testsuite/Portable.sh b/testsuite/Portable.sh index bbe18c70..fe0bc308 100644 --- a/testsuite/Portable.sh +++ b/testsuite/Portable.sh @@ -1,5 +1,9 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + export LANG=C SHFILE=/tmp/unittest-parallel.sh diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index d7575120..a57b07f0 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -1,5 +1,9 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2002-2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + install_packages() { # After first run, set this: # export INSTALL=echo @@ -10,7 +14,7 @@ install_packages() { test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs" test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl" test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386" - test_pkgs="$test_pkgs libtest-nowarnings-perl" + test_pkgs="$test_pkgs libtest-nowarnings-perl xemacs21-bin" # Debian package packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15" @@ -377,25 +381,32 @@ rsync_versions() { ) } +install_tangetools() { + git clone https://gitlab.com/ole.tange/tangetools.git + cd tangetools && + make && + sudo make install +} + misc() { parallel --record-env - #sudo cp /usr/bin/tmux /usr/local/bin/tmux1.8 # Build locale for LC_ALL=zh_HK.big5hkscs perl -e 1 sudo locale-gen zh_HK } run() { install_packages && - install_oracle_client && - setup_databases && - add_server_to_hosts && - create_shell_logins && - copy_ssh_keys && - lsh_setup && - add_freebsd && - tmux_versions && - bash_versions && - rsync_versions && - misc + install_tangetools && + install_oracle_client && + setup_databases && + add_server_to_hosts && + create_shell_logins && + copy_ssh_keys && + lsh_setup && + add_freebsd && + tmux_versions && + bash_versions && + rsync_versions && + misc } diff --git a/testsuite/Start.sh b/testsuite/Start.sh index debaa4e6..4d45bc29 100755 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. +# +# SPDX-License-Identifier: GPL-3.0-or-later + # Argument can be substring of tests (such as 'local') export LANG=C diff --git a/testsuite/input-files/segfault/items-0.xi b/testsuite/input-files/segfault/items-0.xi deleted file mode 100644 index b217e3acac501907edc18ec4473db18bdefc7edf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63 zcmW;C$q9f!5Jkav?Jb9nP1KtsIBNWUggVwaHag3~S)XH}XA8NfP*(|dn-Fiv-|^wX E2W$NbCIA2c diff --git a/testsuite/input-files/segfault/trysegfault b/testsuite/input-files/segfault/trysegfault deleted file mode 100755 index 39384477..00000000 --- a/testsuite/input-files/segfault/trysegfault +++ /dev/null @@ -1,16 +0,0 @@ -stdout parallel -i -s28 -0 true from \{\} to x{}y < items-0.xi | egrep -v 'exit|Command|\.\.\.' | grep . - -# Fejler -#stdout parallel -Di -s28 -0 true from \{\} to x{}y < items-0.xi > /dev/null -# stdout parallel -Di -s28 -0 echo from \{\} to x{}y < items-0.xi > /dev/null - -#grep Segmentation /tmp/out && cat >/tmp/:out < /tmp/out - -# Denne fejler: seq 1 100 | parallel --eta trysegfault -#stdout stdout /usr/local/bin/parallel -Di -s26 -0 echo from \{\} to x{}y < items-0.xi > /tmp/out; -#grep Segmentation /tmp/out && cat >/tmp/:out < /tmp/out - - - -#/usr/local/bin/parallel -s26 -0 echo < items-0.xi > /tmp/out - diff --git a/testsuite/input-files/shebang/shebangwrap.gp b/testsuite/input-files/shebang/shebangwrap.gp deleted file mode 100755 index 050cef56..00000000 --- a/testsuite/input-files/shebang/shebangwrap.gp +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k A={} /usr/bin/gnuplot - -name=system("echo $A") -print name diff --git a/testsuite/input-files/shebang/shebangwrap.oct b/testsuite/input-files/shebang/shebangwrap.oct deleted file mode 100755 index 99630f3d..00000000 --- a/testsuite/input-files/shebang/shebangwrap.oct +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/octave -qf - -arg_list = argv (); -filename = arg_list{1}; -printf(filename); -printf("\n"); diff --git a/testsuite/input-files/shebang/shebangwrap.pl b/testsuite/input-files/shebang/shebangwrap.pl deleted file mode 100755 index b9fde22b..00000000 --- a/testsuite/input-files/shebang/shebangwrap.pl +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl - -print @ARGV,"\n"; diff --git a/testsuite/input-files/shebang/shebangwrap.py b/testsuite/input-files/shebang/shebangwrap.py deleted file mode 100755 index b316b631..00000000 --- a/testsuite/input-files/shebang/shebangwrap.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/python3 - -import sys - -print(str(sys.argv[1])) diff --git a/testsuite/input-files/shebang/shebangwrap.r b/testsuite/input-files/shebang/shebangwrap.r deleted file mode 100755 index 27cb1c9d..00000000 --- a/testsuite/input-files/shebang/shebangwrap.r +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave - -options <- commandArgs(trailingOnly = TRUE) -options diff --git a/testsuite/input-files/shebang/shebangwrap.rb b/testsuite/input-files/shebang/shebangwrap.rb deleted file mode 100755 index 297b1de6..00000000 --- a/testsuite/input-files/shebang/shebangwrap.rb +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ruby - -p ARGV diff --git a/testsuite/input-files/shebang/shebangwrap.sh b/testsuite/input-files/shebang/shebangwrap.sh deleted file mode 100755 index c2516531..00000000 --- a/testsuite/input-files/shebang/shebangwrap.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/local/bin/parallel --shebang-wrap -k /bin/sh - -echo "$@" diff --git a/testsuite/input-files/test08/a b/testsuite/input-files/test08/a deleted file mode 100644 index 75c86238..00000000 --- a/testsuite/input-files/test08/a +++ /dev/null @@ -1,3 +0,0 @@ -flyp -_PRE 8 -hatchname> 8 diff --git a/testsuite/input-files/test08/b b/testsuite/input-files/test08/b deleted file mode 100644 index dde9fdd6..00000000 --- a/testsuite/input-files/test08/b +++ /dev/null @@ -1,3 +0,0 @@ -flyp -_PRE 9 -hatchname> 8 diff --git a/testsuite/input-files/test08/c b/testsuite/input-files/test08/c deleted file mode 100644 index 3712521e..00000000 --- a/testsuite/input-files/test08/c +++ /dev/null @@ -1,3 +0,0 @@ -flyp -_PRE 19 -hatchname> 19 diff --git a/testsuite/input-files/test08/d b/testsuite/input-files/test08/d deleted file mode 100644 index 2f3c6a95..00000000 --- a/testsuite/input-files/test08/d +++ /dev/null @@ -1,4 +0,0 @@ -flyp -_PRE 19 -hatchname> 9 - diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 98d16afa..9c9f6ffa 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -59,8 +59,13 @@ par_open_files_blocks() { yes "`seq 3000`" | head -c 20M | stdout parallel -j10 --pipe -k echo {#} of 21 | - grep -v 'parallel: Warning: No more file handles.' | - grep -v 'Warning: Raising ulimit -n or /etc/security/limits.conf' + grep -v 'No more file handles.' | + grep -v 'Only enough file handles to run .* jobs in parallel.' | + grep -v 'Raising ulimit -n or /etc/security/limits.conf' | + grep -v 'Try running .parallel -j0 -N .* --pipe parallel -j0.' | + grep -v 'or increasing .ulimit -n. .try: ulimit -n .ulimit -Hn..' | + grep -v 'or increasing .nofile. in /etc/security/limits.conf' | + grep -v 'or increasing /proc/sys/fs/file-max' } par_pipe_unneeded_procs() { @@ -690,7 +695,7 @@ par_test_cpu_detection_cpuinfo() { } export -f test_one compgen -A function | grep ^cpu | sort | parallel -j0 -k test_one - rm ~/.parallel/tmp/sshlogin/*/cpuspec + rm ~/.parallel/tmp/sshlogin/*/cpuspec 2>/dev/null } par_test_cpu_detection_lscpu() { diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 21eeb0f9..cadb9281 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -469,6 +469,8 @@ par_keeporder_roundrobin() { fi } +# was -j6 before segfault circus export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | - parallel --delay 0.3 --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' + # parallel --delay 0.3 --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' + parallel --delay 0.3 --timeout 1000% -j1 --lb --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index ddb8c24a..95fd763c 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -8,6 +8,75 @@ # Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins +par_shebang() { + echo '### Test different shebangs' + gp() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k A={} /usr/bin/gnuplot +name=system("echo $A") +print name +EOF + true + } + oct() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/octave -qf +arg_list = argv (); +filename = arg_list{1}; +printf(filename); +printf("\n"); +EOF + true + } + pl() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/perl +print @ARGV,"\n"; +EOF + true + } + py() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/python3 +import sys +print(str(sys.argv[1])) +EOF + true + } + r() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/Rscript --vanilla --slave +options <- commandArgs(trailingOnly = TRUE) +options +EOF + true + } + rb() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k /usr/bin/ruby +p ARGV +EOF + true + } + sh() { + cat <<'EOF' +#!/usr/local/bin/parallel --shebang-wrap -k /bin/sh +echo "$@" +EOF + true + } + run() { + tmp=`tempfile` + "$@" > "$tmp" + chmod +x "$tmp" + "$tmp" A B C + rm "$tmp" + } + export -f run gp oct pl py r rb sh + + parallel --tag -k run ::: gp oct pl py r rb sh +} + par_pipe_regexp() { echo '### --pipe --regexp' gen() { @@ -36,6 +105,33 @@ EOF $p --recstart '.*, Start' -N1 'echo Record;cat' } +par_pipe_regexp_non_quoted() { + echo '### --pipe --regexp non_quoted \n' + gen() { + cat < "$tmp" + cat "$tmp" | perl -pe 's/\d+/999/g' + rm "$tmp" } par_maxargs() { @@ -197,7 +296,7 @@ par_totaljob_repl() { parallel -k -N7 --plus echo {#} {##} ::: {1..14} parallel -k -N7 --plus echo {#} {##} ::: {1..15} parallel -k -S 8/: -X --plus echo {#} {##} ::: {1..15} - parallel -k --plus -j 10 echo '{0#}/{##}:{0%}' ::: {1..5} ::: {1..4} + parallel -k --plus --delay 0.1 -j 10 'echo {0#}/{##}:{0%}' ::: {1..5} ::: {1..4} } par_jobslot_repl() { diff --git a/testsuite/tests-to-run/parallel-local-ssh6.sh b/testsuite/tests-to-run/parallel-local-ssh6.sh index 1673122e..72cd5e4c 100644 --- a/testsuite/tests-to-run/parallel-local-ssh6.sh +++ b/testsuite/tests-to-run/parallel-local-ssh6.sh @@ -192,12 +192,15 @@ par_timeout_onall() { par_rsync_3.2.3() { echo "bug #59006: rsync version 3.2.3 is not detected correctly" - PATH=$HOME/bin/rsync:$PATH + tmp=$(mktemp -d) + cp $(which rsync-v3.2.3) "$tmp"/rsync + PATH="$tmp":$PATH rsync --version | grep version rm -f bug59006 parallel --return {} -Sparallel@lo touch ::: bug59006 ls bug59006 rm -f bug59006 + rm -r "$tmp" } export -f $(compgen -A function | grep par_) diff --git a/testsuite/tests-to-run/parallel-local12.sh b/testsuite/tests-to-run/parallel-local12.sh index d18be68e..d9ca560a 100644 --- a/testsuite/tests-to-run/parallel-local12.sh +++ b/testsuite/tests-to-run/parallel-local12.sh @@ -20,6 +20,33 @@ resize=`resize` # sleep 2 # rm /tmp/parallel-script-for-script +echo '### Test xargs compatibility' + +echo /tmp/1 > /tmp/files +echo 1 > /tmp/1 + +echo 'xargs Expect: 3 1' +echo 3 | xargs -P 1 -n 1 -a /tmp/files cat - +echo 'parallel Expect: 3 1 via psedotty 2' +cat >/tmp/parallel-script-for-script </tmp/parallel-script-for-script2 </tmp/parallel-script-for-script2 <$b} <>;print $a[$#a/2]';} @@ -71,11 +70,6 @@ echo '### Test make .deb package'; cd ~/privat/parallel/packager/debian; stdout make | grep 'To install the GNU Parallel Debian package, run:' -echo '### Test of segfaulting issue' - echo 'This gave ~/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; - echo 'before adding wait() before exit'; - seq 1 300 | stdout parallel ./trysegfault - echo '### Test basic --arg-sep' parallel -k echo ::: a b diff --git a/testsuite/tests-to-run/parallel-local164.sh b/testsuite/tests-to-run/parallel-local164.sh deleted file mode 100644 index 77e9d41e..00000000 --- a/testsuite/tests-to-run/parallel-local164.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. -# -# SPDX-License-Identifier: GPL-3.0-or-later - -# -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r - -echo "### test08" - cd input-files/test08; - ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort -EOF diff --git a/testsuite/tests-to-run/parallel-local23.sh b/testsuite/tests-to-run/parallel-local23.sh index b432705f..a4dec605 100755 --- a/testsuite/tests-to-run/parallel-local23.sh +++ b/testsuite/tests-to-run/parallel-local23.sh @@ -66,9 +66,6 @@ echo '### --header regexp --round-robin' echo '### --header num --round-robin' (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort -echo '### shebang-wrap' - $NICEPAR -k {} {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] - echo 'bug #43967: Error if there exists a bin/zsh or bin/bash dir (with zsh or bash).' mkdir -p /tmp/bash$$/bash; PATH=/tmp/bash$$:$PATH parallel echo ::: OK; rm -rf /tmp/bash$$ diff --git a/testsuite/tests-to-run/test15.sh b/testsuite/tests-to-run/test15.sh deleted file mode 100755 index 3cfd74cc..00000000 --- a/testsuite/tests-to-run/test15.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc. -# -# SPDX-License-Identifier: GPL-3.0-or-later - -# Test xargs compatibility - -#rm -f ~/.parallel/will-cite - -cd input-files/test15 || cd ../input-files/test15 - -echo 'xargs Expect: 3 1' -echo 3 | xargs -P 1 -n 1 -a files cat - -echo 'parallel Expect: 3 1 via psedotty 2' -cat >/tmp/parallel-script-for-script </tmp/parallel-script-for-script2 <>8)'"'"; +par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$ENV{"PARALLEL_JOBSLOT"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;$termseq="";do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}my$parent=getppid;do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid!=$parent);if(not$done){my@term_seq=split/,/,$termseq;if(not@term_seq){@term_seq=("TERM",200,"TERM",100,"TERM",50,"KILL",25);}while(@term_seq&&kill(0,-$pid)){kill(shift@term_seq,-$pid);select(undef,undef,undef,(shift@term_seq)/1000);}}wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; par_remote_nice a b par_ssh ### use --ssh par_ssh Run through BARSSH? diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 98a6d5e5..61424a8c 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -1,3 +1,48 @@ +### Test xargs compatibility +xargs Expect: 3 1 +3 +1 +parallel Expect: 3 1 via psedotty 2 +via pseudotty +Academic tradition requires you to cite works you base your article on. +If you use programs that use GNU Parallel to process data for an article in a +scientific publication, please cite: + + Tange, O. (2021, March 22). GNU Parallel 20210322 ('2002-01-06'). + Zenodo. https://doi.org/10.5281/zenodo.4628277 + +This helps funding further development; AND IT WON'T COST YOU A CENT. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. + +More about funding GNU Parallel and the citation notice: +https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + +To silence this citation notice: run 'parallel --citation' once. + +via pseudotty +1 +xargs Expect: 1 3 +1 +3 +parallel Expect: 1 3 2 via pseudotty +via pseudotty +Academic tradition requires you to cite works you base your article on. +If you use programs that use GNU Parallel to process data for an article in a +scientific publication, please cite: + + Tange, O. (2021, March 22). GNU Parallel 20210322 ('2002-01-06'). + Zenodo. https://doi.org/10.5281/zenodo.4628277 + +This helps funding further development; AND IT WON'T COST YOU A CENT. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. + +More about funding GNU Parallel and the citation notice: +https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + +To silence this citation notice: run 'parallel --citation' once. + +1 +via pseudotty ### Test stdin goes to first command only ("cat" as argument) via pseudotty Academic tradition requires you to cite works you base your article on. diff --git a/testsuite/wanted-results/parallel-local150 b/testsuite/wanted-results/parallel-local150 index 22ee7ca2..a8244c39 100644 --- a/testsuite/wanted-results/parallel-local150 +++ b/testsuite/wanted-results/parallel-local150 @@ -144,11 +144,6 @@ echo "### bug #35268: shell_quote doesn't treats [] brackets correctly" echo '### Test make .deb package'; cd ~/privat/parallel/packager/debian; stdout make | grep 'To install the GNU Parallel Debian package, run:' ### Test make .deb package To install the GNU Parallel Debian package, run: -echo '### Test of segfaulting issue' -### Test of segfaulting issue - echo 'This gave ~/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; echo 'before adding wait() before exit'; seq 1 300 | stdout parallel ./trysegfault -This gave ~/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 -before adding wait() before exit echo '### Test basic --arg-sep' ### Test basic --arg-sep parallel -k echo ::: a b diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 deleted file mode 100644 index 29bf8e04..00000000 --- a/testsuite/wanted-results/parallel-local164 +++ /dev/null @@ -1,5 +0,0 @@ -echo "### test08" -### test08 - cd input-files/test08; ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort -b -d diff --git a/testsuite/wanted-results/parallel-local18 b/testsuite/wanted-results/parallel-local18 index 0b59c140..b27a4d4f 100644 --- a/testsuite/wanted-results/parallel-local18 +++ b/testsuite/wanted-results/parallel-local18 @@ -41,4 +41,4 @@ par_special_char a' * ? >o o o o o o &STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;'"'" BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -rlDzR -e'ssh -l parallel' lo:./abc-file.out ./.;ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file.out; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- rm -rf .TMPWORKDIR;exit $_EXIT_status; +ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -e "'"'@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;'"'" BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -rlDzR -e'ssh -l parallel' lo:./abc-file.out ./.;ssh -l parallel lo -- 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- rm -rf .TMPWORKDIR;exit $_EXIT_status; parset myvar1,myvar2 echo ::: a b echo $myvar1 echo $myvar2 @@ -1257,7 +1257,7 @@ scientific publication, please cite: author = {Tange, Ole}, - year = 2020, + year = 2021, note = {{GNU Parallel is a general parallelizer to run multiple serial command line programs in parallel without changing them.}}, @@ -1285,4 +1285,4 @@ mentioned in the release notes of next version of GNU Parallel. echo A echo B echo C -7 +8 diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 deleted file mode 100644 index db3a16a5..00000000 --- a/testsuite/wanted-results/test15 +++ /dev/null @@ -1,14 +0,0 @@ -xargs Expect: 3 1 -3 -1 -parallel Expect: 3 1 via psedotty 2 -via pseudotty -via pseudotty -1 -xargs Expect: 1 3 -1 -3 -parallel Expect: 1 3 2 via pseudotty -via pseudotty -1 -via pseudotty diff --git a/testsuite/wanted-results/test19 b/testsuite/wanted-results/test19 index 6e108572..b4cda840 100644 --- a/testsuite/wanted-results/test19 +++ b/testsuite/wanted-results/test19 @@ -82,12 +82,12 @@ Input for ssh newlineX.out' -l vagrant one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. newlineX.out2' --l vagrant one-server -- rm -f './tmp/parallel.file. -newlineX'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l vagrant one-server -- rm -f './tmp/parallel.file. -newlineX.out'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l vagrant one-server -- rm -f './tmp/parallel.file. -newlineX.out2'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' +-l vagrant one-server -- sh -c 'rm -f '"'"'./tmp/parallel.file. +newlineX'"'"' 2>/dev/null;rmdir ./tmp/ ./ 2>/dev/null;' +-l vagrant one-server -- sh -c 'rm -f '"'"'./tmp/parallel.file. +newlineX.out'"'"' 2>/dev/null;rmdir ./tmp/ ./ 2>/dev/null;' +-l vagrant one-server -- sh -c 'rm -f '"'"'./tmp/parallel.file. +newlineX.out2'"'"' 2>/dev/null;rmdir ./tmp/ ./ 2>/dev/null;' -l vagrant one-server -- mkdir -p ./. -l vagrant one-server rsync --server -lDrRze.iLsfxC . ./. -l vagrant one-server -- exec perl -e '@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;' base64 @@ -95,9 +95,9 @@ newlineX.out2'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' newlineX.out' -l vagrant one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. newlineX.out2' --l vagrant one-server -- rm -f './tmp/parallel.file. -newlineX'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l vagrant one-server -- rm -f './tmp/parallel.file. -newlineX.out'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l vagrant one-server -- rm -f './tmp/parallel.file. -newlineX.out2'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' +-l vagrant one-server -- sh -c 'rm -f '"'"'./tmp/parallel.file. +newlineX'"'"' 2>/dev/null;rmdir ./tmp/ ./ 2>/dev/null;' +-l vagrant one-server -- sh -c 'rm -f '"'"'./tmp/parallel.file. +newlineX.out'"'"' 2>/dev/null;rmdir ./tmp/ ./ 2>/dev/null;' +-l vagrant one-server -- sh -c 'rm -f '"'"'./tmp/parallel.file. +newlineX.out2'"'"' 2>/dev/null;rmdir ./tmp/ ./ 2>/dev/null;'