From 79b56269bbe665fd3ec00b312919418ad82a8c3b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 7 Aug 2020 22:20:30 +0200 Subject: [PATCH] parallel: Fixed bug #58802. Now passes testsuite. --- NEWS | 21 +- README | 4 + doc/haikus | 19 +- doc/release_new_version | 8 +- packager/obs/Makefile | 3 + packager/obs/home:tange/parallel/.osc/_files | 9 +- src/niceload | 2 +- src/parallel | 105 ++++- src/parallel.pod | 4 +- src/parallel_tutorial.pod | 3 +- src/sql | 2 +- testsuite/Makefile | 21 +- testsuite/tests-to-run/parallel-freebsd.sh | 20 +- testsuite/tests-to-run/parallel-local-10s.sh | 90 ++-- testsuite/tests-to-run/parallel-local-30s.sh | 10 +- testsuite/tests-to-run/parallel-local-ssh5.sh | 21 +- testsuite/tests-to-run/parallel-local-ssh7.sh | 85 ++-- testsuite/tests-to-run/parallel-local-ssh8.sh | 14 +- testsuite/tests-to-run/parallel-local164.sh | 2 +- testsuite/tests-to-run/parallel-local7.sh | 18 - testsuite/tests-to-run/parallel-remote1.sh | 22 +- testsuite/tests-to-run/test17.sh | 11 +- testsuite/tests-to-run/test18.sh | 14 +- testsuite/tests-to-run/test19.sh | 8 +- testsuite/tests-to-run/test21.sh | 22 +- testsuite/tests-to-run/test23.sh | 8 +- testsuite/tests-to-run/test30.sh | 9 +- testsuite/tests-to-run/test35.sh | 10 +- testsuite/tests-to-run/test36.sh | 14 +- testsuite/tests-to-run/test37.sh | 34 +- testsuite/tests-to-run/test41.sh | 8 +- testsuite/tests-to-run/test47.sh | 16 +- testsuite/tests-to-run/test60.sh | 11 +- testsuite/tests-to-run/test61.sh | 6 +- testsuite/vagrant/Vagrantfile.tmpl | 75 +++ testsuite/vagrant/setup.sh | 107 +++++ testsuite/wanted-results/parallel-local-10s | 103 ++++- testsuite/wanted-results/parallel-local-30s | 426 +++++++++--------- testsuite/wanted-results/parallel-local-3s | 19 - testsuite/wanted-results/parallel-local-ssh7 | 41 +- testsuite/wanted-results/parallel-local164 | 6 +- testsuite/wanted-results/parallel-local19 | 2 +- testsuite/wanted-results/parallel-local22 | 4 +- testsuite/wanted-results/parallel-local23 | 4 +- testsuite/wanted-results/parallel-remote1 | 17 +- testsuite/wanted-results/parallel-tutorial | 5 +- testsuite/wanted-results/sql02 | 2 +- testsuite/wanted-results/test17 | 20 +- testsuite/wanted-results/test18 | 14 +- testsuite/wanted-results/test19 | 46 +- testsuite/wanted-results/test21 | 16 +- testsuite/wanted-results/test23 | 4 +- testsuite/wanted-results/test37 | 4 +- testsuite/wanted-results/test47 | 26 +- testsuite/wanted-results/test60 | 40 +- 55 files changed, 998 insertions(+), 637 deletions(-) create mode 100644 testsuite/vagrant/Vagrantfile.tmpl create mode 100644 testsuite/vagrant/setup.sh diff --git a/NEWS b/NEWS index d63a16e4..61428707 100644 --- a/NEWS +++ b/NEWS @@ -42,21 +42,28 @@ New in this release: News about GNU Parallel: -* GNU Parallel is used at NERSC: https://www.nersc.gov/assets/Uploads/08-Workflows-20200616.pdf +* GNU Parallel is used at NERSC: + https://www.nersc.gov/assets/Uploads/08-Workflows-20200616.pdf * Resumable batch jobs with GNU Parallel https://igor.io/parallel/ -* Lab Meeting technical talk https://www.slideshare.net/hoffmanlab/gnu-parallel-194030490 +* Lab Meeting technical talk + https://www.slideshare.net/hoffmanlab/gnu-parallel-194030490 -* Open Sky Software Website Credits https://www.openskysoftware.com/site-credits.htm +* Open Sky Software Website Credits + https://www.openskysoftware.com/site-credits.htm -* Speed up your Python Scripts using Ubuntu and GNU Parallel https://ecce.esri.ca/wpecce/2019/12/24/speed-up-your-python-scripts-using-ubuntu-and-gnu-parallel/ +* Speed up your Python Scripts using Ubuntu and GNU Parallel + https://ecce.esri.ca/wpecce/2019/12/24/speed-up-your-python-scripts-using-ubuntu-and-gnu-parallel/ -* Indispensable command-line tools https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html +* Indispensable command-line tools + https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html -* GNU Parallel for simulations http://pdebuyl.be/blog/2020/gnu-parallel-for-simulations.html +* GNU Parallel for simulations + http://pdebuyl.be/blog/2020/gnu-parallel-for-simulations.html -* Introduction to GNU parallel https://bioinformaticsworkbook.org/Appendix/GNUparallel/GNU_parallel_examples.html +* Introduction to GNU parallel + https://bioinformaticsworkbook.org/Appendix/GNUparallel/GNU_parallel_examples.html 20200522 diff --git a/README b/README index 97ef6e0a..33cb4ef4 100644 --- a/README +++ b/README @@ -125,6 +125,10 @@ publication please cite: Tange, O. (2020, July 22). GNU Parallel 20200722 ('Privacy Shield'). Zenodo. https://doi.org/10.5281/zenodo.3956817 +Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, +2016, 2017, 2018, 2019, 2020 Ole Tange, http://ole.tange.dk and Free +Software Foundation, Inc. + = New versions = diff --git a/doc/haikus b/doc/haikus index b0af68f2..aed96cd2 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,14 +1,15 @@ Quote of the month: -Gnu parallel is also awesome, fwiw. - -- Rogan Dawes @RoganDawes@twitter + Gnu parallel is also awesome, fwiw. + -- Rogan Dawes @RoganDawes@twitter -I get a weird sense of satisfaction every single time I see the lovely logo of #GNU Parallel (plus, what an underrated piece of great software!) - - -- Emre Sevinç @EmreSevinc@twitter + I get a weird sense of satisfaction every single time I see the + lovely logo of #GNU Parallel (plus, what an underrated piece of + great software!) + -- Emre Sevinç @EmreSevinc@twitter -I have gotten a *ton* of mileage out of jq, awk, and GNU parallel, + I have gotten a *ton* of mileage out of jq, awk, and GNU parallel, even at multi-GB sizes. -- Eric Wolak @ericthewolak@twitter @@ -31,9 +32,6 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html GNU Parallel is awesome -- Gaurav Jain @gauravjain49@twitter - With multicore systems everywhere GNU Parallel is a must have tool. - -- Neil H. Watson @neil_h_watson@twitter - My new favourite command line tool is GNU Parallel. After knowing it for a while I just recently came back to it for a refactor of a script and now I've changed like 4 different huge scripts into tiny parallel scripts that do their job literally 3 to 10 times faster and just [mind blown] -- Izzy @angry_izzy@twitter @@ -70,6 +68,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + With multicore systems everywhere GNU Parallel is a must have tool. + -- Neil H. Watson @neil_h_watson@twitter + Who needs spark when GNU Parallel exists -- MatthijsB @MatthijsBrs@twitter diff --git a/doc/release_new_version b/doc/release_new_version index 4d578338..b7504916 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -24,7 +24,7 @@ https://zenodo.org/deposit/new (*) Software (Reserve DOI) -GNU Parallel 20200622 ('Floyd') +GNU Parallel 20xxxx22 ('Xxxx') Ole Tange https://orcid.org/0000-0002-6345-1437 Description @@ -69,7 +69,7 @@ export YYYYMMDD cd ~/privat/parallel/packager/obs find home:tange/parallel/* -type f | - grep -v parallel.spec | + grep -Ev '(PKGBUILD|parallel.spec)' | parallel -j1 'osc rm {} || rm {}' # This should not create new files osc up home:tange/parallel/ @@ -189,9 +189,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20200722 ('Privacy Shield') released <<[stable]>> +Subject: GNU Parallel 20200822 ('Beirut/Juan Carlos') released <<[stable]>> -GNU Parallel 20200622 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20200822 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> diff --git a/packager/obs/Makefile b/packager/obs/Makefile index 3d4df5b1..f95d4bf2 100644 --- a/packager/obs/Makefile +++ b/packager/obs/Makefile @@ -5,7 +5,10 @@ all: cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/ cp `ls ../../parallel-2*.tar.bz2|grep -v latest|tail -n1` home\:tange/parallel/ perl -i -pe '/Version:/ and s/20\d\d\d\d\d\d/$$ENV{YYYYMMDD}/g' home:tange/parallel/parallel.spec + # Update sha256sum cd home:tange/parallel && s=`sha256sum parallel-*.tar.bz2|field 1` perl -i -pe 's/(sha256sums=..)\w+(..)/$$1$$ENV{s}$$2/' PKGBUILD + # Update version + cd home:tange/parallel && perl -i -pe 's/pkgver=.*/pkgver=$$ENV{YYYYMMDD}/' PKGBUILD cd home\:tange/parallel/ && osc diff cd home\:tange/parallel/ && osc up cd home\:tange/parallel/ && parallel osc add ::: *.spec *.dsc *.tar.gz *.tar.bz2 && echo Src added OK || true diff --git a/packager/obs/home:tange/parallel/.osc/_files b/packager/obs/home:tange/parallel/.osc/_files index da283c77..e4f5f949 100644 --- a/packager/obs/home:tange/parallel/.osc/_files +++ b/packager/obs/home:tange/parallel/.osc/_files @@ -1,10 +1,7 @@ - - - - - + + + - diff --git a/src/niceload b/src/niceload index 4028b043..2a3e79da 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20200722; +$Global::version = 20200723; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index a14f228e..e0a5dc8e 100755 --- a/src/parallel +++ b/src/parallel @@ -1618,6 +1618,7 @@ sub options_hash() { "halt-on-error|halt=s" => \$opt::halt, "limit=s" => \$opt::limit, "memfree=s" => \$opt::memfree, + "memsuspend=s" => \$opt::memsuspend, "retries=s" => \$opt::retries, "timeout=s" => \$opt::timeout, "termseq|term-seq=s" => \$opt::termseq, @@ -1838,6 +1839,8 @@ sub parse_options(@) { warning("--blocksize >= 64K causes problems on Cygwin."); } $opt::memfree = multiply_binary_prefix($opt::memfree); + $opt::memsuspend = multiply_binary_prefix($opt::memsuspend); + $Global::memlimit = $opt::memsuspend + $opt::memfree; check_invalid_option_combinations(); if((defined $opt::fifo or defined $opt::cat) and not $opt::pipepart) { @@ -2157,7 +2160,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20200722; + $Global::version = 20200623; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -3343,7 +3346,9 @@ sub init_run_jobs() { # Over limit next; } - if($opt::memfree and $sshlogin->memfree() < $opt::memfree) { + if(($opt::memfree or $opt::memsuspend) + and + $sshlogin->memfree() < $Global::memlimit) { # The server has not enough mem free ::debug("mem", "Not starting job: not enough mem\n"); next; @@ -3835,7 +3840,8 @@ sub progress() { $last_column_time = time; $columns = $ENV{'COLUMNS'}; if(not $columns) { - my $stty = ::qqx("stty -a /dev/null && true}; # FreeBSD/OpenBSD/NetBSD/Dragonfly/MirOS # MacOSX/IRIX/AIX/Tru64 $stty =~ /(\d+) columns/ and do { $columns = $1; }; @@ -3847,7 +3853,8 @@ sub progress() { $stty =~ /rows=\d+,(\d+)/ and do { $columns = $1; }; } if(not $columns) { - my $resize = ::qqx("resize"); + # && true is to force spawning a shell and not just exec'ing + my $resize = qx{resize 2>/dev/null && true}; $resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; }; } $columns ||= 80; @@ -4519,6 +4526,7 @@ sub onall($@) { my $options = join(" ", ((defined $opt::memfree) ? "--memfree ".$opt::memfree : ""), + ((defined $opt::memsuspend) ? "--memfree ".$opt::memsuspend : ""), ((defined $opt::D) ? "-D $opt::D" : ""), ((defined $opt::group) ? "-g" : ""), ((defined $opt::jobs) ? "-P $opt::jobs" : ""), @@ -4800,8 +4808,12 @@ sub kill_sleep_seq(@) { # Convert pids to process groups ($processgroup = -$pid) my @pgrps = map { -$_ } @_; my @term_seq = split/,/,$opt::termseq; - if(not @term_seq) { - @term_seq = ("TERM",200,"TERM",100,"TERM",50,"KILL",25); + if($opt::memsuspend) { + @term_seq = ("STOP",1); + } else { + if(not @term_seq) { + @term_seq = ("TERM",200,"TERM",100,"TERM",50,"KILL",25); + } } while(@term_seq) { @pgrps = kill_sleep(shift @term_seq, shift @term_seq, @pgrps); @@ -4907,8 +4919,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2020, July 22). GNU Parallel 20200722 ('Privacy Shield').", - " Zenodo. https://doi.org/10.5281/zenodo.3956817", + " Tange, O. (2020, June 22). GNU Parallel 20200622 ('Floyd').", + " Zenodo. https://doi.org/10.5281/zenodo.3903853", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice @@ -4938,8 +4950,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2020, July 22). GNU Parallel 20200722 ('Privacy Shield').", - " Zenodo. https://doi.org/10.5281/zenodo.3956817", + " Tange, O. (2020, June 22). GNU Parallel 20200622 ('Floyd').", + " Zenodo. https://doi.org/10.5281/zenodo.3903853", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -5052,20 +5064,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2020_3956817,", + "\@software{tange_2020_3903853,", " author = {Tange, Ole},", - " title = {GNU Parallel 20200722 ('Privacy Shield')},", - " month = Jul,", + " title = {GNU Parallel 20200622 ('Floyd')},", + " month = Jun,", " year = 2020,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.3956817},", - " url = {https://doi.org/10.5281/zenodo.3956817}", + " doi = {10.5281/zenodo.3903853},", + " url = {https://doi.org/10.5281/zenodo.3903853}", "}", "", - "(Feel free to use \\nocite{tange_2020_3956817})", + "(Feel free to use \\nocite{tange_2020_3903853})", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -5327,7 +5339,7 @@ sub qqx(@) { # ssh with ssh-agent needs PATH SSH_AUTH_SOCK SSH_AGENT_PID # ssh with Kerberos needs KRB5CCNAME # tmux needs LC_CTYPE - # lsh needs LOGNAME HOME + # lsh needs HOME LOGNAME my @keep = qw(PATH SSH_AUTH_SOCK SSH_AGENT_PID KRB5CCNAME LC_CTYPE HOME LOGNAME); @env{@keep} = @ENV{@keep}; local %ENV; @@ -5336,10 +5348,29 @@ sub qqx(@) { # && true is to force spawning a shell and not just exec'ing return qx{ @_ && true }; } else { - local *STDERR; - open(STDERR, ">", "/dev/null"); - # && true is to force spawning a shell and not just exec'ing - return qx{ @_ && true }; + # CygWin does not respect 2>/dev/null + # so we do that by hand + # This trick does not work: + # https://stackoverflow.com/questions/13833088/why-doesnt-local-work-on-stderr-and-stdout + # local *STDERR; + # open(STDERR, ">", "/dev/null"); + open(local *CHILD_STDIN, '<', '/dev/null') or die $!; + open(local *CHILD_STDERR, '>', '/dev/null') or die $!; + my $out; + # eval is needed if open3 fails (e.g. command line too long) + eval { + my $pid = open3( + '<&CHILD_STDIN', + $out, + '>&CHILD_STDERR', + # && true is to force spawning a shell and not just exec'ing + "@_ && true"); + my @arr = <$out>; + close $out; + # Make sure $? is set + waitpid($pid, 0); + return wantarray ? @arr : join "",@arr; + } } } @@ -5846,7 +5877,10 @@ sub reap_usleep() { $Global::timeoutq->process_timeouts(); } if($opt::memfree) { - kill_youngster_if_not_enough_mem(); + kill_youngster_if_not_enough_mem($opt::memfree*0.5); + } + if($opt::memsuspend) { + kill_youngster_if_not_enough_mem($opt::memsuspend*0.5); } if($opt::limit) { kill_youngest_if_over_limit(); @@ -5917,6 +5951,7 @@ sub kill_youngster_if_not_enough_mem() { # Put the child back in the queue. # Uses: # %Global::running + my $limit = shift; my %jobs_of; my @sshlogins; @@ -5928,12 +5963,12 @@ sub kill_youngster_if_not_enough_mem() { } for my $sshlogin (@sshlogins) { for my $job (sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}}) { - if($sshlogin->memfree() < $opt::memfree * 0.5) { + if($sshlogin->memfree() < $limit) { ::debug("mem","\n",map { $_->seq()." " } (sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}})); ::debug("mem","\n", $job->seq(), "killed ", - $sshlogin->memfree()," < ",$opt::memfree * 0.5); + $sshlogin->memfree()," < ",$limit); $job->kill(); $sshlogin->memfree_recompute(); } else { @@ -5941,7 +5976,7 @@ sub kill_youngster_if_not_enough_mem() { } } ::debug("mem","Free mem OK ", - $sshlogin->memfree()," > ",$opt::memfree * 0.5); + $sshlogin->memfree()," > ",$limit); } } @@ -8498,6 +8533,21 @@ sub kill($) { my $self = shift; $self->set_exitstatus(-1); ::kill_sleep_seq($self->pid()); + # push job onto start stack + if($opt::memsuspend) { + $self->{'suspended'} = 1; + $Global::JobQueue->{'commandlinequeue'}->unget($self); + } +} + +sub set_suspended($$) { + my $self = shift; + $self->{'suspended'} = shift; +} + +sub suspended($) { + my $self = shift; + return $self->{'suspended'}; } sub failed($) { @@ -9502,6 +9552,11 @@ sub start($) { } my $job = shift; + if($job->suspended()) { + # Job is kill -STOP'ped: Restart it. + kill "CONT", $job->pid(); + return $job; + } # Get the shell command to be executed (possibly with ssh infront). my $command = $job->wrapped(); my $pid; diff --git a/src/parallel.pod b/src/parallel.pod index 9395c056..5eb7a55a 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -481,8 +481,8 @@ completed, estimated seconds left, and number of jobs started. It is compatible with B: seq 1000 | parallel -j30 --bar '(echo {};sleep 0.1)' \ - 2> >(zenity --progress --auto-kill) | wc - + 2> >(perl -pe 'BEGIN{$/="\r";$|=1};s/\r/\n/g' | + zenity --progress --auto-kill) | wc =item B<--basefile> I diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 622126f1..8edbb060 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -1489,7 +1489,8 @@ A progress bar can be shown with B<--bar>: And a graphic bar can be shown with B<--bar> and B: seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \ - 2> >(zenity --progress --auto-kill --auto-close) + 2> >(perl -pe 'BEGIN{$/="\r";$|=1};s/\r/\n/g' | + zenity --progress --auto-kill --auto-close) A logfile of the jobs completed so far can be generated with B<--joblog>: diff --git a/src/sql b/src/sql index ead5c85e..79be87f6 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20200722; + $Global::version = 20200723; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/Makefile b/testsuite/Makefile index 2bce3804..23c7e19d 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -65,7 +65,7 @@ portable: time bash Portable.sh ### Prerequisites -prereqlocal: installparallel +prereqlocal: installparallel startvm tcsh -c echo tcsh installed || (echo tcsh is required for testsuite; /bin/false) seq 1 2 >/dev/null || (echo seq is required for testsuite; /bin/false) stdout echo || (echo stdout is required for testsuite; /bin/false) @@ -81,26 +81,21 @@ prereqlocal: installparallel which lsh || (echo lsh is required for testsuite; /bin/false) timeout 20 ssh sh@lo true || (echo ssh sh@lo is required for testsuite; /bin/false) timeout 5 ssh csh@lo true || (echo ssh csh@lo is required for testsuite; /bin/false) + parallel -j1 ssh-keygen -R parallel-server{} ::: 1 2 3 || true + parallel 'ssh vagrant@parallel-server{} mkdir -p bin; scp `which parallel` vagrant@parallel-server{}:bin/' ::: 1 2 3 4 prereqremote: installparallel startvm - parallel -j0 --timeout 10 --tag ssh parallel@parallel-server{} parallel --minversion 20121021 ::: 1 2 || (echo parallel on remote required for testsuite; /bin/true) + parallel -j0 --timeout 10 --tag ssh vagrant@parallel-server{} parallel --minversion 20121021 ::: 1 2 || (echo parallel on remote required for testsuite; /bin/true) startvm: - # Make sure we can reach the virtual machines - #sudo ifconfig wlp13s0:0 192.168.1.72 || sudo ifconfig wlan0:0 192.168.1.72 - sudo ifconfig enp14s0:0 192.168.1.72 || sudo ifconfig eth0:0 192.168.1.72 - # If they are already running: Don't fail - VBoxManage startvm CentOS3-root:centos3\ Clone2 || VBoxManage controlvm CentOS3-root:centos3\ Clone2 resume || true - VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true - parallel 'ping {} | read' ::: redhat9.tange.dk centos3.tange.dk + parallel 'cd vagrant/generic/{} && vagrant up' ::: centos8 freebsd11 freebsd12 rhel8 stopvm: - # || true - because this should not fail if the VM is not running - VBoxManage controlvm CentOS3-root:centos3 savestate || true - VBoxManage controlvm RedHat9-root:redhat9 savestate || true - VBoxManage controlvm OracleXE savestate || true + parallel 'cd vagrant/generic/{} && vagrant suspend' ::: centos8 freebsd11 freebsd12 rhel8 startdb: + true should start Oracle in vagrant +startdb-old: #echo shutdown abort | sudo su - oracle -c "sqlplus / as sysdba" sudo parallel /etc/init.d/{} status '||' /etc/init.d/{} restart ::: postgresql mysql # oracle-xe VBoxManage startvm OracleXE || true diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index 5c2ecbd0..a3e1c032 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -3,11 +3,15 @@ echo "### These tests requires VirtualBox running with the following images" echo `whoami`"@freebsd7" -VBoxManage startvm FreeBSD71 >/dev/null 2>&1 -ping -c 1 freebsd7.tange.dk >/dev/null 2>&1 +SERVER1=freebsd11 +SSHUSER1=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 -ssh freebsd7.tange.dk touch .parallel/will-cite -scp -q .*/src/{parallel,sem,sql,niceload,env_parallel*} freebsd7.tange.dk:bin/ +#VBoxManage startvm FreeBSD71 >/dev/null 2>&1 +#ping -c 1 freebsd7.tange.dk >/dev/null 2>&1 + +ssh $SSHLOGIN1 touch .parallel/will-cite +scp -q .*/src/{parallel,sem,sql,niceload,env_parallel*} $SSHLOGIN1:bin/ . `which env_parallel.bash` env_parallel --session @@ -49,13 +53,13 @@ par_shebang() { chmod 755 ./shebang; ./shebang echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' - (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; + (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/local/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap chmod 755 ./shebang-wrap ./shebang-wrap wrap works | sort -r echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' - (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/bin/perl -w :::' + (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/local/bin/perl -w :::' echo 'print @ARGV,"\n";') > shebang-wrap-opt; chmod 755 ./shebang-wrap-opt ./shebang-wrap-opt wrap works with options @@ -108,8 +112,8 @@ unset TMPDIR # We can safely ignore those. PARALLEL_SHELL=sh env_parallel --env _ -vj9 -k --joblog /tmp/jl-`basename $0` --retries 3 \ - -S freebsd7.tange.dk --tag '{} 2>&1' \ + -S $SSHLOGIN1 --tag '{} 2>&1' \ ::: $(compgen -A function | grep par_ | sort) \ 2> >(grep -Ev 'shopt: not found|declare: not found') -VBoxManage controlvm FreeBSD71 savestate +#VBoxManage controlvm FreeBSD71 savestate diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 00974e34..35afe4a3 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -16,7 +16,7 @@ par_bin() { paste <(seq 10) <(seq 10 -1 1) | parallel --pipe --colsep '\t' --bin '2 $_%=2' -j4 wc | sort # Fails - blocks! - # paste <(seq 10000000) <(seq 10000000 -1 1) | parallel --pipe --colsep '\t' --bin 2 wc + # paste <(seq 10) <(seq 10 -1 1) | parallel --pipe --colsep '\t' --bin 2 wc } par_nice() { @@ -103,10 +103,10 @@ par_colsep() { parallel -k --colsep %+ echo {1} {3} {2} {4} ::: a%c%%b a%c%b%d (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo {1} {3} {2} {4} (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --colsep %+ echo '"{1}_{3}_{2}_{4}"' - + echo '### Test of -C' (echo 'a% c %%b'; echo a%c% b %d) | parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' - + echo '### Test of --trim n' (echo 'a% c %%b'; echo a%c% b %d) | parallel -k --trim n --colsep %+ echo '"{1}_{3}_{2}_{4}"' parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' @@ -276,66 +276,77 @@ par_END() { par_xargs_compat() { echo xargs compatibility + a_b-c() { echo a_b; echo c; } + a_b_-c-d() { echo a_b' '; echo c; echo d; } + a_b_-c-d-e() { echo a_b' '; echo c; echo d; echo e; } + one_mb_line() { perl -e 'print "z"x1000000'; } + stdsort() { "$@" | LC_ALL=C sort; } echo '### Test -L -l and --max-lines' - (echo a_b;echo c) | parallel -km -L2 echo - (echo a_b;echo c) | parallel -k -L2 echo - (echo a_b;echo c) | xargs -L2 echo + a_b-c | parallel -km -L2 echo + a_b-c | parallel -k -L2 echo + a_b-c | xargs -L2 echo echo '### xargs -L1 echo' - (echo a_b;echo c) | parallel -km -L1 echo - (echo a_b;echo c) | parallel -k -L1 echo - (echo a_b;echo c) | xargs -L1 echo + a_b-c | parallel -km -L1 echo + a_b-c | parallel -k -L1 echo + a_b-c | xargs -L1 echo echo 'Lines ending in space should continue on next line' echo '### xargs -L1 echo' - (echo a_b' ';echo c;echo d) | parallel -km -L1 echo - (echo a_b' ';echo c;echo d) | parallel -k -L1 echo - (echo a_b' ';echo c;echo d) | xargs -L1 echo + a_b_-c-d | parallel -km -L1 echo + a_b_-c-d | parallel -k -L1 echo + a_b_-c-d | xargs -L1 echo echo '### xargs -L2 echo' - (echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo - (echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo - (echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo + a_b_-c-d-e | parallel -km -L2 echo + a_b_-c-d-e | parallel -k -L2 echo + a_b_-c-d-e | xargs -L2 echo echo '### xargs -l echo' - (echo a_b' ';echo c;echo d;echo e) | parallel -l -km echo # This behaves wrong - (echo a_b' ';echo c;echo d;echo e) | parallel -l -k echo # This behaves wrong - (echo a_b' ';echo c;echo d;echo e) | xargs -l echo + a_b_-c-d-e | parallel -l -km echo # This behaves wrong + a_b_-c-d-e | parallel -l -k echo # This behaves wrong + a_b_-c-d-e | xargs -l echo echo '### xargs -l2 echo' - (echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo - (echo a_b' ';echo c;echo d;echo e) | parallel -k -l2 echo - (echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo + a_b_-c-d-e | parallel -km -l2 echo + a_b_-c-d-e | parallel -k -l2 echo + a_b_-c-d-e | xargs -l2 echo echo '### xargs -l1 echo' - (echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo - (echo a_b' ';echo c;echo d;echo e) | parallel -k -l1 echo - (echo a_b' ';echo c;echo d;echo e) | xargs -l1 echo + a_b_-c-d-e | parallel -km -l1 echo + a_b_-c-d-e | parallel -k -l1 echo + a_b_-c-d-e | xargs -l1 echo echo '### xargs --max-lines=2 echo' - (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo - (echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo - (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo + a_b_-c-d-e | parallel -km --max-lines 2 echo + a_b_-c-d-e | parallel -k --max-lines 2 echo + a_b_-c-d-e | xargs --max-lines=2 echo echo '### xargs --max-lines echo' - (echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -km echo # This behaves wrong - (echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -k echo # This behaves wrong - (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo + a_b_-c-d-e | parallel --max-lines -km echo # This behaves wrong + a_b_-c-d-e | parallel --max-lines -k echo # This behaves wrong + a_b_-c-d-e | xargs --max-lines echo echo '### test too long args' - perl -e 'print "z"x1000000' | parallel echo 2>&1 - perl -e 'print "z"x1000000' | xargs echo 2>&1 - (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -km -s 10 echo - (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort xargs -s 10 echo - (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -kX -s 10 echo + one_mb_line | parallel echo 2>&1 + one_mb_line | xargs echo 2>&1 + (seq 1 10; one_mb_line; seq 12 15) | stdsort parallel -j1 -km -s 10 echo + (seq 1 10; one_mb_line; seq 12 15) | stdsort xargs -s 10 echo + (seq 1 10; one_mb_line; seq 12 15) | stdsort parallel -j1 -kX -s 10 echo echo '### Test -x' + echo '-km' (seq 1 10; echo 12345; seq 12 15) | stdsort parallel -j1 -km -s 10 -x echo + echo '-kX' (seq 1 10; echo 12345; seq 12 15) | stdsort parallel -j1 -kX -s 10 -x echo + echo '-x' (seq 1 10; echo 12345; seq 12 15) | stdsort xargs -s 10 -x echo + echo '-km -x' (seq 1 10; echo 1234; seq 12 15) | stdsort parallel -j1 -km -s 10 -x echo + echo '-kX -x' (seq 1 10; echo 1234; seq 12 15) | stdsort parallel -j1 -kX -s 10 -x echo + echo '-x' (seq 1 10; echo 1234; seq 12 15) | stdsort xargs -s 10 -x echo } @@ -534,6 +545,7 @@ par_retries_all_fail() { par_sockets_cores_threads() { echo '### Test --number-of-sockets/cores/threads' + unset PARALLEL_CPUINFO parallel --number-of-sockets parallel --number-of-cores parallel --number-of-threads @@ -593,6 +605,8 @@ export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | parallel --timeout 1000% -j10 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | perl -pe 's/,31,0/,15,0/' | - perl -pe 's:~:'$HOME':' | - perl -pe 's:'$PWD':.:' | - perl -pe 's:'$HOME':~:' + perl -pe 's/131\d\d\d/131XXX/' | + # Replace $PWD with . even if given as ~/... + perl -pe 's:~:'$HOME':g' | + perl -pe 's:'$PWD':.:g' | + perl -pe 's:'$HOME':~:g' diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 9354c0bb..1f69d835 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -146,8 +146,11 @@ par_groupby() { par_groupby_pipepart() { tsv() { - printf "%s\t" a1 b1 c1 d1 e1 f1; echo + # TSV file + printf "%s\t" header_a1 head_b1 c1 d1 e1 f1; echo + # Make 6 columns: 123456 => 1\t2\t3\t4\t5\t6 seq 100000 999999 | perl -pe '$_=join"\t",split//' | + # Sort reverse on column 3 (This should group on col 3) sort --parallel=8 --buffer-size=50% -rk3 } export -f tsv @@ -178,7 +181,8 @@ par_groupby_pipepart() { echo "### test $generator | --colsep $colsep --groupby $groupby" $generator > $tmp - parallel --pipepart -a $tmp --colsep "$colsep" --groupby "$groupby" -k 'echo NewRec; wc' + parallel --header 1 --pipepart -k \ + -a $tmp --colsep "$colsep" --groupby "$groupby" 'echo NewRec; wc' } export -f tester parallel --tag -k tester \ @@ -213,7 +217,7 @@ par_memory_leak() { } export -f a_run echo "### Test for memory leaks" - echo "Of 100 runs of 1 job none should be bigger than a 3000 job run" + echo "Of 100 runs of 1 job at least one should be bigger than a 3000 job run" . `which env_parallel.bash` parset small_max,big ::: 'seq 100 | parallel a_run 1 | jq -s max' 'a_run 3000' if [ $small_max -lt $big ] ; then diff --git a/testsuite/tests-to-run/parallel-local-ssh5.sh b/testsuite/tests-to-run/parallel-local-ssh5.sh index 8b6c9e97..63b30042 100644 --- a/testsuite/tests-to-run/parallel-local-ssh5.sh +++ b/testsuite/tests-to-run/parallel-local-ssh5.sh @@ -4,15 +4,17 @@ par_autossh() { echo '### --ssh autossh' - export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0 - parallel -S lo echo ::: OK - echo OK | parallel --pipe -S lo cat - parallel -S lo false ::: a || echo OK should fail - echo '### --ssh autossh - add commands that fail here' - touch foo_autossh - stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh - ls foo_autossh* - rm foo_autossh* + ( + export PARALLEL_SSH=autossh; export AUTOSSH_PORT=0 + stdout parallel -S lo echo ::: OK + echo OK | stdout parallel --pipe -S lo cat + stdout parallel -S lo false ::: a || echo OK should fail + echo '### --ssh autossh - add commands that fail here' + touch foo_autossh + stdout parallel -S csh@lo --trc {}.out touch {}.out ::: foo_autossh + ls foo_autossh* + rm foo_autossh* + ) | grep -Ev 'Warning: remote port forwarding failed for listen' } par_basefile_cleanup() { @@ -76,7 +78,6 @@ par_command_len_shellquote() { stdout parallel --tag -k outer ::: '-Slo -j10' '' | perl -pe 's/131\d\d\d/131xxx/g'; - } export -f $(compgen -A function | grep par_) diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 5fc86fb5..69a99618 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -1713,22 +1713,21 @@ _EOF par_bash_environment_too_big() { myscript=$(cat <<'_EOF' echo 'bug #50815: env_parallel should warn if the environment is too big' - len_functions=-$(typeset -f | wc -c)/1000 - len_variables=-$(typeset -p | wc -c)/1000 - len_var=$len_variables+110 - len_var_remote=$len_variables+100-50+25-12+6 - len_var_quote=$len_variables+100-50 - len_var_quote_remote=$len_variables+100-50-25+12 - len_fun=$len_functions+100 - len_fun_remote=$len_functions+100-50+25-12+6 - len_fun_quote=$len_functions+100 - len_fun_quote_remote=$len_functions+100-50 + len_overhead=-$( (shopt;alias;typeset -f;typeset -p) | wc -c)/1000 + len_var=$len_overhead+108 + len_var_remote=$len_overhead+50+25+6-3+1 + len_var_quote=$len_overhead+50+25-12-6 + len_var_quote_remote=$len_overhead+50-25+12+6-3 + len_fun=$len_overhead+100 + len_fun_remote=$len_overhead+50+25 + len_fun_quote=$len_overhead+100 + len_fun_quote_remote=$len_overhead+50+25 . `which env_parallel.bash`; repeat() { # Repeat input string n*1000 times - perl -e 'print ((shift)x(eval "1000*(".shift.")"))' "$@" + perl -e 'print ((shift)x(eval "1000*int(".shift.")"))' "$@" } bigvar=$(repeat x $len_var) @@ -2816,8 +2815,8 @@ par_dash_env_parallel_session() { alias level1alias='echo l1alias' echo '### level0 should be hidden, level1 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK @@ -2830,18 +2829,18 @@ par_dash_env_parallel_session() { alias level2alias='echo l2alias' echo '### level0+1 should be hidden, level2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel --endsession echo '### level0 should be hidden, level1+2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK @@ -3047,8 +3046,8 @@ par_mksh_env_parallel_session() { alias level1alias='echo l1alias' echo '### level0 should be hidden, level1 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK @@ -3060,18 +3059,18 @@ par_mksh_env_parallel_session() { alias level2alias='echo l2alias' echo '### level0+1 should be hidden, level2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel --endsession echo '### level0 should be hidden, level1+2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK @@ -3137,8 +3136,8 @@ par_sh_env_parallel_session() { # alias level1alias='echo l1alias' echo '### level0 should be hidden, level1 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK @@ -3150,18 +3149,18 @@ par_sh_env_parallel_session() { # alias level2alias='echo l2alias' echo '### level0+1 should be hidden, level2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel --endsession echo '### level0 should be hidden, level1+2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK @@ -3236,8 +3235,8 @@ par_zsh_env_parallel_session() { alias level1alias='echo l1alias' echo '### level0 should be hidden, level1 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK @@ -3249,18 +3248,18 @@ par_zsh_env_parallel_session() { alias level2alias='echo l2alias' echo '### level0+1 should be hidden, level2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail - env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK env_parallel --endsession echo '### level0 should be hidden, level1+2 should be transferred' - env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail - env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail + env_parallel -Slo 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 + env_parallel 'echo $level0var; level0func; level0alias; echo ${level0arr[*]}' ::: fail 2>&1 env_parallel -Slo 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel 'echo $level1var; level1func; level1alias; echo ${level1arr[*]}' ::: OK env_parallel -Slo 'echo $level2var; level2func; level2alias; echo ${level2arr[*]}' ::: OK diff --git a/testsuite/tests-to-run/parallel-local-ssh8.sh b/testsuite/tests-to-run/parallel-local-ssh8.sh index 32b49b3f..f73b10f5 100644 --- a/testsuite/tests-to-run/parallel-local-ssh8.sh +++ b/testsuite/tests-to-run/parallel-local-ssh8.sh @@ -7,7 +7,7 @@ par_path_remote_bash() { rm -rf /tmp/parallel cp /usr/local/bin/parallel /tmp - cat <<'_EOS' | stdout ssh nopathbash@lo -T | perl -ne '/logged in/..0 and print' | uniq + cat <<'_EOS' | echo logged in echo BASH Path before: $PATH with no parallel parallel echo ::: 1 @@ -21,6 +21,9 @@ par_path_remote_bash() { # --filter to see if $PATH with parallel is transferred env_parallel --filter --env A,PATH -Slo echo '$PATH' ::: OK _EOS + stdout ssh nopathbash@lo -T | + perl -ne '/logged in/..0 and print' | + uniq echo } @@ -29,7 +32,7 @@ par_path_remote_csh() { rm -rf /tmp/parallel cp /usr/local/bin/parallel /tmp - cat <<'_EOS' | stdout ssh nopathcsh@lo -T | perl -ne '/logged in/..0 and print' | uniq + cat <<'_EOS' | echo logged in echo CSH Path before: $PATH with no parallel which parallel >& /dev/stdout @@ -49,6 +52,9 @@ par_path_remote_csh() { sleep 1 echo Done _EOS + stdout ssh nopathcsh@lo -T | + perl -ne '/logged in/..0 and print' | + uniq } par_keep_order() { @@ -109,5 +115,5 @@ par_retries_4() { export -f $(compgen -A function | grep par_) #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' compgen -A function | grep par_ | LC_ALL=C sort | - parallel --joblog /tmp/jl-`basename $0` --delay 0.1 -j10 --tag -k '{} 2>&1' | - grep -Ev 'microk8s|smart connected IoT' + parallel --joblog /tmp/jl-`basename $0` --delay 0.1 -j10 --tag -k '{} 2>&1' + diff --git a/testsuite/tests-to-run/parallel-local164.sh b/testsuite/tests-to-run/parallel-local164.sh index 1962bc15..323420ed 100644 --- a/testsuite/tests-to-run/parallel-local164.sh +++ b/testsuite/tests-to-run/parallel-local164.sh @@ -119,7 +119,7 @@ echo '### Negative replacement strings' parallel --colsep ' ' echo '{-3}orrect' ::: "1 c 3 4" echo 'bug #38439: "open files" with --files --pipe blocks after a while' - ulimit -n 20; yes "`seq 3000`" |head -c 20M | parallel --pipe -k echo {#} of 21 + ulimit -n 28; yes "`seq 3000`" |head -c 20M | parallel --pipe -k echo {#} of 21 echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null; diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index f0791c41..0b1fd480 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -1,23 +1,5 @@ #!/bin/bash -get_tmux() { - # To install tmux in different version run this - cd /tmp - doit() { - wget https://github.com/tmux/tmux/archive/$1.tar.gz - tar xvf $1.tar.gz - cd tmux-$1 - ./autogen.sh - ./configure --prefix /tmp/tmux/$1 - make - make install - sudo cp /tmp/tmux/$1/bin/tmux /usr/local/bin/tmux-$1 - } - - . `which env_parallel.bash` - seq 1.8 0.1 2.9 | env_parallel --tag --lb doit -} - par_tmux_filter() { # /tmp/parallel-local7/tmsOU2Ig perl -pe 's:(/tmp\S+/tms).....:$1XXXXX:;s/ p\d+/pID/;' diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index 20fb43b4..a744c5ca 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -1,10 +1,20 @@ #!/bin/bash SERVER1=parallel-server1 -SERVER2=lo -SSHLOGIN1=parallel@parallel-server1 -SSHLOGIN2=parallel@lo -SSHLOGIN3=parallel@parallel-server2 +SERVER2=parallel-server2 +SERVER3=parallel-server3 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHUSER3=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 +SSHLOGIN3=$SSHUSER3@$SERVER3 + +#SERVER1=parallel-server1 +#SERVER2=lo +#SSHLOGIN1=parallel@parallel-server1 +#SSHLOGIN2=parallel@lo +#SSHLOGIN3=parallel@parallel-server2 echo '### Test use special ssh' echo 'TODO test ssh with > 9 simultaneous' @@ -21,8 +31,8 @@ echo '### test --filter-hosts with server w/o ssh, non-existing server' parallel -S 192.168.1.197,8.8.8.8,$SSHLOGIN1,$SSHLOGIN2,$SSHLOGIN3 --filter-hosts --nonall -k --tag echo echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' - (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & - (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & + (parallel -S $SSHLOGIN1 true ::: {1..20}; echo No --controlmaster - finish last) & + (parallel -M -S $SSHLOGIN1 true ::: {1..20}; echo With --controlmaster - finish first) & wait echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' diff --git a/testsuite/tests-to-run/test17.sh b/testsuite/tests-to-run/test17.sh index 5b0b6532..b6450c46 100755 --- a/testsuite/tests-to-run/test17.sh +++ b/testsuite/tests-to-run/test17.sh @@ -2,8 +2,15 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 -SSHLOGIN1=parallel@$SERVER1 -SSHLOGIN2=parallel@$SERVER2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 + +#SERVER1=parallel-server1 +#SERVER2=parallel-server2 +#SSHLOGIN1=parallel@$SERVER1 +#SSHLOGIN2=parallel@$SERVER2 export PARALLEL=-j0 #export PARALLEL="--sshdelay 0.3" diff --git a/testsuite/tests-to-run/test18.sh b/testsuite/tests-to-run/test18.sh index 699faad0..62c6a640 100644 --- a/testsuite/tests-to-run/test18.sh +++ b/testsuite/tests-to-run/test18.sh @@ -2,8 +2,12 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 -SSHLOGIN1=parallel@$SERVER1 -SSHLOGIN2=parallel@$SERVER2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 +#SSHLOGIN1=parallel@$SERVER1 +#SSHLOGIN2=parallel@$SERVER2 echo '### Check -S .. and --serverloginfile ..' echo $SSHLOGIN1 > ~/.parallel/sshloginfile @@ -19,7 +23,7 @@ cd /tmp (echo ':'; echo file:name; echo file:name.foo; echo file: name.foo; echo file : name.foo;) \ > /tmp/test18 cat /tmp/test18 | parallel echo content-{} ">" {} -cat /tmp/test18 | parallel -j1 --trc {}.{.} -S $SSHLOGIN1,parallel@$SERVER2,: \ +cat /tmp/test18 | parallel -j1 --trc {}.{.} -S $SSHLOGIN1,$SSHLOGIN2,: \ '(echo remote-{}.{.};cat {}) > {}.{.}' cat /tmp/test18 | parallel -j1 -k 'cat {}.{.}' @@ -34,10 +38,10 @@ echo | stdout parallel -k --cleanup echo echo '### Test --sshlogin -S --sshloginfile' echo localhost >/tmp/parallel-sshlogin -seq 1 3 | parallel -k --sshlogin 8/$SSHLOGIN1 -S "7/ssh -l parallel $SERVER2",: --sshloginfile /tmp/parallel-sshlogin echo +seq 1 3 | parallel -k --sshlogin 8/$SSHLOGIN1 -S "7/ssh -l $SSHUSER2 $SERVER2",: --sshloginfile /tmp/parallel-sshlogin echo echo '### Test --sshloginfile with extra content' -echo "2/ssh -l parallel $SERVER2" >>/tmp/parallel-sshlogin +echo "2/ssh -l $SSHUSER2 $SERVER2" >>/tmp/parallel-sshlogin echo ":" >>/tmp/parallel-sshlogin echo "#2/ssh -l tange nothing" >>/tmp/parallel-sshlogin seq 1 10 | parallel -k --sshloginfile /tmp/parallel-sshlogin echo diff --git a/testsuite/tests-to-run/test19.sh b/testsuite/tests-to-run/test19.sh index b7660a12..05bcf6e1 100644 --- a/testsuite/tests-to-run/test19.sh +++ b/testsuite/tests-to-run/test19.sh @@ -4,8 +4,12 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 -SSHLOGIN1=parallel@parallel-server1 -SSHLOGIN2=parallel@parallel-server2 +SSHUSER1=vagrant +SSHUSER2=vagrant +#SSHLOGIN1=parallel@$SERVER1 +#SSHLOGIN2=parallel@$SERVER2 +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 echo '### Test --transfer --return --cleanup - files with newline' diff --git a/testsuite/tests-to-run/test21.sh b/testsuite/tests-to-run/test21.sh index b063a34a..45e93c76 100644 --- a/testsuite/tests-to-run/test21.sh +++ b/testsuite/tests-to-run/test21.sh @@ -2,23 +2,33 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 +SERVER2=parallel-server3 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHUSER3=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 +SSHLOGIN3=$SSHUSER3@$SERVER3 + +#SERVER1=parallel-server1 +#SERVER2=parallel-server2 echo '### Test $PARALLEL - single line' echo | PARALLEL=--number-of-cpus parallel -(echo 1; echo 1) | PARALLEL="-Sparallel\@$SERVER1 -Sssh\ -l\ parallel\ $SERVER2 -j1" parallel -kv hostname\; echo | sort +(echo 1; echo 1) | PARALLEL="-S$SSHLOGIN1 -Sssh\ -l\ $SSHUSER2\ $SERVER2 -j1" parallel -kv hostname\; echo | sort echo '### Test $PARALLEL - multi line' -(echo 1; echo 1) | PARALLEL="-Sparallel\@$SERVER1 --Sssh\ -l\ parallel\ $SERVER2 +(echo 1; echo 1) | PARALLEL="-S$SSHLOGIN1 +-Sssh\ -l\ $SSHUSER2\ $SERVER2 -j1" parallel -kv hostname\; echo | sort echo '### Test ~/.parallel/config - single line' -echo "-Sparallel\@$SERVER1 -Sssh\ -l\ parallel\ $SERVER2 -j1" > ~/.parallel/config +echo "-S$SSHLOGIN1 -Sssh\ -l\ $SSHUSER2\ $SERVER2 -j1" > ~/.parallel/config (echo 1; echo 1) | parallel -kv hostname\; echo | sort echo '### Test ~/.parallel/config - multi line' -echo "-Sparallel\@$SERVER1 --Sssh\ -l\ parallel\ $SERVER2 +echo "-S$SSHLOGIN1 +-Sssh\ -l\ $SSHUSER2\ $SERVER2 -j1" > ~/.parallel/config (echo 1; echo 1) | parallel -kv hostname\; echo | sort rm ~/.parallel/config diff --git a/testsuite/tests-to-run/test23.sh b/testsuite/tests-to-run/test23.sh index 5881308e..e11ad3f6 100644 --- a/testsuite/tests-to-run/test23.sh +++ b/testsuite/tests-to-run/test23.sh @@ -2,8 +2,12 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 -SSHLOGIN1=parallel@$SERVER1 -SSHLOGIN2=parallel@$SERVER2 +SSHUSER1=vagrant +SSHUSER2=vagrant +#SSHLOGIN1=parallel@$SERVER1 +#SSHLOGIN2=parallel@$SERVER2 +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 cd /tmp diff --git a/testsuite/tests-to-run/test30.sh b/testsuite/tests-to-run/test30.sh index 679226cc..cd3d0f6a 100644 --- a/testsuite/tests-to-run/test30.sh +++ b/testsuite/tests-to-run/test30.sh @@ -1,5 +1,12 @@ #!/bin/bash +SERVER1=parallel-server1 +SERVER2=parallel-server2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 + cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test of --eta' seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l @@ -14,5 +21,5 @@ echo '### Test of --progress with no jobs' stdout parallel --progress "sleep 1; echo {}" < /dev/null echo '### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed' - parallel -j0 --timeout 6 --onall -S localhost,parallel@parallel-server1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $? + parallel -j0 --timeout 6 --onall -S localhost,$SSHLOGIN1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $? EOF diff --git a/testsuite/tests-to-run/test35.sh b/testsuite/tests-to-run/test35.sh index b3c34ebb..54e2aa9f 100755 --- a/testsuite/tests-to-run/test35.sh +++ b/testsuite/tests-to-run/test35.sh @@ -1,19 +1,23 @@ #!/bin/bash +SERVER1=parallel-server1 +SSHUSER1=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 + rsync -Ha --delete input-files/testdir/ tmp/ cd tmp SERVER2=parallel@parallel-server2 -echo $SERVER2 >~/.parallel/sshloginfile +echo $SSHLOGIN1 >~/.parallel/sshloginfile echo '### Test --wd newtempdir/newdir/tmp/ with space dirs'; - ssh $SERVER2 rm -rf newtempdir; + ssh $SSHLOGIN1 rm -rf newtempdir; stdout parallel -j9 -k --wd newtempdir/newdir/tmp/ --basefile 1-col.txt --trc {}.6 -S .. -v echo ">"{}.6 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'; find . -name '*.6' | LC_ALL=C sort echo '### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs'; - ssh $SERVER2 rm -rf /tmp/newtempdir; + ssh $SSHLOGIN1 rm -rf /tmp/newtempdir; stdout parallel -j9 -k --wd /tmp/newtempdir/newdir/tmp/ --basefile 1-col.txt --trc {}.7 -S .. -v echo ">"{}.7 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'; find . -name '*.7' | LC_ALL=C sort diff --git a/testsuite/tests-to-run/test36.sh b/testsuite/tests-to-run/test36.sh index 5b9b863a..17944bb2 100644 --- a/testsuite/tests-to-run/test36.sh +++ b/testsuite/tests-to-run/test36.sh @@ -2,15 +2,25 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 +SERVER2=parallel-server3 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHUSER3=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 +SSHLOGIN3=$SSHUSER3@$SERVER3 + +#SERVER1=parallel-server1 +#SERVER2=parallel-server2 echo '### Test $PARALLEL_SEQ - local' seq 1 20 | parallel -kN2 echo arg1:{1} seq:'$'PARALLEL_SEQ arg2:{2} echo '### Test $PARALLEL_SEQ - remote' -seq 1 20 | parallel -kN2 -S parallel@$SERVER1,parallel@$SERVER2 echo arg1:{1} seq:'$'PARALLEL_SEQ arg2:{2} +seq 1 20 | parallel -kN2 -S $SSHLOGIN1,$SSHLOGIN2 echo arg1:{1} seq:'$'PARALLEL_SEQ arg2:{2} echo '### Test $PARALLEL_PID - local' seq 1 20 | parallel -kN2 echo arg1:{1} pid:'$'PARALLEL_PID arg2:{2} | perl -pe 's/\d{3,}/0/g' echo '### Test $PARALLEL_PID - remote' -seq 1 20 | parallel -kN2 -S parallel@$SERVER1,parallel@$SERVER2 echo arg1:{1} pid:'$'PARALLEL_PID arg2:{2} | perl -pe 's/\d{3,}/0/g' +seq 1 20 | parallel -kN2 -S $SSHLOGIN1,$SSHLOGIN2 echo arg1:{1} pid:'$'PARALLEL_PID arg2:{2} | perl -pe 's/\d{3,}/0/g' diff --git a/testsuite/tests-to-run/test37.sh b/testsuite/tests-to-run/test37.sh index f2dbc747..5fb0a924 100644 --- a/testsuite/tests-to-run/test37.sh +++ b/testsuite/tests-to-run/test37.sh @@ -1,7 +1,11 @@ #!/bin/bash -SERVER1=redhat9.tange.dk -SERVER2=centos3.tange.dk +SERVER1=parallel-server1 +SERVER2=parallel-server2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 echo '### Test $PARALLEL' PARALLEL="-k @@ -27,7 +31,7 @@ PARALLEL="-k --jobs 1 echo 1" parallel -v echo 2 ::: a b c echo '### Test ugly quoting from $PARALLEL' PARALLEL="-k --jobs 1 perl -pe \'\$a=1; print\$a\'" parallel -v ::: <(echo a) <(echo b) -PARALLEL='-k --jobs 1 -S parallel@'$SERVER1' perl -pe \"\\$a=1; print\\$a\"' parallel -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 -S '$SSHLOGIN1' perl -pe \"\\$a=1; print\\$a\"' parallel -v '<(echo {})' ::: foo echo '### Test ugly quoting from profile file' cat <~/.parallel/test_profile @@ -39,27 +43,27 @@ parallel -v -J test_profile ::: <(echo a) <(echo b) echo '### Test ugly quoting from profile file --plain' parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b) -PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1 -v ::: foo -PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 echo' parallel -S ssh\ $SSHLOGIN1 -v ::: foo +PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ $SSHLOGIN1 -v '<(echo {})' ::: foo echo '### Test quoting of $ in command from profile file' cat <~/.parallel/test_profile -k --jobs 1 perl -pe \'\\\$a=1; print \\\$a\' EOF -parallel -v -J test_profile -S ssh\ parallel\@$SERVER1 '<(echo {})' ::: foo +parallel -v -J test_profile -S ssh\ $SSHLOGIN1 '<(echo {})' ::: foo echo '### Test quoting of $ in command from profile file --plain' -parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1 'cat <(echo {})' ::: foo +parallel -v -J test_profile --plain -S ssh\ $SSHLOGIN1 'cat <(echo {})' ::: foo echo '### Test quoting of $ in command from $PARALLEL' -PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1 -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ $SSHLOGIN1 -v '<(echo {})' ::: foo echo '### Test quoting of $ in command from $PARALLEL --plain' -PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1 -v 'cat <(echo {})' ::: foo +PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ $SSHLOGIN1 -v 'cat <(echo {})' ::: foo echo '### Test quoting of space in arguments (-S) from profile file' cat <~/.parallel/test_profile --k --jobs 1 -S ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\' +-k --jobs 1 -S ssh\ $SSHLOGIN1 perl -pe \'\$a=1; print \$a\' EOF parallel -v -J test_profile '<(echo {})' ::: foo @@ -67,23 +71,23 @@ echo '### Test quoting of space in arguments (-S) from profile file --plain' parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo echo '### Test quoting of space in arguments (-S) from $PARALLEL' -PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 -S ssh\ '$SSHLOGIN1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo echo '### Test quoting of space in long arguments (--sshlogin) from profile file' cat <~/.parallel/test_profile # testprofile --k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1 perl -pe \'\$a=1; print \$a\' +-k --jobs 1 --sshlogin ssh\ $SSHLOGIN1 perl -pe \'\$a=1; print \$a\' EOF parallel -v -J test_profile '<(echo {})' ::: foo echo '### Test quoting of space in arguments (-S) from $PARALLEL' -PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo +PARALLEL='-k --jobs 1 --sshlogin ssh\ '$SSHLOGIN1' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo echo '### Test merging of profiles - sort needed because -k only works on the single machine' echo --tag > ~/.parallel/test_tag echo -S .. > ~/.parallel/test_S.. -echo parallel@parallel-server1 > ~/.parallel/sshloginfile -echo parallel@parallel-server2 >> ~/.parallel/sshloginfile +echo $SSHLOGIN1 > ~/.parallel/sshloginfile +echo $SSHLOGIN2 >> ~/.parallel/sshloginfile parallel -Jtest_tag -Jtest_S.. --nonall echo a | sort echo '### Test merging of profiles - sort needed because -k only works on the single machine --plain' diff --git a/testsuite/tests-to-run/test41.sh b/testsuite/tests-to-run/test41.sh index f7264079..24efa27b 100644 --- a/testsuite/tests-to-run/test41.sh +++ b/testsuite/tests-to-run/test41.sh @@ -2,8 +2,12 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 echo '### Test -M (--retries to avoid false errors)' -seq 1 30 | parallel -j5 --retries 3 -k -M -S parallel@$SERVER1,parallel@$SERVER2 echo 2>/dev/null -seq 1 30 | parallel -j10 --retries 3 -k -M -S parallel@$SERVER1,parallel@$SERVER2 echo 2>/dev/null +seq 1 30 | parallel -j5 --retries 3 -k -M -S $SSHLOGIN1,$SSHLOGIN2 echo 2>/dev/null +seq 1 30 | parallel -j10 --retries 3 -k -M -S $SSHLOGIN1,$SSHLOGIN2 echo 2>/dev/null diff --git a/testsuite/tests-to-run/test47.sh b/testsuite/tests-to-run/test47.sh index bf433448..b61270d0 100644 --- a/testsuite/tests-to-run/test47.sh +++ b/testsuite/tests-to-run/test47.sh @@ -1,10 +1,20 @@ #!/bin/bash -# must be other servers than localhost SERVER1=parallel-server1 SERVER2=parallel-server2 -SSHLOGIN1=parallel@$SERVER1 -SSHLOGIN2=parallel@$SERVER2 +SERVER2=parallel-server3 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHUSER3=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 +SSHLOGIN3=$SSHUSER3@$SERVER3 + +## must be other servers than localhost +#SERVER1=parallel-server1 +#SERVER2=parallel-server2 +#SSHLOGIN1=parallel@$SERVER1 +#SSHLOGIN2=parallel@$SERVER2 export SSHLOGIN1 export SSHLOGIN2 # Minimal version of test17 diff --git a/testsuite/tests-to-run/test60.sh b/testsuite/tests-to-run/test60.sh index dc25b90b..53d05573 100644 --- a/testsuite/tests-to-run/test60.sh +++ b/testsuite/tests-to-run/test60.sh @@ -2,8 +2,15 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 -SSHLOGIN1=parallel@$SERVER1 -SSHLOGIN2=parallel@$SERVER2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 + +#SERVER1=parallel-server1 +#SERVER2=parallel-server2 +#SSHLOGIN1=parallel@$SERVER1 +#SSHLOGIN2=parallel@$SERVER2 # -L1 will join lines ending in ' ' cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r diff --git a/testsuite/tests-to-run/test61.sh b/testsuite/tests-to-run/test61.sh index f9d19315..5a679a4a 100644 --- a/testsuite/tests-to-run/test61.sh +++ b/testsuite/tests-to-run/test61.sh @@ -2,10 +2,14 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 +SSHUSER1=vagrant +SSHUSER2=vagrant +SSHLOGIN1=$SSHUSER1@$SERVER1 +SSHLOGIN2=$SSHUSER2@$SERVER2 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r | perl -pe 's/(PARALLEL_PID....)\d+/$1XXXXX/g' echo '### Test --return of weirdly named file' -stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b' | +stdout parallel --return {} -vv -S $SSHLOGIN1 echo '>'{} ::: 'aa<${#}" b' | perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'; rm 'aa<${#}" b' echo '### Test if remote login shell is csh' diff --git a/testsuite/vagrant/Vagrantfile.tmpl b/testsuite/vagrant/Vagrantfile.tmpl new file mode 100644 index 00000000..a0cb674a --- /dev/null +++ b/testsuite/vagrant/Vagrantfile.tmpl @@ -0,0 +1,75 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "%%VMBOX%%" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "%%IP%%" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + vb.memory = "512" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell" do |s| + ssh_pub_key = File.readlines("../../authorized_keys").first.strip + s.inline = <<-SHELL + mkdir /root/.ssh + echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys + echo #{ssh_pub_key} >> /root/.ssh/authorized_keys + sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync + SHELL + end +end diff --git a/testsuite/vagrant/setup.sh b/testsuite/vagrant/setup.sh new file mode 100644 index 00000000..f098139e --- /dev/null +++ b/testsuite/vagrant/setup.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +setup_one() { + # setup_one ubuntu/trusty64 172.27.27.2 + if [ -z "$2" ] ; then + echo Usage: setup_one ubuntu/trusty64 172.27.27.2 + return 1 + fi + vfile="$1"/Vagrantfile + mkdir -p "$1" + cp Vagrantfile.tmpl "$vfile" + ( + cd "$1" + perl -i -pe "s{%%VMBOX%%}{$1}g;s/%%IP%%/$2/" Vagrantfile + vagrant up + vagrant ssh -c 'ip addr || ifconfig' + ) + ssh-keygen -R $2 + ssh vagrant@$2 hostname +} +export -f setup_one + +destroy_one() { + ( + cd "$1" + vagrant destroy -f + ) + rm -r ./"$1" +} +export -f destroy_one + + +server_list() { + cat <= 131049) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +par_xargs_compat parallel: Error: Command line too long (1000005 >= 131XXX) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... par_xargs_compat xargs: argument line too long -par_xargs_compat environment: line 109: stdsort: command not found -par_xargs_compat environment: line 112: stdsort: command not found -par_xargs_compat environment: line 115: stdsort: command not found +par_xargs_compat parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +par_xargs_compat 1 2 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 +par_xargs_compat xargs: argument line too long +par_xargs_compat 1 2 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 +par_xargs_compat parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +par_xargs_compat 1 2 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 par_xargs_compat ### Test -x -par_xargs_compat environment: line 119: stdsort: command not found -par_xargs_compat environment: line 122: stdsort: command not found -par_xargs_compat environment: line 125: stdsort: command not found -par_xargs_compat environment: line 128: stdsort: command not found -par_xargs_compat environment: line 131: stdsort: command not found -par_xargs_compat environment: line 134: stdsort: command not found +par_xargs_compat -km +par_xargs_compat parallel: Error: Command line too long (10 >= 10) at input 0: 12345 +par_xargs_compat 1 2 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 +par_xargs_compat -kX +par_xargs_compat parallel: Error: Command line too long (10 >= 10) at input 0: 12345 +par_xargs_compat 1 2 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 +par_xargs_compat -x +par_xargs_compat xargs: argument line too long +par_xargs_compat 1 2 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat -km -x +par_xargs_compat 1 2 +par_xargs_compat 12 +par_xargs_compat 1234 +par_xargs_compat 13 +par_xargs_compat 14 +par_xargs_compat 15 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 +par_xargs_compat -kX -x +par_xargs_compat 1 2 +par_xargs_compat 12 +par_xargs_compat 1234 +par_xargs_compat 13 +par_xargs_compat 14 +par_xargs_compat 15 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 +par_xargs_compat -x +par_xargs_compat 1 2 +par_xargs_compat 12 +par_xargs_compat 1234 +par_xargs_compat 13 +par_xargs_compat 14 +par_xargs_compat 15 +par_xargs_compat 3 4 +par_xargs_compat 5 6 +par_xargs_compat 7 8 +par_xargs_compat 9 10 diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index 9b468e43..5c08f53f 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -740,448 +740,424 @@ par_groupby --block 20 C1 $_%=2 csv , parallel: Warning: A record was longer tha par_groupby --block 20 C1 $_%=2 csv , parallel: Warning: A record was longer than 50. Increasing to --blocksize 66. par_groupby_pipepart tsv \t 3 $_%=2 ### test tsv | --colsep \t --groupby 3 $_%=2 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 1 6 19 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 -par_groupby_pipepart tsv \t 3 $_%=2 NewRec -par_groupby_pipepart tsv \t 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t 3 ### test tsv | --colsep \t --groupby 3 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 1 6 19 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 -par_groupby_pipepart tsv \t 3 NewRec -par_groupby_pipepart tsv \t 3 90000 540000 1170000 +par_groupby_pipepart tsv \t 3 90001 540006 1170031 par_groupby_pipepart tsv \t c1 ### test tsv | --colsep \t --groupby c1 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90001 540006 1170019 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 NewRec -par_groupby_pipepart tsv \t c1 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 ### test tsv | --colsep \t --groupby c1 $_%=2 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170019 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t c1 $_%=2 NewRec -par_groupby_pipepart tsv \t c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart tsv \t c1 $_%=2 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ ### test tsv | --colsep \t --groupby s/^(\d+[\t ,]+){2}(\d+).*/$2/ par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 1 6 19 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart tsv \t s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 ### test ssv | --colsep \s+ --groupby 3 $_%=2 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 1 6 19 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 -par_groupby_pipepart ssv \s+ 3 $_%=2 NewRec -par_groupby_pipepart ssv \s+ 3 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 ### test ssv | --colsep \s+ --groupby 3 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 1 6 19 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 -par_groupby_pipepart ssv \s+ 3 NewRec -par_groupby_pipepart ssv \s+ 3 90000 540000 1170000 +par_groupby_pipepart ssv \s+ 3 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 ### test ssv | --colsep \s+ --groupby c1 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90001 540006 1170019 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 NewRec -par_groupby_pipepart ssv \s+ c1 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 ### test ssv | --colsep \s+ --groupby c1 $_%=2 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170019 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ c1 $_%=2 NewRec -par_groupby_pipepart ssv \s+ c1 $_%=2 90000 540000 1170000 +par_groupby_pipepart ssv \s+ c1 $_%=2 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ ### test ssv | --colsep \s+ --groupby s/^(\d+[\t ,]+){2}(\d+).*/$2/ par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 1 6 19 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1170000 +par_groupby_pipepart ssv \s+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1170031 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 ### test cssv | --colsep [\s,]+ --groupby 3 $_%=2 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 1 6 25 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 ### test cssv | --colsep [\s,]+ --groupby 3 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 1 6 25 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 -par_groupby_pipepart cssv [\s,]+ 3 NewRec -par_groupby_pipepart cssv [\s,]+ 3 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ 3 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 ### test cssv | --colsep [\s,]+ --groupby c1 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710025 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 NewRec -par_groupby_pipepart cssv [\s,]+ c1 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 ### test cssv | --colsep [\s,]+ --groupby c1 $_%=2 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710025 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ c1 $_%=2 NewRec -par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ c1 $_%=2 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ ### test cssv | --colsep [\s,]+ --groupby s/^(\d+[\t ,]+){2}(\d+).*/$2/ par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 1 6 25 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 540000 1710000 +par_groupby_pipepart cssv [\s,]+ s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 540006 1710037 par_groupby_pipepart csv , 3 $_%=2 ### test csv | --colsep , --groupby 3 $_%=2 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 1 1 19 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 -par_groupby_pipepart csv , 3 $_%=2 NewRec -par_groupby_pipepart csv , 3 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , 3 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , 3 ### test csv | --colsep , --groupby 3 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 1 1 19 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 -par_groupby_pipepart csv , 3 NewRec -par_groupby_pipepart csv , 3 90000 90000 1170000 +par_groupby_pipepart csv , 3 90001 90001 1170031 par_groupby_pipepart csv , c1 ### test csv | --colsep , --groupby c1 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90001 90001 1170019 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 NewRec -par_groupby_pipepart csv , c1 90000 90000 1170000 +par_groupby_pipepart csv , c1 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 ### test csv | --colsep , --groupby c1 $_%=2 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170019 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , c1 $_%=2 NewRec -par_groupby_pipepart csv , c1 $_%=2 90000 90000 1170000 +par_groupby_pipepart csv , c1 $_%=2 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ ### test csv | --colsep , --groupby s/^(\d+[\t ,]+){2}(\d+).*/$2/ par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 1 1 19 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ NewRec -par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90000 90000 1170000 +par_groupby_pipepart csv , s/^(\d+[\t ,]+){2}(\d+).*/$2/ 90001 90001 1170031 par_keeporder_roundrobin bug #50081: --keep-order --round-robin should give predictable results par_keeporder_roundrobin OK par_macron ### See if \257\256 \257<\257> is replaced correctly @@ -1295,7 +1271,7 @@ par_memfree Free mem: 1k par_memfree parallel: Warning: This job was killed because it timed out: par_memfree parallel: Warning: parallel --memfree 1t echo Free mem: ::: 1t par_memory_leak ### Test for memory leaks -par_memory_leak Of 100 runs of 1 job none should be bigger than a 3000 job run +par_memory_leak Of 100 runs of 1 job at least one should be bigger than a 3000 job run par_memory_leak Good: No memleak detected. par_no_newline_compress bug #41613: --compress --line-buffer - no newline par_no_newline_compress tagstring=--tagstring {#} compress=--compress diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index eb76172a..79301370 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -1,22 +1,3 @@ -par_bin ### Test --bin -par_bin 2 2 4 -par_bin 2 2 4 -par_bin 3 3 6 -par_bin 3 3 7 -par_bin 2 4 8 -par_bin 2 4 8 -par_bin 3 6 13 -par_bin 3 6 13 -par_bin ### Test --bin with expression that gives 1..n -par_bin 0 0 0 -par_bin 0 0 0 -par_bin 5 10 21 -par_bin 5 10 21 -par_bin ### Test --bin with expression that gives 0..n-1 -par_bin 0 0 0 -par_bin 0 0 0 -par_bin 5 10 21 -par_bin 5 10 21 par_children_receive_sig ### Do children receive --termseq signals par_children_receive_sig parallel: Warning: This job was killed because it timed out: par_children_receive_sig parallel: Warning: show_signals '' diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index bc7e8e12..db6c97d4 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -2110,37 +2110,10 @@ par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And then use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigvar_quote_remote -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_remote -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line XXX: /bin/perl: Argument list too long -par_bash_environment_too_big env_parallel: Error: Your environment is too big. -par_bash_environment_too_big env_parallel: Error: You can try 3 different approaches: -par_bash_environment_too_big env_parallel: Error: 1. Run 'env_parallel --session' before you set -par_bash_environment_too_big env_parallel: Error: variables or define functions. -par_bash_environment_too_big env_parallel: Error: 2. Use --env and only mention the names to copy. -par_bash_environment_too_big env_parallel: Error: 3. Try running this in a clean environment once: -par_bash_environment_too_big env_parallel: Error: env_parallel --record-env -par_bash_environment_too_big env_parallel: Error: And then use '--env _' -par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_quote +par_bash_environment_too_big parallel: Error: Command line too long (XXX >= XXX) at input 0: fail_bigfunc_quote_remote par_bash_env_parallel_session ### Test env_parallel --session / --end-session par_bash_env_parallel_session ### level0 should be hidden, level1 should be transferred par_bash_env_parallel_session @@ -2254,6 +2227,14 @@ par_bash_env_parallel OK par_bash_env_parallel OK par_bash_env_parallel alias line 1 par_bash_env_parallel alias line 1 +par_bash_env_parallel alias line 2 +par_bash_env_parallel alias line 2 +par_bash_env_parallel alias line 3 +par_bash_env_parallel alias line 3 +par_bash_env_parallel alias2 line 1 +par_bash_env_parallel alias2 line 1 +par_bash_env_parallel alias2 line 2 +par_bash_env_parallel alias2 line 2 par_bash_env_parallel bug #50435: Remote fifo broke in 20150522 par_bash_env_parallel bug #52534: Tail of multiline alias is ignored par_bash_env_parallel data from stdin diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index 893f9fff..c5153220 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -278,7 +278,7 @@ correct correct echo 'bug #38439: "open files" with --files --pipe blocks after a while' bug #38439: "open files" with --files --pipe blocks after a while - ulimit -n 20; yes "`seq 3000`" |head -c 20M | parallel --pipe -k echo {#} of 21 + ulimit -n 28; yes "`seq 3000`" |head -c 20M | parallel --pipe -k echo {#} of 21 1 of 21 2 of 21 3 of 21 @@ -300,8 +300,8 @@ bug #38439: "open files" with --files --pipe blocks after a while 19 of 21 20 of 21 21 of 21 -parallel: Warning: Only enough file handles to run 1 jobs in parallel. -parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or +parallel: Warning: Only enough file handles to run 2 jobs in parallel. +parallel: Warning: Running 'parallel -j0 -N 2 --pipe parallel -j0' or parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf parallel: Warning: or /proc/sys/fs/file-max may help. echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index 8da77eaa..410e0047 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -107,7 +107,7 @@ a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6 a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 echo '### Test -m with 10000 args'; seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -km echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1 ### Test -m with 10000 args -dc1624c3316077d0d95803a2eb30f455 - +c606aec1723ee5cc15f2a1b95d83d3cf - 2 29996 186684 echo '### Test -X with 10000 args'; seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1 ### Test -X with 10000 args diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 4f65788a..f4e25fb4 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -5,12 +5,12 @@ Chars per line: 116300 6 119994 697800 echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$ ### Test of xargs -X command lines > 130k -3cfc69ee81b0fe7fdbe8eb059ad2da61 - +201ecb8ca789cecf39ee914a6d75611b - Chars per line: 116826 7 60000 817788 echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum ### Test of xargs -m command lines > 130k -00b8aed2c1bc28368381273aa2ae104e - +590091fd85dcb98f91c5e89cdddf21ef - echo '### This causes problems if we kill child processes'; # 2>/dev/null to avoid parallel: Warning: Starting 45 processes took > 2 sec. ### This causes problems if we kill child processes seq 2 40 | parallel -j 0 seq 1 10 2>/dev/null | sort | md5sum diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 28bf274c..d69d5306 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -73,8 +73,8 @@ please cite as described in 'parallel --citation'. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' -131040 -37854 +131052 +37842 echo '### --delay should grow by 3 sec per arg' ### --delay should grow by 3 sec per arg stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' diff --git a/testsuite/wanted-results/parallel-remote1 b/testsuite/wanted-results/parallel-remote1 index 4589f0e8..b3cc2072 100644 --- a/testsuite/wanted-results/parallel-remote1 +++ b/testsuite/wanted-results/parallel-remote1 @@ -102,19 +102,16 @@ TODO test ssh with > 9 simultaneous 100 echo '### test --timeout --retries' ### test --timeout --retries - parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 parallel@parallel-server1 parallel@lo parallel@parallel-server2 -parallel@parallel-server1 -parallel@lo -parallel@parallel-server2 + parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 vagrant@parallel-server1 vagrant@parallel-server2 vagrant@parallel-server3 +vagrant@parallel-server1 +vagrant@parallel-server2 +vagrant@parallel-server3 echo '### test --filter-hosts with server w/o ssh, non-existing server' ### test --filter-hosts with server w/o ssh, non-existing server - parallel -S 192.168.1.197,8.8.8.8,parallel@parallel-server1,parallel@lo,parallel@parallel-server2 --filter-hosts --nonall -k --tag echo -parallel@lo -parallel@parallel-server1 -parallel@parallel-server2 + parallel -S 192.168.1.197,8.8.8.8,vagrant@parallel-server1,vagrant@parallel-server2,vagrant@parallel-server3 --filter-hosts --nonall -k --tag echo echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' ### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host - (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & wait + (parallel -S vagrant@parallel-server1 true ::: {1..20}; echo No --controlmaster - finish last) & (parallel -M -S vagrant@parallel-server1 true ::: {1..20}; echo With --controlmaster - finish first) & wait With --controlmaster - finish first No --controlmaster - finish last echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' @@ -123,7 +120,7 @@ echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' aspire echo '### test --workdir . in $HOME' ### test --workdir . in $HOME - cd && mkdir -p parallel-test && cd parallel-test && echo OK > testfile && parallel --workdir . --transfer -S parallel@parallel-server1 cat {} ::: testfile + cd && mkdir -p parallel-test && cd parallel-test && echo OK > testfile && parallel --workdir . --transfer -S vagrant@parallel-server1 cat {} ::: testfile OK echo '### TODO: test --filter-hosts proxied through the one host' ### TODO: test --filter-hosts proxied through the one host diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index e1b9ca45..88103d68 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -565,7 +565,8 @@ Computers / CPU cores / Max jobs to run 1:local / 8 / 8 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \ - 2> >(zenity --timeout=15 --progress --auto-kill --auto-close) + 2> >(perl -pe 'BEGIN{$/="\r";$|=1};s/\r/\n/g' | + zenity --timeout=15 --progress --auto-kill --auto-close) BASE64--joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log; parallel --joblog /tmp/log exit ::: 1 2 3 0 @@ -1424,4 +1425,4 @@ C echo A echo B echo C -7 +8 diff --git a/testsuite/wanted-results/sql02 b/testsuite/wanted-results/sql02 index 46842665..ca08ade2 100644 --- a/testsuite/wanted-results/sql02 +++ b/testsuite/wanted-results/sql02 @@ -37,7 +37,7 @@ bytes 3072 Current command: sqlite3 foo -sqltest.sqlite3: SQLite 3.x database, last written using SQLite version 3022000 +sqltest.sqlite3: SQLite 3.x database, last written using SQLite version 3031001 n|t 1|Line 1 2|Line 2 diff --git a/testsuite/wanted-results/test17 b/testsuite/wanted-results/test17 index 8b5e261f..cb2a4d34 100644 --- a/testsuite/wanted-results/test17 +++ b/testsuite/wanted-results/test17 @@ -4,25 +4,25 @@ file>fire file : & ) \n*.jpg good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory ls: /tmp/parallel.file*: No such file or directory ### --transfer - relpath file>fire file : & ) \n*.jpg good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory ls: tmp/parallel.file*: No such file or directory ### --transfer --cleanup - abspath file>fire file : & ) \n*.jpg good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory ls: /tmp/parallel.file*: No such file or directory ### --transfer --cleanup - relpath file>fire file : & ) \n*.jpg good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -61,7 +61,7 @@ good if no file /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK @@ -71,7 +71,7 @@ OK /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -80,7 +80,7 @@ OK /tmp/parallel.file : & ) \n*.jpg.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -90,7 +90,7 @@ OK /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK @@ -100,7 +100,7 @@ tmp/parallel.file : & ) \n*.jpg.out tmp/parallel.file>fire.file.done tmp/parallel.file>fire.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -110,7 +110,7 @@ OK /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK diff --git a/testsuite/wanted-results/test18 b/testsuite/wanted-results/test18 index 934f2bf8..3c8168c4 100644 --- a/testsuite/wanted-results/test18 +++ b/testsuite/wanted-results/test18 @@ -91,13 +91,13 @@ aspire aspire aspire aspire -redhat9.tange.dk -redhat9.tange.dk -redhat9.tange.dk -redhat9.tange.dk -redhat9.tange.dk -redhat9.tange.dk -redhat9.tange.dk +centos8.localdomain +centos8.localdomain +centos8.localdomain +centos8.localdomain +centos8.localdomain +centos8.localdomain +centos8.localdomain ### Check more than 9 simultaneous sshlogins 1 2 diff --git a/testsuite/wanted-results/test19 b/testsuite/wanted-results/test19 index e14b141c..6e108572 100644 --- a/testsuite/wanted-results/test19 +++ b/testsuite/wanted-results/test19 @@ -3,13 +3,13 @@ newline newline good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory ls: tmp/parallel.file*: No such file or directory ### --transfer --cleanup - file with newline newline newline good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory ls: tmp/parallel.file*: No such file or directory ### --return - file with newline tmp/parallel.file. @@ -22,7 +22,7 @@ newline1.out tmp/parallel.file. newline2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -32,7 +32,7 @@ newline1.out tmp/parallel.file. newline2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -42,7 +42,7 @@ newline1.out tmp/parallel.file. newline2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -56,7 +56,7 @@ newline2.out tmp/parallel.file. newline2.out2 good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -70,34 +70,34 @@ newline2.out tmp/parallel.file. newline2.out2 good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK Input for ssh --l parallel one-server -- mkdir -p ./. --l parallel one-server rsync --server -lDrRze.iLsfxC . ./. --l parallel one-server -- exec perl -e '@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;' base64 --l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. +-l vagrant one-server -- mkdir -p ./. +-l vagrant one-server rsync --server -lDrRze.iLsfxC . ./. +-l vagrant one-server -- exec perl -e '@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;' base64 +-l vagrant one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. newlineX.out' --l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. +-l vagrant one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. newlineX.out2' --l parallel one-server -- rm -f './tmp/parallel.file. +-l vagrant one-server -- rm -f './tmp/parallel.file. newlineX'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l parallel one-server -- rm -f './tmp/parallel.file. +-l vagrant one-server -- rm -f './tmp/parallel.file. newlineX.out'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l parallel one-server -- rm -f './tmp/parallel.file. +-l vagrant one-server -- rm -f './tmp/parallel.file. newlineX.out2'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l parallel one-server -- mkdir -p ./. --l parallel one-server rsync --server -lDrRze.iLsfxC . ./. --l parallel one-server -- exec perl -e '@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;' base64 --l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. +-l vagrant one-server -- mkdir -p ./. +-l vagrant one-server rsync --server -lDrRze.iLsfxC . ./. +-l vagrant one-server -- exec perl -e '@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;' base64 +-l vagrant one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. newlineX.out' --l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. +-l vagrant one-server cd ././tmp; rsync --server --sender -lDrRze.iLsfxC . './parallel.file. newlineX.out2' --l parallel one-server -- rm -f './tmp/parallel.file. +-l vagrant one-server -- rm -f './tmp/parallel.file. newlineX'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l parallel one-server -- rm -f './tmp/parallel.file. +-l vagrant one-server -- rm -f './tmp/parallel.file. newlineX.out'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' --l parallel one-server -- rm -f './tmp/parallel.file. +-l vagrant one-server -- rm -f './tmp/parallel.file. newlineX.out2'; sh -c 'rmdir ./tmp/ ./ 2>/dev/null;' diff --git a/testsuite/wanted-results/test21 b/testsuite/wanted-results/test21 index f860a04f..3f8aaec4 100644 --- a/testsuite/wanted-results/test21 +++ b/testsuite/wanted-results/test21 @@ -2,28 +2,28 @@ 4 1 1 -centos3.tange.dk +centos8.localdomain +freebsd12.localdomain hostname; echo 1 hostname; echo 1 -redhat9.tange.dk ### Test $PARALLEL - multi line 1 1 -centos3.tange.dk +centos8.localdomain +freebsd12.localdomain hostname; echo 1 hostname; echo 1 -redhat9.tange.dk ### Test ~/.parallel/config - single line 1 1 -centos3.tange.dk +centos8.localdomain +freebsd12.localdomain hostname; echo 1 hostname; echo 1 -redhat9.tange.dk ### Test ~/.parallel/config - multi line 1 1 -centos3.tange.dk +centos8.localdomain +freebsd12.localdomain hostname; echo 1 hostname; echo 1 -redhat9.tange.dk diff --git a/testsuite/wanted-results/test23 b/testsuite/wanted-results/test23 index acd534e7..1eabe31d 100644 --- a/testsuite/wanted-results/test23 +++ b/testsuite/wanted-results/test23 @@ -13,9 +13,9 @@ script2 run 4 script1 run 5 script2 run 5 good if no file -ls: script1: No such file or directory +ls: cannot access 'script1': No such file or directory OK -ls: script2: No such file or directory +ls: cannot access 'script2': No such file or directory OK ### Test --basefile + --sshlogin : 1 diff --git a/testsuite/wanted-results/test37 b/testsuite/wanted-results/test37 index 402aab69..040ffc80 100644 --- a/testsuite/wanted-results/test37 +++ b/testsuite/wanted-results/test37 @@ -72,7 +72,7 @@ perl -pe '$a=1; print $a' <(echo foo) perl -pe "\$a=1; print \$a" <(echo foo) 1foo ### Test merging of profiles - sort needed because -k only works on the single machine -parallel@parallel-server1 a -parallel@parallel-server2 a +vagrant@parallel-server1 a +vagrant@parallel-server2 a ### Test merging of profiles - sort needed because -k only works on the single machine --plain a diff --git a/testsuite/wanted-results/test47 b/testsuite/wanted-results/test47 index d5636ecd..e0e8f9b2 100644 --- a/testsuite/wanted-results/test47 +++ b/testsuite/wanted-results/test47 @@ -4,25 +4,25 @@ file1 file2 good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory ls: /tmp/parallel.file*: No such file or directory ### --transfer - relpath file1 file2 good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory ls: tmp/parallel.file*: No such file or directory ### --transfer --cleanup - abspath file1 file2 good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory ls: /tmp/parallel.file*: No such file or directory ### --transfer --cleanup - relpath file1 file2 good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -43,7 +43,7 @@ tmp/parallel.file2.out /tmp/parallel.file2.file.done /tmp/parallel.file2.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK @@ -53,7 +53,7 @@ tmp/parallel.file1.out tmp/parallel.file2.file.done tmp/parallel.file2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -63,7 +63,7 @@ OK /tmp/parallel.file2.file.done /tmp/parallel.file2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -73,7 +73,7 @@ OK /tmp/parallel.file2.file.done /tmp/parallel.file2.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK @@ -83,7 +83,7 @@ OK /tmp/parallel.file2.file.done /tmp/parallel.file2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -92,7 +92,7 @@ OK /tmp/parallel.file1.out /tmp/parallel.file2.file.done /tmp/parallel.file2.out -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -102,7 +102,7 @@ OK /tmp/parallel.file2.file.done /tmp/parallel.file2.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK @@ -112,7 +112,7 @@ tmp/parallel.file1.out tmp/parallel.file2.file.done tmp/parallel.file2.out good if no file -ls: tmp/parallel.file*: No such file or directory +ls: cannot access 'tmp/parallel.file*': No such file or directory OK ls: tmp/parallel.file*: No such file or directory OK @@ -122,7 +122,7 @@ OK /tmp/parallel.file2.file.done /tmp/parallel.file2.out good if no file -ls: /tmp/parallel.file*: No such file or directory +ls: cannot access '/tmp/parallel.file*': No such file or directory OK ls: /tmp/parallel.file*: No such file or directory OK diff --git a/testsuite/wanted-results/test60 b/testsuite/wanted-results/test60 index 4a53237d..8dfbe1d3 100644 --- a/testsuite/wanted-results/test60 +++ b/testsuite/wanted-results/test60 @@ -1,4 +1,4 @@ -echo '### Test --onall'; parallel --onall -S parallel@parallel-server1,parallel@parallel-server2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 +echo '### Test --onall'; parallel --onall -S vagrant@parallel-server1,vagrant@parallel-server2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 ### Test --onall 1 2 @@ -12,7 +12,7 @@ echo '### Test --onall'; parallel --onall -S parallel@parallel-server1,paralle 2 1 2 -echo '### Test | --onall'; seq 3 | parallel --onall -S parallel@parallel-server1,parallel@parallel-server2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - +echo '### Test | --onall'; seq 3 | parallel --onall -S vagrant@parallel-server1,vagrant@parallel-server2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c :::: - ### Test | --onall 1 2 @@ -32,7 +32,7 @@ echo '### Test | --onall'; seq 3 | parallel --onall -S parallel@parallel-serve 1 2 3 -echo '### Test --onall -u'; parallel --onall -S parallel@parallel-server1,parallel@parallel-server2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort +echo '### Test --onall -u'; parallel --onall -S vagrant@parallel-server1,vagrant@parallel-server2 -u '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 | sort ### Test --onall -u 1 1 @@ -52,31 +52,31 @@ echo '### Test --onall -u'; parallel --onall -S parallel@parallel-server1,para 3 3 3 -echo '### Test --nonall'; parallel --nonall -k -S parallel@parallel-server1,parallel@parallel-server2 'hostname' | sort +echo '### Test --nonall'; parallel --nonall -k -S vagrant@parallel-server1,vagrant@parallel-server2 'hostname' | sort ### Test --nonall -centos3.tange.dk -redhat9.tange.dk -echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall -S parallel@parallel-server1,parallel@parallel-server2 -u 'hostname|grep -q redhat9 && sleep 3; hostname;sleep 12;hostname;' +centos8.localdomain +freebsd11.localdomain +echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall -S vagrant@parallel-server1,vagrant@parallel-server2 -u 'hostname|grep -q redhat9 && sleep 3; hostname;sleep 12;hostname;' ### Test --nonall -u - should be interleaved x y x y -centos3.tange.dk -redhat9.tange.dk -centos3.tange.dk -redhat9.tange.dk -echo '### Test read sshloginfile from STDIN'; echo parallel@parallel-server1 | parallel -S - --nonall hostname; echo parallel@parallel-server1 | parallel --sshloginfile - --nonall hostname +freebsd11.localdomain +centos8.localdomain +freebsd11.localdomain +centos8.localdomain +echo '### Test read sshloginfile from STDIN'; echo vagrant@parallel-server1 | parallel -S - --nonall hostname; echo vagrant@parallel-server1 | parallel --sshloginfile - --nonall hostname ### Test read sshloginfile from STDIN -redhat9.tange.dk -redhat9.tange.dk -echo '### Test --nonall --basefile'; touch /tmp/nonall--basefile; parallel --nonall --basefile /tmp/nonall--basefile -S parallel@parallel-server1,parallel@parallel-server2 ls /tmp/nonall--basefile\; rm /tmp/nonall--basefile; rm /tmp/nonall--basefile +centos8.localdomain +centos8.localdomain +echo '### Test --nonall --basefile'; touch /tmp/nonall--basefile; parallel --nonall --basefile /tmp/nonall--basefile -S vagrant@parallel-server1,vagrant@parallel-server2 ls /tmp/nonall--basefile\; rm /tmp/nonall--basefile; rm /tmp/nonall--basefile ### Test --nonall --basefile /tmp/nonall--basefile /tmp/nonall--basefile -echo '### Test --onall --basefile'; touch /tmp/onall--basefile; parallel --onall --basefile /tmp/onall--basefile -S parallel@parallel-server1,parallel@parallel-server2 ls {}\; rm {} ::: /tmp/onall--basefile; rm /tmp/onall--basefile +echo '### Test --onall --basefile'; touch /tmp/onall--basefile; parallel --onall --basefile /tmp/onall--basefile -S vagrant@parallel-server1,vagrant@parallel-server2 ls {}\; rm {} ::: /tmp/onall--basefile; rm /tmp/onall--basefile ### Test --onall --basefile /tmp/onall--basefile /tmp/onall--basefile -echo '### Test --workdir .'; ssh parallel@parallel-server1 mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S parallel@parallel-server1 ::: pwd +echo '### Test --workdir .'; ssh vagrant@parallel-server1 mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S vagrant@parallel-server1 ::: pwd ### Test --workdir . -/home/parallel/mydir -echo '### Test --wd .'; ssh parallel@parallel-server2 mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S parallel@parallel-server2 ::: pwd +/home/vagrant/mydir +echo '### Test --wd .'; ssh vagrant@parallel-server2 mkdir -p mydir; mkdir -p $HOME/mydir; cd $HOME/mydir; parallel --workdir . -S vagrant@parallel-server2 ::: pwd ### Test --wd . -/home/parallel/mydir +/usr/home/vagrant/mydir