From f8782f43a752310e46707a3d2fa3ef3e5af34296 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 25 Jun 2019 21:18:27 +0200 Subject: [PATCH] parallel: Cache CPU spec and setpgrp method in file. Startup 15% faster. --- doc/release_new_version | 3 + src/parallel | 452 +++++++++---------- src/parallel.pod | 4 +- testsuite/tests-to-run/parallel-polarhome.sh | 43 +- testsuite/wanted-results/parallel-polarhome | 163 ++++--- 5 files changed, 317 insertions(+), 348 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 9dc5d55d..ead460a1 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -222,6 +222,9 @@ Quote of the month: New in this release: +* {= uq() =} will cause the replacement string to be unquoted. Example: parallel echo '{=uq()=}.jpg' ::: '*' + +https://techieroop.com/how-to-run-multiple-bash-scripts-in-parallel/ * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index f993e58a..1f607446 100755 --- a/src/parallel +++ b/src/parallel @@ -5940,7 +5940,7 @@ sub memfree_recompute() { # Run the script twice if it gives 0 (typically intermittent error) $self->{'memfree'} = ::qqx($script) || ::qqx($script); if(not $self->{'memfree'}) { - ::die_bug("Less than 1 byte free"); + ::die_bug("Less than 1 byte memory free"); } #::debug("mem","New free:",$self->{'memfree'}," "); } @@ -6970,53 +6970,75 @@ sub socket_core_thread() { # 'active' => #taskset_threads = number of taskset limited cores # } my $cpu; - + my $cached_cpuspec = $Global::cache_dir . "/tmp/sshlogin/" . + ::hostname() . "/cpuspec"; + if(-e $cached_cpuspec and -M $cached_cpuspec < 1) { + # Reading cached copy instead of /proc/cpuinfo is 17 ms faster + if(open(my $in_fh, "<", $cached_cpuspec)) { + ::debug("init","Read $cached_cpuspec"); + $cpu->{'sockets'} = int(<$in_fh>); + $cpu->{'cores'} = int(<$in_fh>); + $cpu->{'threads'} = int(<$in_fh>); + close $in_fh; + } + } if ($^O eq 'linux') { - $cpu = sct_gnu_linux(); + $cpu = sct_gnu_linux($cpu); } elsif ($^O eq 'android') { - $cpu = sct_android(); + $cpu = sct_android($cpu); } elsif ($^O eq 'freebsd') { - $cpu = sct_freebsd(); + $cpu = sct_freebsd($cpu); } elsif ($^O eq 'netbsd') { - $cpu = sct_netbsd(); + $cpu = sct_netbsd($cpu); } elsif ($^O eq 'openbsd') { - $cpu = sct_openbsd(); + $cpu = sct_openbsd($cpu); } elsif ($^O eq 'gnu') { - $cpu = sct_hurd(); + $cpu = sct_hurd($cpu); } elsif ($^O eq 'darwin') { - $cpu = sct_darwin(); + $cpu = sct_darwin($cpu); } elsif ($^O eq 'solaris') { - $cpu = sct_solaris(); + $cpu = sct_solaris($cpu); } elsif ($^O eq 'aix') { - $cpu = sct_aix(); + $cpu = sct_aix($cpu); } elsif ($^O eq 'hpux') { - $cpu = sct_hpux(); + $cpu = sct_hpux($cpu); } elsif ($^O eq 'nto') { - $cpu = sct_qnx(); + $cpu = sct_qnx($cpu); } elsif ($^O eq 'svr5') { - $cpu = sct_openserver(); + $cpu = sct_openserver($cpu); } elsif ($^O eq 'irix') { - $cpu = sct_irix(); + $cpu = sct_irix($cpu); } elsif ($^O eq 'dec_osf') { - $cpu = sct_tru64(); + $cpu = sct_tru64($cpu); } else { # Try all methods until we find something that works - $cpu = (sct_gnu_linux() - || sct_android() - || sct_freebsd() - || sct_netbsd() - || sct_openbsd() - || sct_hurd() - || sct_darwin() - || sct_solaris() - || sct_aix() - || sct_hpux() - || sct_qnx() - || sct_openserver() - || sct_irix() - || sct_tru64() + $cpu = (sct_gnu_linux($cpu) + || sct_android($cpu) + || sct_freebsd($cpu) + || sct_netbsd($cpu) + || sct_openbsd($cpu) + || sct_hurd($cpu) + || sct_darwin($cpu) + || sct_solaris($cpu) + || sct_aix($cpu) + || sct_hpux($cpu) + || sct_qnx($cpu) + || sct_openserver($cpu) + || sct_irix($cpu) + || sct_tru64($cpu) ); } + if(not grep { $_ > 0 } values %$cpu) { + $cpu = undef; + } + # Write cached copy instead of /proc/cpuinfo is 17 ms faster + if($cpu and open(my $out_fh, ">", $cached_cpuspec)) { + print $out_fh (map { chomp; "$_\n" } + $cpu->{'sockets'}, + $cpu->{'cores'}, + $cpu->{'threads'}); + close $out_fh; + } if(not $cpu) { my $nproc = nproc(); if($nproc) { @@ -7035,7 +7057,13 @@ sub socket_core_thread() { $cpu->{'active'} = 1 } - + $cpu->{'sockets'} ||= 1; + $cpu->{'threads'} ||= $cpu->{'cores'}; + $cpu->{'active'} ||= $cpu->{'threads'}; + chomp($cpu->{'sockets'}, + $cpu->{'cores'}, + $cpu->{'threads'}, + $cpu->{'active'}); # Choose minimum of active and actual my $mincpu; $mincpu->{'sockets'} = ::min($cpu->{'sockets'},$cpu->{'active'}); @@ -7044,33 +7072,35 @@ sub socket_core_thread() { return $mincpu; } -sub sct_gnu_linux() { +sub sct_gnu_linux($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } - my $cpu; + my $cpu = shift; local $/ = "\n"; # If delimiter is set, then $/ will be wrong - if($ENV{'PARALLEL_CPUINFO'} or -e "/proc/cpuinfo") { + my @cpuinfo; + if($ENV{'PARALLEL_CPUINFO'}) { + # Use CPUINFO from environment - used for testing only + @cpuinfo = split/(?<=\n)/,$ENV{'PARALLEL_CPUINFO'}; + } elsif($cpu->{'sockets'} and $cpu->{'cores'} and + $cpu->{'threads'}) { + # Skip /proc/cpuinfo - already set + } elsif(open(my $in_fh, "<", "/proc/cpuinfo")) { + # Read /proc/cpuinfo + @cpuinfo = <$in_fh>; + } + if(@cpuinfo) { $cpu->{'sockets'} = 0; $cpu->{'cores'} = 0; $cpu->{'threads'} = 0; my %seen; my %phy_seen; - my @cpuinfo; my $physicalid; - if(open(my $in_fh, "<", "/proc/cpuinfo")) { - @cpuinfo = <$in_fh>; - close $in_fh; - } - if($ENV{'PARALLEL_CPUINFO'}) { - # Use CPUINFO from environment - used for testing only - @cpuinfo = split/(?<=\n)/,$ENV{'PARALLEL_CPUINFO'}; - } for(@cpuinfo) { if(/^physical id.*[:](.*)/) { - $physicalid=$1; + $physicalid = $1; if(not $phy_seen{$1}++) { $cpu->{'sockets'}++; } @@ -7080,8 +7110,7 @@ sub sct_gnu_linux() { } /^processor.*[:]/i and $cpu->{'threads'}++; } - $cpu->{'sockets'} ||= 1; - $cpu->{'cores'} ||= $cpu->{'threads'}; + $cpu->{'cores'} ||= $cpu->{'threads'} || $cpu->{'sockets'}; } if(-e "/proc/self/status" and not $ENV{'PARALLEL_CPUINFO'}) { # if 'taskset' is used to limit number of threads @@ -7096,326 +7125,231 @@ sub sct_gnu_linux() { close $in_fh; } } - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_android() { +sub sct_android($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } # Use GNU/Linux - return sct_gnu_linux(); + return sct_gnu_linux(@_); } -sub sct_freebsd() { +sub sct_freebsd($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = (::qqx(qq{ sysctl -a dev.cpu | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }' }) + my $cpu = shift; + $cpu->{'cores'} ||= + (::qqx(qq{ sysctl -a dev.cpu | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }' }) or ::qqx(qq{ sysctl hw.ncpu | awk '{ print \$2 }' })); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - $cpu->{'threads'} = + $cpu->{'threads'} ||= (::qqx(qq{ sysctl hw.ncpu | awk '{ print \$2 }' }) or ::qqx(qq{ sysctl -a dev.cpu | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }' })); - $cpu->{'threads'} and chomp $cpu->{'threads'}; - $cpu->{'sockets'} ||= $cpu->{'cores'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_netbsd() { +sub sct_netbsd($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = ::qqx("sysctl -n hw.ncpu"); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - $cpu->{'threads'} = ::qqx("sysctl -n hw.ncpu"); - $cpu->{'threads'} and chomp $cpu->{'threads'}; - $cpu->{'sockets'} ||= $cpu->{'cores'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + my $cpu = shift; + $cpu->{'cores'} ||= ::qqx("sysctl -n hw.ncpu"); + return $cpu; } -sub sct_openbsd() { +sub sct_openbsd($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = ::qqx('sysctl -n hw.ncpu'); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - $cpu->{'threads'} = ::qqx('sysctl -n hw.ncpu'); - $cpu->{'threads'} and chomp $cpu->{'threads'}; - $cpu->{'sockets'} ||= $cpu->{'cores'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + my $cpu = shift; + $cpu->{'cores'} ||= ::qqx('sysctl -n hw.ncpu'); + return $cpu; } -sub sct_hurd() { +sub sct_hurd($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = ::qqx("nproc"); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - $cpu->{'threads'} = ::qqx("nproc"); - $cpu->{'threads'} and chomp $cpu->{'threads'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + my $cpu = shift; + $cpu->{'cores'} ||= ::qqx("nproc"); + return $cpu; } -sub sct_darwin() { +sub sct_darwin($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = + my $cpu = shift; + $cpu->{'cores'} ||= (::qqx('sysctl -n hw.physicalcpu') or ::qqx(qq{ sysctl -a hw | grep [^a-z]physicalcpu[^a-z] | awk '{ print \$2 }' })); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - $cpu->{'threads'} = + $cpu->{'threads'} ||= (::qqx('sysctl -n hw.logicalcpu') or ::qqx(qq{ sysctl -a hw | grep [^a-z]logicalcpu[^a-z] | awk '{ print \$2 }' })); - $cpu->{'threads'} and chomp $cpu->{'threads'}; - $cpu->{'sockets'} ||= $cpu->{'cores'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_solaris() { +sub sct_solaris($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - if(-x "/usr/sbin/psrinfo") { - my @psrinfo = ::qqx("/usr/sbin/psrinfo"); - if($#psrinfo >= 0) { - $cpu->{'cores'} = $#psrinfo +1; - } - } - if(-x "/usr/sbin/prtconf") { - my @prtconf = ::qqx("/usr/sbin/prtconf | grep cpu..instance"); - if($#prtconf >= 0) { - $cpu->{'cores'} = $#prtconf +1; - } - } - if(-x "/usr/sbin/prtconf") { - my @prtconf = ::qqx("/usr/sbin/prtconf | grep cpu..instance"); - if($#prtconf >= 0) { - $cpu->{'cores'} = $#prtconf +1; - } - } - $cpu->{'cores'} and chomp $cpu->{'cores'}; - - if(-x "/usr/sbin/psrinfo") { - my @psrinfo = ::qqx("/usr/sbin/psrinfo"); - if($#psrinfo >= 0) { - $cpu->{'threads'} = $#psrinfo +1; - } - } - if(-x "/usr/sbin/prtconf") { - my @prtconf = ::qqx("/usr/sbin/prtconf | grep cpu..instance"); - if($#prtconf >= 0) { - $cpu->{'threads'} = $#prtconf +1; - } - } - $cpu->{'threads'} and chomp $cpu->{'threads'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } -} - -sub sct_aix() { - # Returns: - # { 'sockets' => #sockets - # 'cores' => #cores - # 'threads' => #threads - # 'active' => #taskset_threads } - local $/ = "\n"; - my $cpu; - if(-x "/usr/sbin/lscfg") { - if(open(my $in_fh, "-|", "/usr/sbin/lscfg -vs |grep proc | wc -l|tr -d ' '")) { - $cpu->{'cores'} = <$in_fh>; - chomp ($cpu->{'cores'}); - close $in_fh; - } - } - if(-x "/usr/bin/vmstat") { - if(open(my $in_fh, "-|", "/usr/bin/vmstat 1 1")) { - while(<$in_fh>) { - /lcpu=([0-9]*) / and $cpu->{'threads'} = $1; + my $cpu = shift; + if(not $cpu->{'cores'}) { + if(-x "/usr/bin/kstat") { + my @chip_id = ::qqx("/usr/bin/kstat cpu_info|grep chip_id"); + if($#chip_id >= 0) { + $cpu->{'sockets'} ||= $#chip_id +1; + } + my @core_id = ::qqx("/usr/bin/kstat -m cpu_info|grep -w core_id|uniq"); + if($#core_id >= 0) { + $cpu->{'cores'} ||= $#core_id +1; + } + } + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = ::qqx("/usr/sbin/psrinfo -p"); + if($#psrinfo >= 0) { + $cpu->{'sockets'} ||= $psrinfo[0]; + } + } + if(-x "/usr/sbin/prtconf") { + my @prtconf = ::qqx("/usr/sbin/prtconf | grep cpu..instance"); + if($#prtconf >= 0) { + $cpu->{'cores'} ||= $#prtconf +1; } - close $in_fh; } } - - if(grep { /\d/ } values %$cpu) { - # BUG It is not not known how to calculate this - $cpu->{'sockets'} = 1; - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_hpux() { +sub sct_aix($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = + my $cpu = shift; + if(not $cpu->{'cores'}) { + if(-x "/usr/sbin/lscfg") { + if(open(my $in_fh, "-|", "/usr/sbin/lscfg -vs |grep proc | wc -l|tr -d ' '")) { + $cpu->{'cores'} = <$in_fh>; + close $in_fh; + } + } + } + if(not $cpu->{'threads'}) { + if(-x "/usr/bin/vmstat") { + if(open(my $in_fh, "-|", "/usr/bin/vmstat 1 1")) { + while(<$in_fh>) { + /lcpu=([0-9]*) / and $cpu->{'threads'} = $1; + } + close $in_fh; + } + } + } + return $cpu; +} + +sub sct_hpux($) { + # Returns: + # { 'sockets' => #sockets + # 'cores' => #cores + # 'threads' => #threads + # 'active' => #taskset_threads } + local $/ = "\n"; + my $cpu = shift; + $cpu->{'cores'} ||= ::qqx(qq{ /usr/bin/mpsched -s 2>&1 | grep 'Locality Domain Count' | awk '{ print \$4 }'}); - chomp($cpu->{'cores'}); - $cpu->{'threads'} = + $cpu->{'threads'} ||= ::qqx(qq{ /usr/bin/mpsched -s 2>&1 | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1"'}); - - if(grep { /\d/ } values %$cpu) { - # BUG It is not not known how to calculate this - $cpu->{'sockets'} = 1; - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_qnx() { +sub sct_qnx($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; + my $cpu = shift; # BUG: It is not known how to calculate this. - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_openserver() { +sub sct_openserver($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - if(-x "/usr/sbin/psrinfo") { - my @psrinfo = ::qqx("/usr/sbin/psrinfo"); - if($#psrinfo >= 0) { - $cpu->{'cores'} = $#psrinfo +1; - } - } - if(-x "/usr/sbin/psrinfo") { - my @psrinfo = ::qqx("/usr/sbin/psrinfo"); - if($#psrinfo >= 0) { - $cpu->{'threads'} = $#psrinfo +1; - } + my $cpu = shift; + if(not $cpu->{'cores'}) { + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = ::qqx("/usr/sbin/psrinfo"); + if($#psrinfo >= 0) { + $cpu->{'cores'} = $#psrinfo +1; + } + } } $cpu->{'sockets'} ||= $cpu->{'cores'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + return $cpu; } -sub sct_irix() { +sub sct_irix($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = ::qqx(qq{ hinv | grep HZ | grep Processor | awk '{print \$1}' }); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + my $cpu = shift; + $cpu->{'cores'} ||= + ::qqx(qq{ hinv | grep HZ | grep Processor | awk '{print \$1}' }); + return $cpu; } -sub sct_tru64() { +sub sct_tru64($) { # Returns: # { 'sockets' => #sockets # 'cores' => #cores # 'threads' => #threads # 'active' => #taskset_threads } local $/ = "\n"; - my $cpu; - $cpu->{'cores'} = ::qqx("sizer -pr"); - $cpu->{'cores'} and chomp $cpu->{'cores'}; - $cpu->{'cores'} ||= 1; + my $cpu = shift; + $cpu->{'cores'} ||= ::qqx("sizer -pr"); $cpu->{'sockets'} ||= $cpu->{'cores'}; $cpu->{'threads'} ||= $cpu->{'cores'}; - if(grep { /\d/ } values %$cpu) { - return $cpu; - } else { - return undef; - } + return $cpu; } sub sshcommand($) { @@ -9221,7 +9155,8 @@ sub start($) { return $pid; } - sub open3_setpgrp { + sub redefine_open3_setpgrp { + my $setgprp_cache = shift; # Select and run open3_setpgrp_internal/open3_setpgrp_external no warnings 'redefine'; my ($outfh,$name) = ::tmpfile(SUFFIX => ".tst"); @@ -9235,6 +9170,7 @@ sub start($) { "perl -MIPC::Open3 -e ". ::shell_quote_scalar_default($script) ); + my $redefine_eval; # Redirect STDERR temporarily, # so errors on MacOS X are ignored. open my $saveerr, ">&STDERR"; @@ -9249,14 +9185,30 @@ sub start($) { # or does not have bash: # Use (slow) external version unlink($name); - *open3_setpgrp = \&open3_setpgrp_external; + $redefine_eval = '*open3_setpgrp = \&open3_setpgrp_external'; ::debug("init","open3_setpgrp_external chosen\n"); } else { # Supports open3(x,x,x,"-") # This is 0.5 ms faster to run - *open3_setpgrp = \&open3_setpgrp_internal; + $redefine_eval = '*open3_setpgrp = \&open3_setpgrp_internal'; ::debug("init","open3_setpgrp_internal chosen\n"); } + open(my $fh, ">", $setgprp_cache) || die; + print $fh $redefine_eval; + close $fh; + eval $redefine_eval; + } + + sub open3_setpgrp { + my $setgprp_cache = $Global::cache_dir . "/tmp/sshlogin/" . + ::hostname() . "/setpgrp_func"; + if(-e $setgprp_cache) { + open(my $fh, "<", $setgprp_cache) || die; + eval <$fh> || die; + close $fh; + } else { + redefine_open3_setpgrp($setgprp_cache); + } # The sub is now redefined. Call it return open3_setpgrp(@_); } diff --git a/src/parallel.pod b/src/parallel.pod index f05432e4..023a2ada 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -277,9 +277,9 @@ shell quote a string perl quote a string -=item Z<> B +=item Z<> B (or B) -unquote current replacement string +do not quote current replacement string =item Z<> B diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index 5909ddb2..1bd4b9f9 100644 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -10,6 +10,7 @@ P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openi # Skip irix until Perl is upgraded (I cannot due to too small disk quota) P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax minix hurd beaglebone cubieboard2" P="$P_ALL" +#P="unixware freebsd netbsd" # tru64 takes 22s to run 4 parallels MAXTIME=50 @@ -20,13 +21,14 @@ MAXINNERPROC=${maxinnerproc:-3} export PARALLEL_SSH="ssh -oLogLevel=quiet" -# select a running master (debian-ppc, suse, ubuntu, or debian) +# select a running master (debian-ppc, suse, ubuntu, redhat, or debian) +# 2019-06-25 debian has too little free memory (and swap) MASTER=$(parallel -j0 --delay 0.1 --halt now,success=1 $PARALLEL_SSH {} echo {} \ - ::: {debian-ppc,ubuntu,debian,suse}.polarhome.com) + ::: {debian-ppc,ubuntu,suse,redhat}.polarhome.com) parallel -j0 --delay 0.1 --retries $RETRIES \ - rsync -L /usr/local/bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \ - ::: $MASTER:bin/ + rsync -L /usr/local/bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \ + ::: $MASTER:bin/ doit() { # Avoid the stupid /etc/issue.net banner at Polarhome: -oLogLevel=quiet @@ -35,7 +37,7 @@ doit() { export MAXTIME export RETRIES export MAXPROC - export RET_TIME_K="--memfree 150m -k --retries $RETRIES --timeout $MAXTIME" + export RET_TIME_K="--memfree 100m -k --retries $RETRIES --timeout $MAXTIME" LC_ALL=C MAXPROC=$(echo $(seq 300 | parallel -j0 echo {%} | sort -n | tail -n1) /$MAXINNERPROC | bc) @@ -44,8 +46,8 @@ doit() { echo '### Filter out working servers' # syllable often gives false positive parallel --timeout $MAXTIME -j10 ssh syllable true ::: {1..10} 2>/dev/null >/dev/null & - POLAR_ALL="`bin/parallel --memfree 200m -j0 -k --timeout 10 echo {} ::: $P`" - POLAR="`bin/parallel --memfree 200m -j0 -k --timeout 10 $PARALLEL_SSH {} echo {} ::: $P`" + POLAR_ALL="`bin/parallel --memfree 100m -j0 -k --timeout 10 echo {} ::: $P`" + POLAR="`bin/parallel --memfree 100m -j0 -k --timeout 10 $PARALLEL_SSH {} echo {} ::: $P`" diff <(echo "$POLAR_ALL") <(echo "$POLAR") S_POLAR=`bin/parallel -j0 $RET_TIME_K echo -S 1/{} ::: $POLAR` @@ -77,7 +79,7 @@ doit() { par_nonall() { parallel -j$MAXPROC $RET_TIME_K --delay 0.1 --tag \ --nonall $S_POLAR -S "1/sshwithpass minix" --argsep ,:- \ - 'source setupenv >&/dev/null || . `pwd`/setupenv;' "$@" + 'source setupenv 2>/dev/null; . `pwd`/setupenv;' "$@" # setupenv contains something like this (adapted to the local path and shell) # # PATH=$HOME/bin:$PATH:/usr/local/bin @@ -93,7 +95,7 @@ doit() { ::: $POLAR minix echo Done copying - env_parallel -d '\n\n' -vkj$MAXINNERPROC --delay 2 <<'EOF' + env_parallel -d '\n\n' -vkj$MAXINNERPROC --delay 2 <<'EOF' | echo echo '### Works on ...' @@ -123,7 +125,7 @@ doit() { cat <(echo bash only A) } export -f funcA; - bin/parallel funcA ::: 1' 2>&1 | sort + bin/parallel funcA ::: 1' 2>&1 | LANG=C sort echo echo '### Does PARALLEL_SHELL help exporting a bash function' @@ -142,16 +144,29 @@ doit() { bin/parallel funcB ::: 1' 2>&1 echo - echo '### env_parallel echo :::: <(echo OK)' + echo '### env_parallel --install' echo '(bash ksh mksh zsh only)' echo par_nonall 'bin/env_parallel --install && echo install-OK' 2>&1 + + echo + echo '### env_parallel echo env_parallel ::: run-OK' + echo '(bash ksh mksh zsh only)' + echo par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1 + + echo + echo '### env_parallel echo reading from process substitution :::: <(echo OK)' + echo '(bash ksh mksh zsh only)' + echo # csh on NetBSD does not support process substitution par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 | grep -v ': /tmp/.*: No such file or directory' - # Test empty command name in process list + echo + echo '### Test empty command name in process list' + echo '(bash ksh mksh zsh only)' + echo test_empty_cmd() { echo '### Test if empty command name in process list causes problems' perl -e '$0=" ";sleep 1000' & @@ -179,7 +194,9 @@ doit() { echo '### env_parset var1,var2,var3 seq ::: 2 3 4' par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1 EOF - + perl -ne 'm{UX:sh ./bin/sh.: ERROR: source: Not found} and next; + m{/usr/X11R7/bin/.: Permission denied.} and next; + print' } env_parallel -u -S$MASTER doit ::: 1 diff --git a/testsuite/wanted-results/parallel-polarhome b/testsuite/wanted-results/parallel-polarhome index a72d3ec3..085bd8d6 100644 --- a/testsuite/wanted-results/parallel-polarhome +++ b/testsuite/wanted-results/parallel-polarhome @@ -1,4 +1,4 @@ -MAXTIME=50 RETRIES=3 MAXPROC=84 MAXINNERPROC=3 +MAXTIME=50 RETRIES=3 MAXPROC=100 MAXINNERPROC=3 ### Filter out working servers 1d0 < openstep @@ -17,7 +17,6 @@ MAXTIME=50 RETRIES=3 MAXPROC=84 MAXINNERPROC=3 bin/parallel qnx copy qnx bin/parallel parallel bin/parallel pidora copy pidora bin/parallel parallel bin/parallel tru64 copy tru64 bin/parallel parallel -bin/parallel hpux-ia64 copy hpux-ia64 bin/parallel parallel bin/parallel raspbian copy raspbian bin/parallel parallel bin/parallel solaris copy solaris bin/parallel parallel bin/parallel openindiana copy openindiana bin/parallel parallel @@ -38,12 +37,10 @@ bin/parallel netbsd copy netbsd bin/parallel parallel bin/parallel openbsd copy openbsd bin/parallel parallel bin/parallel freebsd copy freebsd bin/parallel parallel bin/parallel debian copy debian bin/parallel parallel -bin/parallel hurd copy hurd bin/parallel parallel bin/parallel minix copy minix bin/parallel parallel bin/env_parallel qnx copy qnx bin/env_parallel env_parallel bin/env_parallel pidora copy pidora bin/env_parallel env_parallel bin/env_parallel tru64 copy tru64 bin/env_parallel env_parallel -bin/env_parallel hpux-ia64 copy hpux-ia64 bin/env_parallel env_parallel bin/env_parallel raspbian copy raspbian bin/env_parallel env_parallel bin/env_parallel solaris copy solaris bin/env_parallel env_parallel bin/env_parallel openindiana copy openindiana bin/env_parallel env_parallel @@ -64,12 +61,10 @@ bin/env_parallel netbsd copy netbsd bin/env_parallel env_parallel bin/env_parallel openbsd copy openbsd bin/env_parallel env_parallel bin/env_parallel freebsd copy freebsd bin/env_parallel env_parallel bin/env_parallel debian copy debian bin/env_parallel env_parallel -bin/env_parallel hurd copy hurd bin/env_parallel env_parallel bin/env_parallel minix copy minix bin/env_parallel env_parallel bin/env_parallel.ash qnx copy qnx bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash pidora copy pidora bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash tru64 copy tru64 bin/env_parallel.ash env_parallel.ash -bin/env_parallel.ash hpux-ia64 copy hpux-ia64 bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash raspbian copy raspbian bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash solaris copy solaris bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash openindiana copy openindiana bin/env_parallel.ash env_parallel.ash @@ -90,12 +85,10 @@ bin/env_parallel.ash netbsd copy netbsd bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash openbsd copy openbsd bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash freebsd copy freebsd bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash debian copy debian bin/env_parallel.ash env_parallel.ash -bin/env_parallel.ash hurd copy hurd bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash minix copy minix bin/env_parallel.ash env_parallel.ash bin/env_parallel.bash qnx copy qnx bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash pidora copy pidora bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash tru64 copy tru64 bin/env_parallel.bash env_parallel.bash -bin/env_parallel.bash hpux-ia64 copy hpux-ia64 bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash raspbian copy raspbian bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash solaris copy solaris bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash openindiana copy openindiana bin/env_parallel.bash env_parallel.bash @@ -116,12 +109,10 @@ bin/env_parallel.bash netbsd copy netbsd bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash openbsd copy openbsd bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash freebsd copy freebsd bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash debian copy debian bin/env_parallel.bash env_parallel.bash -bin/env_parallel.bash hurd copy hurd bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash minix copy minix bin/env_parallel.bash env_parallel.bash bin/env_parallel.csh qnx copy qnx bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh pidora copy pidora bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh tru64 copy tru64 bin/env_parallel.csh env_parallel.csh -bin/env_parallel.csh hpux-ia64 copy hpux-ia64 bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh raspbian copy raspbian bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh solaris copy solaris bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh openindiana copy openindiana bin/env_parallel.csh env_parallel.csh @@ -142,12 +133,10 @@ bin/env_parallel.csh netbsd copy netbsd bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh openbsd copy openbsd bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh freebsd copy freebsd bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh debian copy debian bin/env_parallel.csh env_parallel.csh -bin/env_parallel.csh hurd copy hurd bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh minix copy minix bin/env_parallel.csh env_parallel.csh bin/env_parallel.dash qnx copy qnx bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash pidora copy pidora bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash tru64 copy tru64 bin/env_parallel.dash env_parallel.dash -bin/env_parallel.dash hpux-ia64 copy hpux-ia64 bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash raspbian copy raspbian bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash solaris copy solaris bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash openindiana copy openindiana bin/env_parallel.dash env_parallel.dash @@ -168,12 +157,10 @@ bin/env_parallel.dash netbsd copy netbsd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash openbsd copy openbsd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash freebsd copy freebsd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash debian copy debian bin/env_parallel.dash env_parallel.dash -bin/env_parallel.dash hurd copy hurd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash minix copy minix bin/env_parallel.dash env_parallel.dash bin/env_parallel.fish qnx copy qnx bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish pidora copy pidora bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish tru64 copy tru64 bin/env_parallel.fish env_parallel.fish -bin/env_parallel.fish hpux-ia64 copy hpux-ia64 bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish raspbian copy raspbian bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish solaris copy solaris bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish openindiana copy openindiana bin/env_parallel.fish env_parallel.fish @@ -194,12 +181,10 @@ bin/env_parallel.fish netbsd copy netbsd bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish openbsd copy openbsd bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish freebsd copy freebsd bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish debian copy debian bin/env_parallel.fish env_parallel.fish -bin/env_parallel.fish hurd copy hurd bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish minix copy minix bin/env_parallel.fish env_parallel.fish bin/env_parallel.ksh qnx copy qnx bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh pidora copy pidora bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh tru64 copy tru64 bin/env_parallel.ksh env_parallel.ksh -bin/env_parallel.ksh hpux-ia64 copy hpux-ia64 bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh raspbian copy raspbian bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh solaris copy solaris bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh openindiana copy openindiana bin/env_parallel.ksh env_parallel.ksh @@ -220,12 +205,10 @@ bin/env_parallel.ksh netbsd copy netbsd bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh openbsd copy openbsd bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh freebsd copy freebsd bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh debian copy debian bin/env_parallel.ksh env_parallel.ksh -bin/env_parallel.ksh hurd copy hurd bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh minix copy minix bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.mksh qnx copy qnx bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh pidora copy pidora bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh tru64 copy tru64 bin/env_parallel.mksh env_parallel.mksh -bin/env_parallel.mksh hpux-ia64 copy hpux-ia64 bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh raspbian copy raspbian bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh solaris copy solaris bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh openindiana copy openindiana bin/env_parallel.mksh env_parallel.mksh @@ -246,12 +229,10 @@ bin/env_parallel.mksh netbsd copy netbsd bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh openbsd copy openbsd bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh freebsd copy freebsd bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh debian copy debian bin/env_parallel.mksh env_parallel.mksh -bin/env_parallel.mksh hurd copy hurd bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh minix copy minix bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.pdksh qnx copy qnx bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh pidora copy pidora bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh tru64 copy tru64 bin/env_parallel.pdksh env_parallel.pdksh -bin/env_parallel.pdksh hpux-ia64 copy hpux-ia64 bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh raspbian copy raspbian bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh solaris copy solaris bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh openindiana copy openindiana bin/env_parallel.pdksh env_parallel.pdksh @@ -272,12 +253,10 @@ bin/env_parallel.pdksh netbsd copy netbsd bin/env_parallel.pdksh env_parallel.pd bin/env_parallel.pdksh openbsd copy openbsd bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh freebsd copy freebsd bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh debian copy debian bin/env_parallel.pdksh env_parallel.pdksh -bin/env_parallel.pdksh hurd copy hurd bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh minix copy minix bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.sh qnx copy qnx bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh pidora copy pidora bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh tru64 copy tru64 bin/env_parallel.sh env_parallel.sh -bin/env_parallel.sh hpux-ia64 copy hpux-ia64 bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh raspbian copy raspbian bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh solaris copy solaris bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh openindiana copy openindiana bin/env_parallel.sh env_parallel.sh @@ -298,12 +277,10 @@ bin/env_parallel.sh netbsd copy netbsd bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh openbsd copy openbsd bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh freebsd copy freebsd bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh debian copy debian bin/env_parallel.sh env_parallel.sh -bin/env_parallel.sh hurd copy hurd bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh minix copy minix bin/env_parallel.sh env_parallel.sh bin/env_parallel.tcsh qnx copy qnx bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh pidora copy pidora bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh tru64 copy tru64 bin/env_parallel.tcsh env_parallel.tcsh -bin/env_parallel.tcsh hpux-ia64 copy hpux-ia64 bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh raspbian copy raspbian bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh solaris copy solaris bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh openindiana copy openindiana bin/env_parallel.tcsh env_parallel.tcsh @@ -324,12 +301,10 @@ bin/env_parallel.tcsh netbsd copy netbsd bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh openbsd copy openbsd bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh freebsd copy freebsd bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh debian copy debian bin/env_parallel.tcsh env_parallel.tcsh -bin/env_parallel.tcsh hurd copy hurd bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh minix copy minix bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.zsh qnx copy qnx bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh pidora copy pidora bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh tru64 copy tru64 bin/env_parallel.zsh env_parallel.zsh -bin/env_parallel.zsh hpux-ia64 copy hpux-ia64 bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh raspbian copy raspbian bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh solaris copy solaris bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh openindiana copy openindiana bin/env_parallel.zsh env_parallel.zsh @@ -350,12 +325,10 @@ bin/env_parallel.zsh netbsd copy netbsd bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh openbsd copy openbsd bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh freebsd copy freebsd bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh debian copy debian bin/env_parallel.zsh env_parallel.zsh -bin/env_parallel.zsh hurd copy hurd bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh minix copy minix bin/env_parallel.zsh env_parallel.zsh bin/parcat qnx copy qnx bin/parcat parcat bin/parcat pidora copy pidora bin/parcat parcat bin/parcat tru64 copy tru64 bin/parcat parcat -bin/parcat hpux-ia64 copy hpux-ia64 bin/parcat parcat bin/parcat raspbian copy raspbian bin/parcat parcat bin/parcat solaris copy solaris bin/parcat parcat bin/parcat openindiana copy openindiana bin/parcat parcat @@ -376,7 +349,6 @@ bin/parcat netbsd copy netbsd bin/parcat parcat bin/parcat openbsd copy openbsd bin/parcat parcat bin/parcat freebsd copy freebsd bin/parcat parcat bin/parcat debian copy debian bin/parcat parcat -bin/parcat hurd copy hurd bin/parcat parcat bin/parcat minix copy minix bin/parcat parcat Done copying @@ -391,9 +363,8 @@ aix Works on aix7 centos Works on centos.polarhome.com debian Works on debian debian-ppc Works on debian-ppc -freebsd Syntax error: Bad fd number +freebsd Works on freebsd.polarhome.com hpux Works on hpux64 -hpux-ia64 Works on hpux-ia64 macosx Works on macosx.polarhome.com mandriva Works on mandriva.polarhome.com miros Works on miros.polarhome.com @@ -402,7 +373,6 @@ openbsd Works on openbsd.polarhome.com openindiana Works on openindiana pidora Works on pidora qnx Works on qnx -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx parallel: Warning: Cannot figure out number of cpus. Using 1. raspbian Works on raspbian redhat Works on redhat.polarhome.com @@ -413,7 +383,6 @@ suse Works on suse tru64 Works on tru64.polarhome.com ubuntu Works on ubuntu unixware Works on unixware.polarhome.com -unixware UX:sh (/bin/sh): ERROR: source: Not found echo echo '### --number-of-cores/--number-of-cpus should work with no error' echo @@ -430,12 +399,13 @@ debian 1 debian 2 debian-ppc 1 debian-ppc 1 -freebsd Syntax error: Bad fd number +freebsd 1 +freebsd 1 hpux 1 hpux 1 hpux-ia64 1 hpux-ia64 1 -macosx 2 +macosx 1 macosx 2 mandriva 1 mandriva 1 @@ -451,7 +421,6 @@ pidora 1 pidora 1 qnx 1 qnx 1 -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx parallel: Warning: Cannot figure out number of cpus. Using 1. qnx parallel: Warning: Cannot figure out number of cpus. Using 1. raspbian 1 @@ -460,9 +429,9 @@ redhat 1 redhat 2 scosysv 1 scosysv 1 -solaris solaris 2 -solaris-x86 +solaris 2 +solaris-x86 1 solaris-x86 1 suse 1 suse 1 @@ -472,7 +441,6 @@ ubuntu 1 ubuntu 2 unixware 1 unixware 1 -unixware UX:sh (/bin/sh): ERROR: source: Not found aix 1 aix 1 centos 1 @@ -481,7 +449,8 @@ debian 2 debian 2 debian-ppc 1 debian-ppc 1 -freebsd Syntax error: Bad fd number +freebsd 1 +freebsd 1 hpux 1 hpux 1 hpux-ia64 2 @@ -502,7 +471,6 @@ pidora 1 pidora 1 qnx 1 qnx 1 -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx parallel: Warning: Cannot figure out number of cpus. Using 1. qnx parallel: Warning: Cannot figure out number of cpus. Using 1. raspbian 1 @@ -523,7 +491,6 @@ ubuntu 2 ubuntu 2 unixware 1 unixware 1 -unixware UX:sh (/bin/sh): ERROR: source: Not found echo echo '### Fails if tmpdir is R/O' echo @@ -542,7 +509,8 @@ debian Error in tempfile() using template /XXXXXXXX.arg: Could not create temp f debian OK readonly tmp debian-ppc Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. debian-ppc OK readonly tmp -freebsd Syntax error: Bad fd number +freebsd Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. +freebsd OK readonly tmp hpux Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 hpux OK readonly tmp hpux-ia64 Error in tempfile() using /XXXXXXXX.arg: Parent directory (/) is not writable @@ -566,7 +534,6 @@ pidora Error in tempfile() using template /XXXXXXXX.arg: Could not create temp f pidora OK readonly tmp qnx Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 qnx OK readonly tmp -qnx /bin/sh: >&/dev/null : illegal file descriptor name raspbian Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 raspbian OK readonly tmp redhat Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 @@ -577,7 +544,7 @@ scosysv OK readonly tmp solaris Error in tempfile() using /XXXXXXXX.arg: Parent directory (/) is not writable solaris at /home/t/tange/bin/parallel line 0000 solaris OK readonly tmp -solaris-x86 Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/tange/bin/parallel line 0000 +solaris-x86 Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at ~/bin/parallel line 0000 solaris-x86 OK readonly tmp suse Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. suse OK readonly tmp @@ -588,7 +555,6 @@ ubuntu Error in tempfile() using template /XXXXXXXX.arg: Could not create temp f ubuntu OK readonly tmp unixware Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 unixware OK readonly tmp -unixware UX:sh (/bin/sh): ERROR: source: Not found echo echo '### Does exporting a bash function make parallel fail?' echo 'If login shell is not bash compatible it fails' @@ -599,7 +565,7 @@ unixware UX:sh (/bin/sh): ERROR: source: Not found cat <(echo bash only A) } export -f funcA; - bin/parallel funcA ::: 1' 2>&1 | sort + bin/parallel funcA ::: 1' 2>&1 | LANG=C sort ### Does exporting a bash function make parallel fail? If login shell is not bash compatible it fails @@ -612,7 +578,8 @@ debian bash only A debian test funcA debian-ppc bash only A debian-ppc test funcA -freebsd Syntax error: Bad fd number +freebsd Syntax error: "(" unexpected (expecting word) +freebsd test funcA hpux bash only A hpux test funcA hpux-ia64 bash only A @@ -653,7 +620,6 @@ tru64 test funcA ubuntu bash only A ubuntu test funcA unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 3: `(' unexpected -unixware UX:sh (/bin/sh): ERROR: source: Not found unixware test funcA echo echo '### Does PARALLEL_SHELL help exporting a bash function' @@ -725,16 +691,12 @@ ubuntu bash only B unixware test funcB unixware bash only B echo - echo '### env_parallel echo :::: <(echo OK)' + echo '### env_parallel --install' echo '(bash ksh mksh zsh only)' echo par_nonall 'bin/env_parallel --install && echo install-OK' 2>&1 - par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1 - # csh on NetBSD does not support process substitution - par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 | - grep -v ': /tmp/.*: No such file or directory' -### env_parallel echo :::: <(echo OK) +### env_parallel --install (bash ksh mksh zsh only) aix Installed env_parallel in: @@ -781,7 +743,17 @@ debian-ppc /home/t/tange/.profile debian-ppc /home/t/tange/.cshrc debian-ppc /home/t/tange/.tcshrc debian-ppc install-OK -freebsd Syntax error: Bad fd number +freebsd Installed env_parallel in: +freebsd /home/t/tange/.bashrc +freebsd /home/t/tange/.shrc +freebsd /home/t/tange/.zshenv +freebsd /home/t/tange/.config/fish/config.fish +freebsd /home/t/tange/.kshrc +freebsd /home/t/tange/.mkshrc +freebsd /home/t/tange/.profile +freebsd /home/t/tange/.cshrc +freebsd /home/t/tange/.tcshrc +freebsd install-OK hpux Installed env_parallel in: hpux /home/t/tange/.bashrc hpux /home/t/tange/.shrc @@ -893,7 +865,6 @@ qnx /home/t/tange/.profile qnx /home/t/tange/.cshrc qnx /home/t/tange/.tcshrc qnx install-OK -qnx /bin/sh: >&/dev/null : illegal file descriptor name raspbian Installed env_parallel in: raspbian /home/t/tange/.bashrc raspbian /home/t/tange/.shrc @@ -939,15 +910,15 @@ solaris /home/t/tange/.cshrc solaris /home/t/tange/.tcshrc solaris install-OK solaris-x86 Installed env_parallel in: -solaris-x86 /home/tange/.bashrc -solaris-x86 /home/tange/.shrc -solaris-x86 /home/tange/.zshenv -solaris-x86 /home/tange/.config/fish/config.fish -solaris-x86 /home/tange/.kshrc -solaris-x86 /home/tange/.mkshrc -solaris-x86 /home/tange/.profile -solaris-x86 /home/tange/.cshrc -solaris-x86 /home/tange/.tcshrc +solaris-x86 ~/.bashrc +solaris-x86 ~/.shrc +solaris-x86 ~/.zshenv +solaris-x86 ~/.config/fish/config.fish +solaris-x86 ~/.kshrc +solaris-x86 ~/.mkshrc +solaris-x86 ~/.profile +solaris-x86 ~/.cshrc +solaris-x86 ~/.tcshrc solaris-x86 install-OK suse Installed env_parallel in: suse /home/t/tange/.bashrc @@ -993,12 +964,21 @@ unixware /home/t/tange/.profile unixware /home/t/tange/.cshrc unixware /home/t/tange/.tcshrc unixware install-OK -unixware UX:sh (/bin/sh): ERROR: source: Not found + echo + echo '### env_parallel echo env_parallel ::: run-OK' + echo '(bash ksh mksh zsh only)' + echo + par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1 + +### env_parallel echo env_parallel ::: run-OK +(bash ksh mksh zsh only) + aix env_parallel run-OK centos env_parallel run-OK debian env_parallel run-OK debian-ppc env_parallel run-OK -freebsd Syntax error: Bad fd number +freebsd env_parallel run-OK +freebsd Cannot fork: Resource temporarily unavailable hpux env_parallel run-OK hpux-ia64 env_parallel run-OK macosx env_parallel run-OK @@ -1009,7 +989,6 @@ openbsd env_parallel run-OK openindiana env_parallel run-OK pidora env_parallel run-OK qnx env_parallel run-OK -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: compgen: cannot execute - No such file or directory qnx /bin/sh: compgen: cannot execute - No such file or directory qnx /bin/sh: compgen: cannot execute - No such file or directory @@ -1024,7 +1003,17 @@ suse env_parallel run-OK tru64 env_parallel run-OK ubuntu env_parallel run-OK unixware env_parallel run-OK -unixware UX:sh (/bin/sh): ERROR: source: Not found + echo + echo '### env_parallel echo reading from process substitution :::: <(echo OK)' + echo '(bash ksh mksh zsh only)' + echo + # csh on NetBSD does not support process substitution + par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 | + grep -v ': /tmp/.*: No such file or directory' + +### env_parallel echo reading from process substitution :::: <(echo OK) +(bash ksh mksh zsh only) + aix reading from process substitution OK centos reading from process substitution OK debian reading from process substitution OK @@ -1051,7 +1040,10 @@ suse reading from process substitution OK tru64 reading from process substitution OK ubuntu reading from process substitution OK unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `(' unexpected - # Test empty command name in process list + echo + echo '### Test empty command name in process list' + echo '(bash ksh mksh zsh only)' + echo test_empty_cmd() { echo '### Test if empty command name in process list causes problems' perl -e '$0=" ";sleep 1000' & @@ -1062,6 +1054,10 @@ unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `(' unexpected export -f test_empty_cmd export PARALLEL_SHELL=bin/bash PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1 + +### Test empty command name in process list +(bash ksh mksh zsh only) + aix ### Test if empty command name in process list causes problems aix OK_with_empty_cmd centos ### Test if empty command name in process list causes problems @@ -1127,7 +1123,7 @@ aix 1 2 1 2 3 1 2 3 4 centos 1 2 1 2 3 1 2 3 4 debian 1 2 1 2 3 1 2 3 4 debian-ppc 1 2 1 2 3 1 2 3 4 -freebsd Syntax error: Bad fd number +freebsd eval: 1: Syntax error: word unexpected (expecting ")") hpux hpux-ia64 1 2 1 2 3 1 2 3 4 macosx 1 2 1 2 3 1 2 3 4 @@ -1139,7 +1135,6 @@ openbsd openindiana 1 2 1 2 3 1 2 3 4 pidora 1 2 1 2 3 1 2 3 4 qnx -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx parallel: Warning: Cannot figure out number of cpus. Using 1. qnx /bin/sh: syntax error: `(' unexpected raspbian 1 2 1 2 3 1 2 3 4 @@ -1150,14 +1145,17 @@ solaris-x86 1 2 1 2 3 1 2 3 4 suse 1 2 1 2 3 1 2 3 4 tru64 1 2 1 2 3 1 2 3 4 ubuntu 1 2 1 2 3 1 2 3 4 -unixware UX:sh (/bin/sh): ERROR: source: Not found unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `arr=' unexpected ### env_parset arr seq ::: 2 3 4 aix 2 2 3 2 3 4 centos 2 2 3 2 3 4 debian 2 2 3 2 3 4 debian-ppc 2 2 3 2 3 4 -freebsd Syntax error: Bad fd number +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable +freebsd eval: 1: Syntax error: word unexpected (expecting ")") hpux hpux-ia64 1 2 1 2 1 2 macosx 2 2 3 2 3 4 @@ -1170,7 +1168,6 @@ openbsd openindiana 2 2 3 2 3 4 pidora 2 2 3 2 3 4 qnx -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: compgen: cannot execute - No such file or directory qnx /bin/sh: compgen: cannot execute - No such file or directory qnx /bin/sh: compgen: cannot execute - No such file or directory @@ -1184,7 +1181,6 @@ solaris-x86 2 2 3 2 3 4 suse 2 2 3 2 3 4 tru64 2 2 3 2 3 4 ubuntu 2 2 3 2 3 4 -unixware UX:sh (/bin/sh): ERROR: source: Not found unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `arr=' unexpected echo echo '### parset var1,var2,var3 seq ::: 2 3 4' @@ -1202,7 +1198,7 @@ aix 1 2,1 2 3,1 2 3 4 centos 1 2,1 2 3,1 2 3 4 debian 1 2,1 2 3,1 2 3 4 debian-ppc 1 2,1 2 3,1 2 3 4 -freebsd Syntax error: Bad fd number +freebsd 1 2,1 2 3,1 2 3 4 hpux ,, hpux-ia64 1 2,1 2 3,1 2 3 4 macosx 1 2,1 2 3,1 2 3 4 @@ -1214,7 +1210,6 @@ openbsd ,, openindiana 1 2,1 2 3,1 2 3 4 pidora 1 2,1 2 3,1 2 3 4 qnx ,, -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx parallel: Warning: Cannot figure out number of cpus. Using 1. qnx parallel: Warning: Cannot figure out number of cpus. Using 1. raspbian 1 2,1 2 3,1 2 3 4 @@ -1226,13 +1221,17 @@ suse 1 2,1 2 3,1 2 3 4 tru64 1 2,1 2 3,1 2 3 4 ubuntu 1 2,1 2 3,1 2 3 4 unixware 1 2,1 2 3,1 2 3 4 -unixware UX:sh (/bin/sh): ERROR: source: Not found ### env_parset var1,var2,var3 seq ::: 2 3 4 aix 2,2 3,2 3 4 centos 2,2 3,2 3 4 debian 2,2 3,2 3 4 debian-ppc 2,2 3,2 3 4 -freebsd Syntax error: Bad fd number +freebsd 2,2 3,2 3 4 +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable +freebsd Cannot fork: Resource temporarily unavailable hpux ,, hpux-ia64 1 2,1 2,1 2 macosx 2,2 3,2 3 4 @@ -1245,7 +1244,6 @@ openbsd ,, openindiana 2,2 3,2 3 4 pidora 2,2 3,2 3 4 qnx ,, -qnx /bin/sh: >&/dev/null : illegal file descriptor name qnx /bin/sh: compgen: cannot execute - No such file or directory qnx /bin/sh: compgen: cannot execute - No such file or directory qnx /bin/sh: compgen: cannot execute - No such file or directory @@ -1260,4 +1258,3 @@ suse 2,2 3,2 3 4 tru64 2,2 3,2 3 4 ubuntu 2,2 3,2 3 4 unixware 2,2 3,2 3 4 -unixware UX:sh (/bin/sh): ERROR: source: Not found