diff --git a/README b/README index 5c8f1d15..262719e3 100644 --- a/README +++ b/README @@ -89,8 +89,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: -O. Tange (2011): GNU Parallel - The Command-Line Power Tool, -;login: The USENIX Magazine, February 2011:42-47. + O. Tange (2011): GNU Parallel - The Command-Line Power Tool, + ;login: The USENIX Magazine, February 2011:42-47. = New versions = diff --git a/src/parallel b/src/parallel index a5639a64..4059fd67 100755 --- a/src/parallel +++ b/src/parallel @@ -2248,7 +2248,7 @@ sub progress { $last_column_time = time; $columns = $ENV{'COLUMNS'}; if(not $columns) { - my $resize = qx{ resize 2>/dev/null }; + my $resize = qx{ sh -c 'resize 2>/dev/null' }; $resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; }; } $columns ||= 80; @@ -2697,6 +2697,7 @@ sub parallelized_host_filtering { # will make it less likely to overload the ssh daemon. # --retries 3: If the ssh daemon it overloaded, try 3 times # -s 16000: Half of the max line on UnixWare + # TODO sh -c wrapper to work in csh my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null"; ::debug("init", $cmd, "\n"); my @out; @@ -4502,6 +4503,7 @@ sub simultaneous_sshlogin { my $sshcmd = $self->sshcommand(); my $serverlogin = $self->serverlogin(); my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : ""; + # TODO sh -c wrapper to work for csh my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin &1 &"x$wanted_processes; ::debug("init", "Trying $wanted_processes logins at $serverlogin\n"); open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or @@ -4717,7 +4719,7 @@ sub no_of_cores { sub nproc { # Returns: # Number of cores using `nproc` - my $no_of_cores = `nproc 2>/dev/null`; + my $no_of_cores = qx{ sh -c 'nproc 2>/dev/null' }; return $no_of_cores; } @@ -4792,6 +4794,7 @@ sub no_of_cpus_freebsd { # Returns: # Number of physical CPUs on FreeBSD # undef if not FreeBSD + # TODO sh -c wrapper to work in csh my $no_of_cpus = (`sysctl -a dev.cpu 2>/dev/null | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }'` or @@ -4804,6 +4807,7 @@ sub no_of_cores_freebsd { # Returns: # Number of CPU cores on FreeBSD # undef if not FreeBSD + # TODO sh -c wrapper to work in csh my $no_of_cores = (`sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'` or @@ -4816,7 +4820,7 @@ sub no_of_cpus_netbsd { # Returns: # Number of physical CPUs on NetBSD # undef if not NetBSD - my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; + my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; chomp $no_of_cpus; return $no_of_cpus; } @@ -4825,7 +4829,7 @@ sub no_of_cores_netbsd { # Returns: # Number of CPU cores on NetBSD # undef if not NetBSD - my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`; + my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; chomp $no_of_cores; return $no_of_cores; } @@ -4834,7 +4838,7 @@ sub no_of_cpus_openbsd { # Returns: # Number of physical CPUs on OpenBSD # undef if not OpenBSD - my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; + my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; chomp $no_of_cpus; return $no_of_cpus; } @@ -4843,7 +4847,7 @@ sub no_of_cores_openbsd { # Returns: # Number of CPU cores on OpenBSD # undef if not OpenBSD - my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`; + my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' }; chomp $no_of_cores; return $no_of_cores; } @@ -4852,7 +4856,7 @@ sub no_of_cpus_hurd { # Returns: # Number of physical CPUs on HURD # undef if not HURD - my $no_of_cpus = `nproc`; + my $no_of_cpus = qx{ nproc }; chomp $no_of_cpus; return $no_of_cpus; } @@ -4870,6 +4874,7 @@ sub no_of_cpus_darwin { # Returns: # Number of physical CPUs on Mac Darwin # undef if not Mac Darwin + # TODO sh -c wrapper to work in csh my $no_of_cpus = (`sysctl -n hw.physicalcpu 2>/dev/null` or @@ -4881,6 +4886,7 @@ sub no_of_cores_darwin { # Returns: # Number of CPU cores on Mac Darwin # undef if not Mac Darwin + # TODO sh -c wrapper to work in csh my $no_of_cores = (`sysctl -n hw.logicalcpu 2>/dev/null` or @@ -4961,7 +4967,7 @@ sub no_of_cpus_hpux { # Number of physical CPUs on HP-UX # undef if not HP-UX my $no_of_cpus = - qx{/usr/bin/mpsched -s 2>&1 | grep 'Locality Domain Count' | awk '{ print \$4 }'}; + qx{ sh -c '/usr/bin/mpsched -s 2>&1' | grep 'Locality Domain Count' | awk '{ print \$4 }'}; return $no_of_cpus; } @@ -4970,7 +4976,7 @@ sub no_of_cores_hpux { # Number of CPU cores on HP-UX # undef if not HP-UX my $no_of_cores = - qx{/usr/bin/mpsched -s 2>&1 | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'}; + qx{ sh -c '/usr/bin/mpsched -s 2>&1' | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'}; return $no_of_cores; } @@ -5188,7 +5194,7 @@ sub cleanup_cmd { $dir .= $_."/"; unshift @rmdir, ::shell_quote_file($dir); } - my $rmdir = @rmdir ? "rmdir @rmdir >&/dev/null;" : ""; + my $rmdir = @rmdir ? "sh -c 'rmdir @rmdir 2>/dev/null';" : ""; if(defined $opt::workdir and $opt::workdir eq "...") { $rmdir .= "rm -rf " . ::shell_quote_file($workdir).';'; } @@ -6697,6 +6703,7 @@ sub print_dryrun_and_verbose { unlink $tmuxsocket; ::status("See output with: $ENV{'TMUX'} -S $tmuxsocket attach\n"); } + # TODO sh -c wrapper for >& $tmux = $ENV{'TMUX'}." -S $tmuxsocket new-session -s p$$ -d 'sleep .2' >&/dev/null;" . $ENV{'TMUX'}." -S $tmuxsocket new-window -t p$$ -n $title"; @@ -8112,26 +8119,16 @@ sub tmux_length { my $len = shift; if($opt::tmux) { $ENV{'TMUX'} ||= "tmux"; - my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb"); - $Global::unlink{$tmpfile}=1; - close $fh; - unlink $tmpfile; - my $b2020 = "x"x2020; - my $b16320 = "x"x16320; - my $b100000 = "x"x100000; - my $blen = "x"x$len; - my @out = qx{ - sleep .2; - echo 1; - $ENV{'TMUX'} -S $tmpfile new-session -d -n echo 2020$b2020 2>/dev/null && echo 2020; - $ENV{'TMUX'} -S $tmpfile new-session -d -n echo 16320$b16320 2>/dev/null && echo 16320; - $ENV{'TMUX'} -S $tmpfile new-session -d -n echo 100000$b100000 2>/dev/null && echo 100000; - }, - # $len can be > 131072 which will confuse bash, so run it on its own - qx{ - $ENV{'TMUX'} -S $tmpfile new-session -d -n echo $len$blen 2>/dev/null && echo $len; - }; - unlink $tmpfile; + my @out; + for my $l (1, 2020, 16320, 100000, $len) { + my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb"); + close $fh; + $Global::unlink{$tmpfile} = 1; + unlink $tmpfile; + my $tmuxcmd = "sh -c '".$ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l". + ("x"x$l). " 2>/dev/null' && echo $l"; + push @out, qx{ $tmuxcmd }; + } ::debug("tmux","tmux-length ",@out); chomp @out; # The arguments is given 3 times on the command line diff --git a/testsuite/Makefile b/testsuite/Makefile index 752ba8f1..1bac6bd7 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -2,13 +2,19 @@ testsuite: 3 true 3: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote - TRIES=3 time sh Start.sh || true + TRIES=3 time sh Start.sh - mem || true touch ~/.parallel/will-cite date make stopvm 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote - time sh Start.sh || true + time sh Start.sh - mem || true + touch ~/.parallel/will-cite + date + make stopvm + +mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* startdb prereqlocal prereqremote + time sh Start.sh mem NONE || true touch ~/.parallel/will-cite date make stopvm @@ -49,9 +55,10 @@ startvm: VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true stopvm: - VBoxManage controlvm CentOS3-root:centos3 savestate - VBoxManage controlvm RedHat9-root:redhat9 savestate - VBoxManage controlvm OracleXE savestate + # || true - because this should not fail if the VM is not running + VBoxManage controlvm CentOS3-root:centos3 savestate || true + VBoxManage controlvm RedHat9-root:redhat9 savestate || true + VBoxManage controlvm OracleXE savestate || true installparallel: ../src/parallel cd .. && make -j && sudo make -j install diff --git a/testsuite/Start.sh b/testsuite/Start.sh index b21897e1..67caa5cd 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -11,12 +11,14 @@ if [ "$TRIES" = "3" ] ; then # Try a failing test thrice echo Retrying 3 times ls -t tests-to-run/*${1}*.sh | + grep -v ${2} | perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh: ' \ >$SHFILE else # Run a failing test once echo Not retrying ls -t tests-to-run/*${1}*.sh | + grep -v ${2} | perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh:' \ >$SHFILE fi diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index e2225628..e424899f 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -62,10 +62,16 @@ echo '### bug #43817: Some JP char cause problems in positional replacement stri echo '**' +echo '### --rpl % that is a substring of longer --rpl %D' parallel --plus --rpl '%' --rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::' 'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f echo '**' +echo '### Disk full' +cat /dev/zero >/mnt/ram/out; + parallel --tmpdir /mnt/ram echo ::: OK; + rm /mnt/ram/out + EOF diff --git a/testsuite/wanted-results/parallel-local-mem b/testsuite/wanted-results/parallel-local-mem index 7fd94bb6..1c269f38 100644 --- a/testsuite/wanted-results/parallel-local-mem +++ b/testsuite/wanted-results/parallel-local-mem @@ -1,8 +1,8 @@ -make[2]: Entering directory `/home/tange/privat/parallel/testsuite' +make[1]: Entering directory `/home/tange/privat/parallel/testsuite' VBoxManage controlvm CentOS3-root:centos3 savestate VBoxManage controlvm RedHat9-root:redhat9 savestate VBoxManage controlvm OracleXE savestate -make[2]: Leaving directory `/home/tange/privat/parallel/testsuite' +make[1]: Leaving directory `/home/tange/privat/parallel/testsuite' echo '### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails")' ### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails") echo '### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf' @@ -49,7 +49,7 @@ echo '### -L >4GB' parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001 parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003 parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647 -make[2]: Entering directory `/home/tange/privat/parallel/testsuite' +make[1]: Entering directory `/home/tange/privat/parallel/testsuite' # Make sure we can reach the virtual machines sudo ifconfig wlan0:0 192.168.1.72 # If they are already running: Don't fail @@ -59,4 +59,4 @@ VM "CentOS3-root:centos3" has been successfully started. VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true Waiting for VM "RedHat9-root:redhat9" to power on... VM "RedHat9-root:redhat9" has been successfully started. -make[2]: Leaving directory `/home/tange/privat/parallel/testsuite' +make[1]: Leaving directory `/home/tange/privat/parallel/testsuite'