mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: More general --tmux length detection.
This commit is contained in:
Ole Tange
parent
8990c7bb42
commit
815d00ac38
4
README
4
README
|
|
@ -89,8 +89,8 @@ will love you for it.
|
|||
When using programs that use GNU Parallel to process data for
|
||||
publication please cite:
|
||||
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
|
||||
|
||||
= New versions =
|
||||
|
|
|
|||
51
src/parallel
51
src/parallel
|
|
@ -2248,7 +2248,7 @@ sub progress {
|
|||
$last_column_time = time;
|
||||
$columns = $ENV{'COLUMNS'};
|
||||
if(not $columns) {
|
||||
my $resize = qx{ resize 2>/dev/null };
|
||||
my $resize = qx{ sh -c 'resize 2>/dev/null' };
|
||||
$resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; };
|
||||
}
|
||||
$columns ||= 80;
|
||||
|
|
@ -2697,6 +2697,7 @@ sub parallelized_host_filtering {
|
|||
# will make it less likely to overload the ssh daemon.
|
||||
# --retries 3: If the ssh daemon it overloaded, try 3 times
|
||||
# -s 16000: Half of the max line on UnixWare
|
||||
# TODO sh -c wrapper to work in csh
|
||||
my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null";
|
||||
::debug("init", $cmd, "\n");
|
||||
my @out;
|
||||
|
|
@ -4502,6 +4503,7 @@ sub simultaneous_sshlogin {
|
|||
my $sshcmd = $self->sshcommand();
|
||||
my $serverlogin = $self->serverlogin();
|
||||
my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : "";
|
||||
# TODO sh -c wrapper to work for csh
|
||||
my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
|
||||
::debug("init", "Trying $wanted_processes logins at $serverlogin\n");
|
||||
open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or
|
||||
|
|
@ -4717,7 +4719,7 @@ sub no_of_cores {
|
|||
sub nproc {
|
||||
# Returns:
|
||||
# Number of cores using `nproc`
|
||||
my $no_of_cores = `nproc 2>/dev/null`;
|
||||
my $no_of_cores = qx{ sh -c 'nproc 2>/dev/null' };
|
||||
return $no_of_cores;
|
||||
}
|
||||
|
||||
|
|
@ -4792,6 +4794,7 @@ sub no_of_cpus_freebsd {
|
|||
# Returns:
|
||||
# Number of physical CPUs on FreeBSD
|
||||
# undef if not FreeBSD
|
||||
# TODO sh -c wrapper to work in csh
|
||||
my $no_of_cpus =
|
||||
(`sysctl -a dev.cpu 2>/dev/null | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }'`
|
||||
or
|
||||
|
|
@ -4804,6 +4807,7 @@ sub no_of_cores_freebsd {
|
|||
# Returns:
|
||||
# Number of CPU cores on FreeBSD
|
||||
# undef if not FreeBSD
|
||||
# TODO sh -c wrapper to work in csh
|
||||
my $no_of_cores =
|
||||
(`sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'`
|
||||
or
|
||||
|
|
@ -4816,7 +4820,7 @@ sub no_of_cpus_netbsd {
|
|||
# Returns:
|
||||
# Number of physical CPUs on NetBSD
|
||||
# undef if not NetBSD
|
||||
my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`;
|
||||
my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
|
||||
chomp $no_of_cpus;
|
||||
return $no_of_cpus;
|
||||
}
|
||||
|
|
@ -4825,7 +4829,7 @@ sub no_of_cores_netbsd {
|
|||
# Returns:
|
||||
# Number of CPU cores on NetBSD
|
||||
# undef if not NetBSD
|
||||
my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`;
|
||||
my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
|
||||
chomp $no_of_cores;
|
||||
return $no_of_cores;
|
||||
}
|
||||
|
|
@ -4834,7 +4838,7 @@ sub no_of_cpus_openbsd {
|
|||
# Returns:
|
||||
# Number of physical CPUs on OpenBSD
|
||||
# undef if not OpenBSD
|
||||
my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`;
|
||||
my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
|
||||
chomp $no_of_cpus;
|
||||
return $no_of_cpus;
|
||||
}
|
||||
|
|
@ -4843,7 +4847,7 @@ sub no_of_cores_openbsd {
|
|||
# Returns:
|
||||
# Number of CPU cores on OpenBSD
|
||||
# undef if not OpenBSD
|
||||
my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`;
|
||||
my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
|
||||
chomp $no_of_cores;
|
||||
return $no_of_cores;
|
||||
}
|
||||
|
|
@ -4852,7 +4856,7 @@ sub no_of_cpus_hurd {
|
|||
# Returns:
|
||||
# Number of physical CPUs on HURD
|
||||
# undef if not HURD
|
||||
my $no_of_cpus = `nproc`;
|
||||
my $no_of_cpus = qx{ nproc };
|
||||
chomp $no_of_cpus;
|
||||
return $no_of_cpus;
|
||||
}
|
||||
|
|
@ -4870,6 +4874,7 @@ sub no_of_cpus_darwin {
|
|||
# Returns:
|
||||
# Number of physical CPUs on Mac Darwin
|
||||
# undef if not Mac Darwin
|
||||
# TODO sh -c wrapper to work in csh
|
||||
my $no_of_cpus =
|
||||
(`sysctl -n hw.physicalcpu 2>/dev/null`
|
||||
or
|
||||
|
|
@ -4881,6 +4886,7 @@ sub no_of_cores_darwin {
|
|||
# Returns:
|
||||
# Number of CPU cores on Mac Darwin
|
||||
# undef if not Mac Darwin
|
||||
# TODO sh -c wrapper to work in csh
|
||||
my $no_of_cores =
|
||||
(`sysctl -n hw.logicalcpu 2>/dev/null`
|
||||
or
|
||||
|
|
@ -4961,7 +4967,7 @@ sub no_of_cpus_hpux {
|
|||
# Number of physical CPUs on HP-UX
|
||||
# undef if not HP-UX
|
||||
my $no_of_cpus =
|
||||
qx{/usr/bin/mpsched -s 2>&1 | grep 'Locality Domain Count' | awk '{ print \$4 }'};
|
||||
qx{ sh -c '/usr/bin/mpsched -s 2>&1' | grep 'Locality Domain Count' | awk '{ print \$4 }'};
|
||||
return $no_of_cpus;
|
||||
}
|
||||
|
||||
|
|
@ -4970,7 +4976,7 @@ sub no_of_cores_hpux {
|
|||
# Number of CPU cores on HP-UX
|
||||
# undef if not HP-UX
|
||||
my $no_of_cores =
|
||||
qx{/usr/bin/mpsched -s 2>&1 | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'};
|
||||
qx{ sh -c '/usr/bin/mpsched -s 2>&1' | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'};
|
||||
return $no_of_cores;
|
||||
}
|
||||
|
||||
|
|
@ -5188,7 +5194,7 @@ sub cleanup_cmd {
|
|||
$dir .= $_."/";
|
||||
unshift @rmdir, ::shell_quote_file($dir);
|
||||
}
|
||||
my $rmdir = @rmdir ? "rmdir @rmdir >&/dev/null;" : "";
|
||||
my $rmdir = @rmdir ? "sh -c 'rmdir @rmdir 2>/dev/null';" : "";
|
||||
if(defined $opt::workdir and $opt::workdir eq "...") {
|
||||
$rmdir .= "rm -rf " . ::shell_quote_file($workdir).';';
|
||||
}
|
||||
|
|
@ -6697,6 +6703,7 @@ sub print_dryrun_and_verbose {
|
|||
unlink $tmuxsocket;
|
||||
::status("See output with: $ENV{'TMUX'} -S $tmuxsocket attach\n");
|
||||
}
|
||||
# TODO sh -c wrapper for >&
|
||||
$tmux = $ENV{'TMUX'}." -S $tmuxsocket new-session -s p$$ -d 'sleep .2' >&/dev/null;" .
|
||||
$ENV{'TMUX'}." -S $tmuxsocket new-window -t p$$ -n $title";
|
||||
|
||||
|
|
@ -8112,26 +8119,16 @@ sub tmux_length {
|
|||
my $len = shift;
|
||||
if($opt::tmux) {
|
||||
$ENV{'TMUX'} ||= "tmux";
|
||||
my @out;
|
||||
for my $l (1, 2020, 16320, 100000, $len) {
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb");
|
||||
$Global::unlink{$tmpfile}=1;
|
||||
close $fh;
|
||||
$Global::unlink{$tmpfile} = 1;
|
||||
unlink $tmpfile;
|
||||
my $b2020 = "x"x2020;
|
||||
my $b16320 = "x"x16320;
|
||||
my $b100000 = "x"x100000;
|
||||
my $blen = "x"x$len;
|
||||
my @out = qx{
|
||||
sleep .2;
|
||||
echo 1;
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 2020$b2020 2>/dev/null && echo 2020;
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 16320$b16320 2>/dev/null && echo 16320;
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 100000$b100000 2>/dev/null && echo 100000;
|
||||
},
|
||||
# $len can be > 131072 which will confuse bash, so run it on its own
|
||||
qx{
|
||||
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo $len$blen 2>/dev/null && echo $len;
|
||||
};
|
||||
unlink $tmpfile;
|
||||
my $tmuxcmd = "sh -c '".$ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l".
|
||||
("x"x$l). " 2>/dev/null' && echo $l";
|
||||
push @out, qx{ $tmuxcmd };
|
||||
}
|
||||
::debug("tmux","tmux-length ",@out);
|
||||
chomp @out;
|
||||
# The arguments is given 3 times on the command line
|
||||
|
|
|
|||
|
|
@ -2,13 +2,19 @@ testsuite: 3
|
|||
true
|
||||
|
||||
3: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
|
||||
TRIES=3 time sh Start.sh || true
|
||||
TRIES=3 time sh Start.sh - mem || true
|
||||
touch ~/.parallel/will-cite
|
||||
date
|
||||
make stopvm
|
||||
|
||||
1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
|
||||
time sh Start.sh || true
|
||||
time sh Start.sh - mem || true
|
||||
touch ~/.parallel/will-cite
|
||||
date
|
||||
make stopvm
|
||||
|
||||
mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* startdb prereqlocal prereqremote
|
||||
time sh Start.sh mem NONE || true
|
||||
touch ~/.parallel/will-cite
|
||||
date
|
||||
make stopvm
|
||||
|
|
@ -49,9 +55,10 @@ startvm:
|
|||
VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true
|
||||
|
||||
stopvm:
|
||||
VBoxManage controlvm CentOS3-root:centos3 savestate
|
||||
VBoxManage controlvm RedHat9-root:redhat9 savestate
|
||||
VBoxManage controlvm OracleXE savestate
|
||||
# || true - because this should not fail if the VM is not running
|
||||
VBoxManage controlvm CentOS3-root:centos3 savestate || true
|
||||
VBoxManage controlvm RedHat9-root:redhat9 savestate || true
|
||||
VBoxManage controlvm OracleXE savestate || true
|
||||
|
||||
installparallel: ../src/parallel
|
||||
cd .. && make -j && sudo make -j install
|
||||
|
|
|
|||
|
|
@ -11,12 +11,14 @@ if [ "$TRIES" = "3" ] ; then
|
|||
# Try a failing test thrice
|
||||
echo Retrying 3 times
|
||||
ls -t tests-to-run/*${1}*.sh |
|
||||
grep -v ${2} |
|
||||
perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh: ' \
|
||||
>$SHFILE
|
||||
else
|
||||
# Run a failing test once
|
||||
echo Not retrying
|
||||
ls -t tests-to-run/*${1}*.sh |
|
||||
grep -v ${2} |
|
||||
perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh:' \
|
||||
>$SHFILE
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -62,10 +62,16 @@ echo '### bug #43817: Some JP char cause problems in positional replacement stri
|
|||
|
||||
echo '**'
|
||||
|
||||
echo '### --rpl % that is a substring of longer --rpl %D'
|
||||
parallel --plus --rpl '%'
|
||||
--rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::'
|
||||
'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f
|
||||
|
||||
echo '**'
|
||||
|
||||
echo '### Disk full'
|
||||
cat /dev/zero >/mnt/ram/out;
|
||||
parallel --tmpdir /mnt/ram echo ::: OK;
|
||||
rm /mnt/ram/out
|
||||
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
make[2]: Entering directory `/home/tange/privat/parallel/testsuite'
|
||||
make[1]: Entering directory `/home/tange/privat/parallel/testsuite'
|
||||
VBoxManage controlvm CentOS3-root:centos3 savestate
|
||||
VBoxManage controlvm RedHat9-root:redhat9 savestate
|
||||
VBoxManage controlvm OracleXE savestate
|
||||
make[2]: Leaving directory `/home/tange/privat/parallel/testsuite'
|
||||
make[1]: Leaving directory `/home/tange/privat/parallel/testsuite'
|
||||
echo '### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails")'
|
||||
### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails")
|
||||
echo '### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf'
|
||||
|
|
@ -49,7 +49,7 @@ echo '### -L >4GB'
|
|||
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001
|
||||
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003
|
||||
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647
|
||||
make[2]: Entering directory `/home/tange/privat/parallel/testsuite'
|
||||
make[1]: Entering directory `/home/tange/privat/parallel/testsuite'
|
||||
# Make sure we can reach the virtual machines
|
||||
sudo ifconfig wlan0:0 192.168.1.72
|
||||
# If they are already running: Don't fail
|
||||
|
|
@ -59,4 +59,4 @@ VM "CentOS3-root:centos3" has been successfully started.
|
|||
VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true
|
||||
Waiting for VM "RedHat9-root:redhat9" to power on...
|
||||
VM "RedHat9-root:redhat9" has been successfully started.
|
||||
make[2]: Leaving directory `/home/tange/privat/parallel/testsuite'
|
||||
make[1]: Leaving directory `/home/tange/privat/parallel/testsuite'
|
||||
|
|
|
|||
Loading…
Reference in a new issue