parallel: Better /proc/cpuinfo parser.

This commit is contained in:
Ole Tange 2018-12-02 14:46:50 +01:00
parent 7cc920e738
commit 1ed248a63c
6 changed files with 1923 additions and 15 deletions

View file

@ -15,7 +15,7 @@
# If that fails, it copies to $HOME/bin # If that fails, it copies to $HOME/bin
# #
# You can download and run the script directly by: # You can download and run the script directly by:
# (wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash # (lynx -source pi.dk/3 || wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
run() { run() {
# tail on openindiana must be /usr/xpg4/bin/tail # tail on openindiana must be /usr/xpg4/bin/tail
@ -26,10 +26,11 @@ run() {
(echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep)) (echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep))
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget' # FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
GET=$( GET=$(
(lynx -source /dev/null && echo lynx -source) ||
(fetch -o /dev/null file:///bin/sh && echo fetch -o -) || (fetch -o /dev/null file:///bin/sh && echo fetch -o -) ||
(curl -h >/dev/null && echo curl -L) || (curl -h >/dev/null && echo curl -L) ||
(wget -h >/dev/null && echo wget -qO -) || (wget -h >/dev/null && echo wget -qO -) ||
echo 'No wget, curl, fetch: Please inform parallel@gnu.org what you use for downloading URLs' >&2 echo 'No lynx, wget, curl, fetch: Please inform parallel@gnu.org what you use for downloading URLs' >&2
) )
if test "$GET" = ""; then if test "$GET" = ""; then
exit 1 exit 1

View file

@ -206,7 +206,7 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20181222 ('Kilogram/brexitdeal/Stan Lee/Cesar Sayoc/Tree of Life/Iran') released <<[stable]>> Subject: GNU Parallel 20181222 ('InSight') released <<[stable]>>
GNU Parallel 20181222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20181222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
@ -235,7 +235,8 @@ GNU Parallel makes sure output from the commands is the same output as you would
You can find more about GNU Parallel at: http://www.gnu.org/s/parallel/ You can find more about GNU Parallel at: http://www.gnu.org/s/parallel/
You can install GNU Parallel in just 10 seconds with: (wget -O - pi.dk/3 || curl pi.dk/3/) | bash You can install GNU Parallel in just 10 seconds with:
(wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
Watch the intro video on http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 Watch the intro video on http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1

View file

@ -6575,28 +6575,38 @@ sub sct_gnu_linux() {
# 'active' => #taskset_threads } # 'active' => #taskset_threads }
my $cpu; my $cpu;
local $/ = "\n"; # If delimiter is set, then $/ will be wrong local $/ = "\n"; # If delimiter is set, then $/ will be wrong
if(-e "/proc/cpuinfo") { if($ENV{'PARALLEL_CPUINFO'} or -e "/proc/cpuinfo") {
$cpu->{'sockets'} = 0; $cpu->{'sockets'} = 0;
$cpu->{'cores'} = 0; $cpu->{'cores'} = 0;
$cpu->{'threads'} = 0; $cpu->{'threads'} = 0;
my %seen; my %seen;
my %phy_seen; my %phy_seen;
my @cpuinfo;
my $physicalid;
if(open(my $in_fh, "<", "/proc/cpuinfo")) { if(open(my $in_fh, "<", "/proc/cpuinfo")) {
while(<$in_fh>) { @cpuinfo = <$in_fh>;
if(/^physical id.*[:](.*)/ and not $phy_seen{$1}++) { close $in_fh;
}
if($ENV{'PARALLEL_CPUINFO'}) {
# Use CPUINFO from environment - used for testing only
@cpuinfo = split/(?<=\n)/,$ENV{'PARALLEL_CPUINFO'};
}
for(@cpuinfo) {
if(/^physical id.*[:](.*)/) {
$physicalid=$1;
if(not $phy_seen{$1}++) {
$cpu->{'sockets'}++; $cpu->{'sockets'}++;
} }
if(/^core id.*[:](.*)/ and not $seen{$1}++) { }
if(/^core id.*[:](.*)/ and not $seen{$physicalid,$1}++) {
$cpu->{'cores'}++; $cpu->{'cores'}++;
} }
/^processor.*[:]/i and $cpu->{'threads'}++; /^processor.*[:]/i and $cpu->{'threads'}++;
} }
close $in_fh;
}
$cpu->{'sockets'} ||= 1; $cpu->{'sockets'} ||= 1;
$cpu->{'cores'} ||= $cpu->{'threads'}; $cpu->{'cores'} ||= $cpu->{'threads'};
} }
if(-e "/proc/self/status") { if(-e "/proc/self/status" and not $ENV{'PARALLEL_CPUINFO'}) {
# if 'taskset' is used to limit number of threads # if 'taskset' is used to limit number of threads
if(open(my $in_fh, "<", "/proc/self/status")) { if(open(my $in_fh, "<", "/proc/self/status")) {
while(<$in_fh>) { while(<$in_fh>) {

View file

@ -865,6 +865,7 @@ par_space_envvar() {
export PARALLEL=" -v" && parallel echo ::: 'space in envvar OK' export PARALLEL=" -v" && parallel echo ::: 'space in envvar OK'
} }
export -f $(compgen -A function | grep par_) export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort | compgen -A function | grep par_ | LC_ALL=C sort |
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

File diff suppressed because it is too large Load diff

View file

@ -619,6 +619,12 @@ par_test_XI_mI a7 b1 2 3 4 5 6 7
par_test_XI_mI a8 b1 2 3 4 5 6 7 8 par_test_XI_mI a8 b1 2 3 4 5 6 7 8
par_test_XI_mI a9 b1 2 3 4 5 6 7 8 9 par_test_XI_mI a9 b1 2 3 4 5 6 7 8 9
par_test_XI_mI a10 b1 2 3 4 5 6 7 8 9 10 par_test_XI_mI a10 b1 2 3 4 5 6 7 8 9 10
par_test_cpu_detection 2-8-8-8 2 8 8 8
par_test_cpu_detection 1-4-8-4 1 4 8 4
par_test_cpu_detection 1-2-4-2 1 2 4 2
par_test_cpu_detection 1-2-2-2 1 2 2 2
par_test_cpu_detection 2-24-48-24 2 24 48 24
par_test_cpu_detection 1-2-2-2 1 2 2 2
par_too_long_line_X bug #54869: Long lines break par_too_long_line_X bug #54869: Long lines break
par_too_long_line_X 1 29302 131012 par_too_long_line_X 1 29302 131012
par_too_long_line_X 1 12698 63490 par_too_long_line_X 1 12698 63490