[Misc][NUMA] Auto-bind to PCT priority cores on DGX B300 + widen EngineCore across shard NUMA nodes (#43270)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Cursor <noreply@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
1521173c17
commit
bf18d7e0b4
+282
-32
@@ -14,7 +14,7 @@ import subprocess
|
||||
from contextlib import contextmanager
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
import psutil
|
||||
|
||||
@@ -112,6 +112,117 @@ def get_auto_numa_nodes() -> list[int] | None:
|
||||
return numa_nodes
|
||||
|
||||
|
||||
# PCT (Priority Core Turbo) auto-detection workaround for Granite Rapids
|
||||
# Xeon SKUs.
|
||||
#
|
||||
# Background:
|
||||
# * The Linux kernel does not expose PCT priority-core membership via any
|
||||
# unprivileged sysfs path. The official interface
|
||||
# (/dev/isst_interface, used by `intel-speed-select`) is root-only,
|
||||
# which is a non-starter in most production deployments (shared
|
||||
# clusters, prebuilt containers, managed cloud).
|
||||
# * Even recent stable kernels (e.g. 6.14, March 2025) do not yet
|
||||
# preferentially schedule work on PCT priority cores, so vLLM cannot
|
||||
# just "let the scheduler handle it".
|
||||
#
|
||||
# Empirical heuristic (DGX B300 / Xeon 6776P, the SKU we measured):
|
||||
# * /proc/cpuinfo `model name` contains the SKU number.
|
||||
# * cpu0 is a PCT priority core on these SKUs, so it reports the
|
||||
# priority-cohort CPPC `highest_perf` (the value matches the SKU's
|
||||
# "Max PCT core frequency" in 100 MHz units, e.g. 4.6 GHz -> 46).
|
||||
# * Priority cores within each NUMA node satisfy `cpu_id % S in (0, 1)`
|
||||
# intersected with the node's cpulist, where `S` is the SKU's logical
|
||||
# CPUs per priority "group" (= total threads / 8 priority cores; 16 on
|
||||
# 64-core SKUs, 18 on 72-core SKUs).
|
||||
#
|
||||
# SKU table:
|
||||
# ``_PCT_CAPABLE_SKUS`` maps each known PCT-capable Granite Rapids part
|
||||
# to a ``_PctSku(highest_perf, priority_stride)`` config:
|
||||
# * highest_perf is the expected ``acpi_cppc/highest_perf`` on cpu0,
|
||||
# derived from Intel ARK's "Max PCT core frequency" * 10 (CPPC max
|
||||
# ratio reports in 100 MHz units).
|
||||
# * priority_stride is the SKU's "Total Cores" / 4 (= total HT threads
|
||||
# / 8 priority cores), used in the ``cpu_id % stride`` filter above.
|
||||
# Values:
|
||||
# * 6776P - 4.6 GHz, 64C/128T -> (46, 16) measured on DGX B300
|
||||
# * 6774P - 4.6 GHz, 64C/128T -> (46, 16) per Intel ARK, not measured
|
||||
# * 6962P - 4.4 GHz, 72C/144T -> (44, 18) per Intel ARK, not measured
|
||||
# The non-measured SKUs are listed best-effort: the gate fails closed
|
||||
# (no PCT engagement) if a host's actual highest_perf doesn't match the
|
||||
# table value, so adding entries is safe. If you have access to a 6962P
|
||||
# or 6774P box and find a different value or cpu-id pattern, update the
|
||||
# table below.
|
||||
#
|
||||
# This whole block is a stop-gap until the kernel exposes PCT membership
|
||||
# in an unprivileged way; see the tracking issue linked from the PR.
|
||||
|
||||
|
||||
class _PctSku(NamedTuple):
|
||||
"""Per-SKU config used by the PCT auto-detection gate."""
|
||||
|
||||
highest_perf: int
|
||||
priority_stride: int
|
||||
|
||||
|
||||
_PCT_CAPABLE_SKUS: dict[str, _PctSku] = {
|
||||
"6776P": _PctSku(highest_perf=46, priority_stride=16),
|
||||
"6774P": _PctSku(highest_perf=46, priority_stride=16),
|
||||
"6962P": _PctSku(highest_perf=44, priority_stride=18),
|
||||
}
|
||||
_PCT_HIGHEST_PERF_PATH = "/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf"
|
||||
_PROC_CPUINFO_PATH = "/proc/cpuinfo"
|
||||
|
||||
|
||||
def _pct_sku_from_cpuinfo() -> _PctSku | None:
|
||||
"""Return the ``_PctSku`` config for this host's SKU, or None.
|
||||
|
||||
Reads ``/proc/cpuinfo``'s ``model name`` and looks the SKU up in
|
||||
``_PCT_CAPABLE_SKUS``. Returns ``None`` when the host is not a known
|
||||
PCT-capable Granite Rapids Xeon (or when ``/proc/cpuinfo`` is
|
||||
unreadable).
|
||||
"""
|
||||
try:
|
||||
with open(_PROC_CPUINFO_PATH) as f:
|
||||
for line in f:
|
||||
if not line.lstrip().lower().startswith("model name"):
|
||||
continue
|
||||
for sku, config in _PCT_CAPABLE_SKUS.items():
|
||||
if sku in line:
|
||||
return config
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
@cache
|
||||
def _pct_sku_config() -> _PctSku | None:
|
||||
"""Detect a PCT-capable Granite Rapids Xeon with PCT enabled.
|
||||
|
||||
See the comment block above ``_PCT_CAPABLE_SKUS`` for the full context
|
||||
(why we hard-code SKUs, why we read CPPC ``highest_perf``, etc.).
|
||||
|
||||
Returns the matching ``_PctSku`` config when both gates hold:
|
||||
* ``/proc/cpuinfo`` ``model name`` contains an SKU listed in
|
||||
``_PCT_CAPABLE_SKUS``.
|
||||
* ``/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf`` matches
|
||||
that SKU's expected ``highest_perf``.
|
||||
Otherwise returns ``None`` and the caller falls back to the default
|
||||
NUMA-node bind.
|
||||
"""
|
||||
sku = _pct_sku_from_cpuinfo()
|
||||
if sku is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(_PCT_HIGHEST_PERF_PATH) as f:
|
||||
actual = int(f.read().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if actual != sku.highest_perf:
|
||||
return None
|
||||
return sku
|
||||
|
||||
|
||||
def _get_gpu_index(
|
||||
parallel_config, local_rank: int, dp_local_rank: int | None = None
|
||||
) -> int:
|
||||
@@ -156,55 +267,183 @@ def _get_numa_node(parallel_config, gpu_index: int) -> int:
|
||||
return numa_nodes[gpu_index]
|
||||
|
||||
|
||||
def _get_cpu_binding(parallel_config, gpu_index: int) -> str | None:
|
||||
def _maybe_get_pct_cpu_binding(numa_nodes: list[int]) -> list[int] | None:
|
||||
"""Return the union of PCT priority cores across ``numa_nodes`` (or None).
|
||||
|
||||
PCT (Priority Core Turbo) lets a subset of cores boost above the rest;
|
||||
we want workers and the EngineCore on those cores. The Linux kernel does
|
||||
not expose PCT membership without root, so we use the empirical heuristic
|
||||
documented above ``_PCT_CAPABLE_SKUS``: priority cores within each NUMA
|
||||
node satisfy ``cpu_id % stride in (0, 1)`` intersected with the node's
|
||||
``cpulist``, where ``stride`` is the SKU's logical CPUs per priority
|
||||
group (16 on 64-core SKUs, 18 on 72-core SKUs). Only triggers on the
|
||||
SKUs in ``_PCT_CAPABLE_SKUS`` with the expected CPPC ``highest_perf``
|
||||
signal; on any other host it returns None and the caller falls back to
|
||||
the default NUMA-node bind.
|
||||
|
||||
Returns the sorted CPU ids as a ``list[int]``; the caller is expected
|
||||
to format them for the chosen tool (e.g. comma-joined for
|
||||
``numactl --physcpubind``).
|
||||
"""
|
||||
sku = _pct_sku_config()
|
||||
if sku is None:
|
||||
return None
|
||||
|
||||
from vllm.utils.cpu_resource_utils import parse_id_list
|
||||
|
||||
stride = sku.priority_stride
|
||||
union_cpus: set[int] = set()
|
||||
for numa_node in numa_nodes:
|
||||
cpulist_path = Path(f"/sys/devices/system/node/node{numa_node}/cpulist")
|
||||
try:
|
||||
cpulist_raw = cpulist_path.read_text().strip()
|
||||
except OSError:
|
||||
continue
|
||||
if not cpulist_raw:
|
||||
continue
|
||||
try:
|
||||
node_cpus = parse_id_list(cpulist_raw)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
priority = [cpu for cpu in node_cpus if cpu % stride in (0, 1)]
|
||||
if not priority:
|
||||
continue
|
||||
union_cpus.update(priority)
|
||||
logger.info(
|
||||
"Detected PCT-capable Granite Rapids Xeon (stride=%d); "
|
||||
"NUMA node %d priority cores: %s",
|
||||
stride,
|
||||
numa_node,
|
||||
",".join(str(c) for c in priority),
|
||||
)
|
||||
|
||||
if not union_cpus:
|
||||
return None
|
||||
return sorted(union_cpus)
|
||||
|
||||
|
||||
def _get_cpu_binding(
|
||||
parallel_config, gpu_index: int, numa_nodes: list[int]
|
||||
) -> str | None:
|
||||
"""Return the CPU list a process should be pinned to (or None)."""
|
||||
cpu_bindings = parallel_config.numa_bind_cpus
|
||||
if cpu_bindings is None:
|
||||
return None
|
||||
pct_cpus = _maybe_get_pct_cpu_binding(numa_nodes)
|
||||
if pct_cpus is None:
|
||||
return None
|
||||
return ",".join(str(c) for c in pct_cpus)
|
||||
|
||||
if gpu_index >= len(cpu_bindings):
|
||||
raise ValueError(
|
||||
f"GPU index {gpu_index} exceeds numa_bind_cpus size "
|
||||
f"{len(cpu_bindings)}. Ensure the binding lists cover every visible GPU."
|
||||
)
|
||||
|
||||
return cpu_bindings[gpu_index]
|
||||
|
||||
|
||||
def _get_numactl_args(
|
||||
vllm_config: "VllmConfig",
|
||||
local_rank: int,
|
||||
dp_local_rank: int | None = None,
|
||||
process_kind: str = "worker",
|
||||
) -> str | None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not parallel_config.numa_bind:
|
||||
return None
|
||||
|
||||
def _get_numactl_worker_args(
|
||||
parallel_config, local_rank: int, dp_local_rank: int | None = None
|
||||
) -> str:
|
||||
"""Compute the numactl args for a single TP/PP worker subprocess."""
|
||||
gpu_index = _get_gpu_index(parallel_config, local_rank, dp_local_rank)
|
||||
numa_node = _get_numa_node(parallel_config, gpu_index)
|
||||
cpu_binding = _get_cpu_binding(parallel_config, gpu_index)
|
||||
cpu_binding = _get_cpu_binding(parallel_config, gpu_index, [numa_node])
|
||||
|
||||
if cpu_binding is not None:
|
||||
bind_arg = f"--physcpubind={cpu_binding}"
|
||||
logger.info(
|
||||
"Binding %s subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
|
||||
process_kind,
|
||||
"Binding worker subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
|
||||
local_rank,
|
||||
gpu_index,
|
||||
cpu_binding,
|
||||
numa_node,
|
||||
)
|
||||
else:
|
||||
bind_arg = f"--cpunodebind={numa_node}"
|
||||
logger.info(
|
||||
"Binding %s subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
|
||||
process_kind,
|
||||
local_rank,
|
||||
gpu_index,
|
||||
numa_node,
|
||||
)
|
||||
return f"--physcpubind={cpu_binding} --membind={numa_node}"
|
||||
|
||||
return f"{bind_arg} --membind={numa_node}"
|
||||
logger.info(
|
||||
"Binding worker subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
|
||||
local_rank,
|
||||
gpu_index,
|
||||
numa_node,
|
||||
)
|
||||
return f"--cpunodebind={numa_node} --membind={numa_node}"
|
||||
|
||||
|
||||
def _get_enginecore_numa_nodes(
|
||||
parallel_config, dp_local_rank: int | None = None
|
||||
) -> list[int]:
|
||||
"""Return the sorted, unique NUMA nodes of the EngineCore's DP shard."""
|
||||
numa_nodes = parallel_config.numa_bind_nodes
|
||||
if numa_nodes is None:
|
||||
# Trigger auto-detection (it caches into parallel_config).
|
||||
_get_numa_node(parallel_config, 0)
|
||||
numa_nodes = parallel_config.numa_bind_nodes
|
||||
|
||||
if (
|
||||
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
):
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = parallel_config.data_parallel_rank_local
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = parallel_config.data_parallel_index
|
||||
|
||||
tp_pp_world_size = (
|
||||
parallel_config.pipeline_parallel_size
|
||||
* parallel_config.tensor_parallel_size
|
||||
)
|
||||
shard_start = dp_local_rank * tp_pp_world_size
|
||||
shard_end = min(shard_start + tp_pp_world_size, len(numa_nodes))
|
||||
shard_indices: range | tuple[int, ...] = range(shard_start, shard_end)
|
||||
else:
|
||||
shard_indices = range(len(numa_nodes))
|
||||
|
||||
if not shard_indices:
|
||||
return [numa_nodes[0]]
|
||||
return sorted({numa_nodes[i] for i in shard_indices})
|
||||
|
||||
|
||||
def _get_numactl_enginecore_args(
|
||||
parallel_config, local_rank: int, dp_local_rank: int | None = None
|
||||
) -> str:
|
||||
"""Compute the numactl args for an EngineCore subprocess.
|
||||
|
||||
``--numa-bind-cpus`` is deliberately ignored here: the user provides a
|
||||
per-worker CPU list, and binding EngineCore to any of those entries
|
||||
would shrink its ``cpus_allowed`` below the strict-superset that the
|
||||
workers' ``--physcpubind`` spawns require. We fall back to
|
||||
``--cpunodebind=<shard nodes>`` instead, which is always a safe
|
||||
superset. PCT auto-detection still applies when the user did not pass
|
||||
``--numa-bind-cpus`` (its priority-core union across the shard nodes
|
||||
is also a safe superset by construction).
|
||||
"""
|
||||
shard_nodes = _get_enginecore_numa_nodes(parallel_config, dp_local_rank)
|
||||
membind_arg = ",".join(str(n) for n in shard_nodes)
|
||||
|
||||
pct_cpus = (
|
||||
None
|
||||
if parallel_config.numa_bind_cpus is not None
|
||||
else _maybe_get_pct_cpu_binding(shard_nodes)
|
||||
)
|
||||
|
||||
if pct_cpus is not None:
|
||||
cpu_binding = ",".join(str(c) for c in pct_cpus)
|
||||
logger.info(
|
||||
"Binding EngineCore subprocess (local_rank=%s) to CPUs %s "
|
||||
"and NUMA nodes %s",
|
||||
local_rank,
|
||||
cpu_binding,
|
||||
membind_arg,
|
||||
)
|
||||
return f"--physcpubind={cpu_binding} --membind={membind_arg}"
|
||||
|
||||
logger.info(
|
||||
"Binding EngineCore subprocess (local_rank=%s) to NUMA nodes %s",
|
||||
local_rank,
|
||||
membind_arg,
|
||||
)
|
||||
return f"--cpunodebind={membind_arg} --membind={membind_arg}"
|
||||
|
||||
|
||||
def _log_numactl_show(label: str) -> bool:
|
||||
@@ -242,13 +481,24 @@ def configure_subprocess(
|
||||
process_kind: str = "worker",
|
||||
):
|
||||
"""Temporarily replace the multiprocessing executable with a numactl wrapper."""
|
||||
numactl_args = _get_numactl_args(
|
||||
vllm_config, local_rank, dp_local_rank, process_kind
|
||||
)
|
||||
if numactl_args is None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not parallel_config.numa_bind:
|
||||
yield
|
||||
return
|
||||
|
||||
if process_kind == "EngineCore":
|
||||
numactl_args = _get_numactl_enginecore_args(
|
||||
parallel_config, local_rank, dp_local_rank
|
||||
)
|
||||
elif process_kind == "worker":
|
||||
numactl_args = _get_numactl_worker_args(
|
||||
parallel_config, local_rank, dp_local_rank
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown process_kind {process_kind!r}; expected 'worker' or 'EngineCore'."
|
||||
)
|
||||
|
||||
executable, debug_str = _get_numactl_executable()
|
||||
python_executable = os.fsdecode(multiprocessing.spawn.get_executable())
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user