[Misc][NUMA] Auto-bind to PCT priority cores on DGX B300 + widen EngineCore across shard NUMA nodes (#43270)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Co-authored-by: Cursor <noreply@cursor.com>
This commit is contained in:
Vadim Gimpelson
2026-05-29 10:07:44 +08:00
committed by GitHub
co-authored by Cursor
parent 1521173c17
commit bf18d7e0b4
3 changed files with 626 additions and 37 deletions
+338 -4
View File
@@ -10,6 +10,33 @@ from vllm.config import ParallelConfig
from vllm.utils import numa_utils
@pytest.fixture(autouse=True)
def _disable_pct_by_default(monkeypatch):
"""Force PCT detection OFF unless a test opts in via ``_patch_pct_gates``.
The CI / dev machines themselves can be Xeon 6776P with PCT enabled, so a
plain ``cache_clear`` would let the gate auto-detect ``True`` from the
live filesystem and silently re-route "baseline" tests through the PCT
path. Stub ``/proc/cpuinfo`` and ``acpi_cppc/highest_perf`` to a state
that fails the gate; ``_patch_pct_gates`` re-stubs on top when needed.
"""
from io import StringIO
real_open = open
def _no_pct_open(path, *args, **kwargs):
if path == numa_utils._PROC_CPUINFO_PATH:
return StringIO("processor\t: 0\nmodel name\t: Generic Test CPU\n")
if path == numa_utils._PCT_HIGHEST_PERF_PATH:
raise OSError("PCT disabled by autouse fixture")
return real_open(path, *args, **kwargs)
monkeypatch.setattr("builtins.open", _no_pct_open)
numa_utils._pct_sku_config.cache_clear()
yield
numa_utils._pct_sku_config.cache_clear()
def _make_config(**parallel_kwargs):
parallel_defaults = dict(
numa_bind=False,
@@ -31,7 +58,7 @@ def _make_config(**parallel_kwargs):
def test_get_numactl_args_with_node_binding():
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
assert (
numa_utils._get_numactl_args(vllm_config, local_rank=1)
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
== "--cpunodebind=1 --membind=1"
)
@@ -43,7 +70,300 @@ def test_get_numactl_args_with_cpu_binding():
numa_bind_cpus=["0-3", "4-7"],
)
assert (
numa_utils._get_numactl_args(vllm_config, local_rank=1)
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
== "--physcpubind=4-7 --membind=1"
)
def _patch_pct_gates(
monkeypatch,
*,
model_match: bool,
highest_perf: int | None,
cpulist: str | None = "0-31,64-95",
cpulist_by_node: dict[int, str | None] | None = None,
sku: str = "6776P",
):
"""Force `_pct_sku_config` and node cpulist read to deterministic state.
``cpulist`` is the default returned for any node not present in
``cpulist_by_node``. ``cpulist_by_node`` lets a test return different
cpulists for different NUMA nodes (e.g. node 0 vs node 1). ``sku`` lets
the test pick which Granite Rapids SKU appears in the fake
``/proc/cpuinfo`` ``model name`` (only used when ``model_match=True``).
"""
import pathlib
from io import StringIO
import regex as re
cpuinfo = (
f"processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Platinum {sku} CPU @ 2.40GHz\n"
if model_match
else "processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Platinum 8480+\n"
)
real_open = open
def fake_open(path, *args, **kwargs):
if path == numa_utils._PROC_CPUINFO_PATH:
return StringIO(cpuinfo)
if path == numa_utils._PCT_HIGHEST_PERF_PATH:
if highest_perf is None:
raise OSError("missing")
return StringIO(f"{highest_perf}\n")
return real_open(path, *args, **kwargs)
real_read_text = pathlib.Path.read_text
cpulist_by_node = cpulist_by_node or {}
def fake_read_text(self, *args, **kwargs):
path_str = str(self)
if path_str.endswith("/cpulist") and "/sys/devices/system/node" in path_str:
match = re.search(r"/node(\d+)/cpulist$", path_str)
if match:
node_id = int(match.group(1))
if node_id in cpulist_by_node:
val = cpulist_by_node[node_id]
if val is None:
raise OSError(f"missing cpulist for node{node_id}")
return val
if cpulist is None:
raise OSError("missing cpulist")
return cpulist
return real_read_text(self, *args, **kwargs)
monkeypatch.setattr("builtins.open", fake_open)
monkeypatch.setattr("pathlib.Path.read_text", fake_read_text)
numa_utils._pct_sku_config.cache_clear()
def test_pct_binding_filters_cpus(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
assert numa_utils._maybe_get_pct_cpu_binding([0]) == [0, 1, 16, 17, 64, 65, 80, 81]
@pytest.mark.parametrize(
"sku,expected_cpus",
[
# 64-core SKUs (stride 16): cpus from "0-31,64-95" with cpu_id % 16
# in (0, 1) -> 0, 1, 16, 17, 64, 65, 80, 81.
("6776P", [0, 1, 16, 17, 64, 65, 80, 81]),
("6774P", [0, 1, 16, 17, 64, 65, 80, 81]),
# 72-core SKU (stride 18): cpus from "0-31,64-95" with cpu_id % 18
# in (0, 1) -> 0, 1, 18, 19, 72, 73, 90, 91.
("6962P", [0, 1, 18, 19, 72, 73, 90, 91]),
],
)
def test_pct_binding_fires_on_every_capable_sku(monkeypatch, sku, expected_cpus):
"""Each SKU in ``_PCT_CAPABLE_SKUS`` engages the gate at its own
expected ``highest_perf`` and uses its own priority-core stride."""
sku_config = numa_utils._PCT_CAPABLE_SKUS[sku]
_patch_pct_gates(
monkeypatch,
model_match=True,
highest_perf=sku_config.highest_perf,
sku=sku,
)
assert numa_utils._maybe_get_pct_cpu_binding([0]) == expected_cpus
def test_pct_binding_fails_closed_when_sku_perf_mismatch(monkeypatch):
"""6962P with 6776P's highest_perf (46 vs expected 44) must fail closed."""
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46, sku="6962P")
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
def test_pct_binding_disabled_when_cpu_model_mismatch(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=False, highest_perf=46)
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
def test_pct_binding_disabled_when_highest_perf_does_not_match(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=42)
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
def test_pct_binding_disabled_when_files_missing(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=None)
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
def test_pct_binding_returns_none_when_node_cpulist_filter_empty(monkeypatch):
_patch_pct_gates(
monkeypatch,
model_match=True,
highest_perf=46,
cpulist="2-15,18-31",
)
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
def test_pct_binding_returns_none_when_node_cpulist_missing(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46, cpulist=None)
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
def test_get_numactl_args_uses_pct_when_user_did_not_specify_cpus(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
assert (
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
== "--physcpubind=0,1,16,17,64,65,80,81 --membind=1"
)
def test_get_numactl_args_engine_core_baseline_single_node_shard():
"""Baseline (no PCT): single-NUMA shard -> single-node bind."""
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
assert (
numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
)
== "--cpunodebind=0 --membind=0"
)
def test_get_numactl_args_engine_core_baseline_spans_shard_numa_nodes():
"""Baseline (no PCT): a TP=4 shard spanning both NUMA nodes -> bind to both."""
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 0, 1, 1],
tensor_parallel_size=4,
)
assert (
numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
)
== "--cpunodebind=0,1 --membind=0,1"
)
def test_get_numactl_args_engine_core_pct_spans_shard_numa_nodes(monkeypatch):
"""PCT: EngineCore for a multi-NUMA shard binds to the union of priority
cores across all shard nodes, so worker `--physcpubind` is always a
subset of EngineCore's `cpus_allowed`."""
_patch_pct_gates(
monkeypatch,
model_match=True,
highest_perf=46,
cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
)
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 0, 1, 1],
tensor_parallel_size=4,
)
assert numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
) == (
"--physcpubind="
"0,1,16,17,64,65,80,81,128,129,144,145,192,193,208,209"
" --membind=0,1"
)
def test_get_numactl_args_engine_core_pct_dp_shard_picks_local_nodes(monkeypatch):
"""With DP=2, each shard's EngineCore binds only to its own NUMA nodes."""
_patch_pct_gates(
monkeypatch,
model_match=True,
highest_perf=46,
cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
)
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 0, 1, 1],
tensor_parallel_size=2,
data_parallel_rank_local=1,
)
# Shard 1 owns gpu_indices 2 and 3 -> nodes [1, 1] -> {1}.
assert (
numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
)
== "--physcpubind=64,65,80,81,192,193,208,209 --membind=1"
)
def test_get_numactl_args_engine_core_pct_external_launcher_spans_local_nodes(
monkeypatch,
):
"""external_launcher (or multi-node-within-DP, or Ray) hits the
fallback branch. EngineCore must still span every local NUMA node
so it can mp-spawn its local workers without ``--physcpubind``
strict-validation failures."""
_patch_pct_gates(
monkeypatch,
model_match=True,
highest_perf=46,
cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
)
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 0, 0, 0, 1, 1, 1, 1],
distributed_executor_backend="external_launcher",
tensor_parallel_size=8,
)
assert numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
) == (
"--physcpubind="
"0,1,16,17,64,65,80,81,128,129,144,145,192,193,208,209"
" --membind=0,1"
)
def test_get_numactl_args_engine_core_baseline_multi_node_within_dp_spans_locals():
"""Multi-node-within-DP fallback: bind EngineCore to all local NUMA
nodes that the visible ``numa_bind_nodes`` reference."""
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 0, 0, 0, 1, 1, 1, 1],
nnodes_within_dp=2,
tensor_parallel_size=8,
)
assert (
numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
)
== "--cpunodebind=0,1 --membind=0,1"
)
def test_get_numactl_args_engine_core_skips_user_cpu_list(monkeypatch):
"""EngineCore ignores ``--numa-bind-cpus``.
Those are per-worker lists; binding EngineCore to any of them would
shrink its ``cpus_allowed`` below the strict-superset workers'
``--physcpubind`` spawns need. We fall back to ``--cpunodebind`` over
the shard's NUMA nodes instead. PCT auto-detect is also bypassed when
the user is explicit (its priority-core union may not be a superset
of the user's per-worker cores)."""
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 0, 1, 1],
numa_bind_cpus=["0-3", "4-7", "64-67", "68-71"],
tensor_parallel_size=4,
)
assert (
numa_utils._get_numactl_enginecore_args(
vllm_config.parallel_config, local_rank=0
)
== "--cpunodebind=0,1 --membind=0,1"
)
def test_get_numactl_args_user_cpus_override_pct(monkeypatch):
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
vllm_config = _make_config(
numa_bind=True,
numa_bind_nodes=[0, 1],
numa_bind_cpus=["0-3", "4-7"],
)
assert (
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
== "--physcpubind=4-7 --membind=1"
)
@@ -57,7 +377,7 @@ def test_get_numactl_args_uses_dp_offset():
tensor_parallel_size=2,
)
assert (
numa_utils._get_numactl_args(vllm_config, local_rank=1)
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
== "--cpunodebind=1 --membind=1"
)
@@ -66,7 +386,21 @@ def test_get_numactl_args_requires_detectable_nodes(monkeypatch):
vllm_config = _make_config(numa_bind=True)
monkeypatch.setattr(numa_utils, "get_auto_numa_nodes", lambda: None)
with pytest.raises(RuntimeError):
numa_utils._get_numactl_args(vllm_config, local_rank=0)
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=0)
def test_configure_subprocess_rejects_unknown_process_kind():
"""configure_subprocess only knows 'worker' and 'EngineCore'; anything
else must raise ValueError instead of silently routing to the worker
path."""
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0])
with (
pytest.raises(ValueError, match="process_kind"),
numa_utils.configure_subprocess(
vllm_config, local_rank=0, process_kind="bogus"
),
):
pass
def test_log_numactl_show(monkeypatch):
+6 -1
View File
@@ -267,7 +267,12 @@ class ParallelConfig:
"""num of nodes for multi-node distributed
inference when distributed_executor_backend is mp."""
numa_bind: bool = False
"""Enable NUMA binding for GPU worker subprocesses."""
"""Enable NUMA binding for GPU worker subprocesses.
By default, workers are pinned to their GPU's NUMA-local CPUs and
memory; on PCT-capable Xeons they also auto-bind to the SKU's
PCT priority cores.
"""
numa_bind_nodes: list[int] | None = None
"""NUMA node to bind each GPU worker to.
+282 -32
View File
@@ -14,7 +14,7 @@ import subprocess
from contextlib import contextmanager
from functools import cache
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, NamedTuple
import psutil
@@ -112,6 +112,117 @@ def get_auto_numa_nodes() -> list[int] | None:
return numa_nodes
# PCT (Priority Core Turbo) auto-detection workaround for Granite Rapids
# Xeon SKUs.
#
# Background:
# * The Linux kernel does not expose PCT priority-core membership via any
# unprivileged sysfs path. The official interface
# (/dev/isst_interface, used by `intel-speed-select`) is root-only,
# which is a non-starter in most production deployments (shared
# clusters, prebuilt containers, managed cloud).
# * Even recent stable kernels (e.g. 6.14, March 2025) do not yet
# preferentially schedule work on PCT priority cores, so vLLM cannot
# just "let the scheduler handle it".
#
# Empirical heuristic (DGX B300 / Xeon 6776P, the SKU we measured):
# * /proc/cpuinfo `model name` contains the SKU number.
# * cpu0 is a PCT priority core on these SKUs, so it reports the
# priority-cohort CPPC `highest_perf` (the value matches the SKU's
# "Max PCT core frequency" in 100 MHz units, e.g. 4.6 GHz -> 46).
# * Priority cores within each NUMA node satisfy `cpu_id % S in (0, 1)`
# intersected with the node's cpulist, where `S` is the SKU's logical
# CPUs per priority "group" (= total threads / 8 priority cores; 16 on
# 64-core SKUs, 18 on 72-core SKUs).
#
# SKU table:
# ``_PCT_CAPABLE_SKUS`` maps each known PCT-capable Granite Rapids part
# to a ``_PctSku(highest_perf, priority_stride)`` config:
# * highest_perf is the expected ``acpi_cppc/highest_perf`` on cpu0,
# derived from Intel ARK's "Max PCT core frequency" * 10 (CPPC max
# ratio reports in 100 MHz units).
# * priority_stride is the SKU's "Total Cores" / 4 (= total HT threads
# / 8 priority cores), used in the ``cpu_id % stride`` filter above.
# Values:
# * 6776P - 4.6 GHz, 64C/128T -> (46, 16) measured on DGX B300
# * 6774P - 4.6 GHz, 64C/128T -> (46, 16) per Intel ARK, not measured
# * 6962P - 4.4 GHz, 72C/144T -> (44, 18) per Intel ARK, not measured
# The non-measured SKUs are listed best-effort: the gate fails closed
# (no PCT engagement) if a host's actual highest_perf doesn't match the
# table value, so adding entries is safe. If you have access to a 6962P
# or 6774P box and find a different value or cpu-id pattern, update the
# table below.
#
# This whole block is a stop-gap until the kernel exposes PCT membership
# in an unprivileged way; see the tracking issue linked from the PR.
class _PctSku(NamedTuple):
"""Per-SKU config used by the PCT auto-detection gate."""
highest_perf: int
priority_stride: int
_PCT_CAPABLE_SKUS: dict[str, _PctSku] = {
"6776P": _PctSku(highest_perf=46, priority_stride=16),
"6774P": _PctSku(highest_perf=46, priority_stride=16),
"6962P": _PctSku(highest_perf=44, priority_stride=18),
}
_PCT_HIGHEST_PERF_PATH = "/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf"
_PROC_CPUINFO_PATH = "/proc/cpuinfo"
def _pct_sku_from_cpuinfo() -> _PctSku | None:
"""Return the ``_PctSku`` config for this host's SKU, or None.
Reads ``/proc/cpuinfo``'s ``model name`` and looks the SKU up in
``_PCT_CAPABLE_SKUS``. Returns ``None`` when the host is not a known
PCT-capable Granite Rapids Xeon (or when ``/proc/cpuinfo`` is
unreadable).
"""
try:
with open(_PROC_CPUINFO_PATH) as f:
for line in f:
if not line.lstrip().lower().startswith("model name"):
continue
for sku, config in _PCT_CAPABLE_SKUS.items():
if sku in line:
return config
except OSError:
return None
return None
@cache
def _pct_sku_config() -> _PctSku | None:
"""Detect a PCT-capable Granite Rapids Xeon with PCT enabled.
See the comment block above ``_PCT_CAPABLE_SKUS`` for the full context
(why we hard-code SKUs, why we read CPPC ``highest_perf``, etc.).
Returns the matching ``_PctSku`` config when both gates hold:
* ``/proc/cpuinfo`` ``model name`` contains an SKU listed in
``_PCT_CAPABLE_SKUS``.
* ``/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf`` matches
that SKU's expected ``highest_perf``.
Otherwise returns ``None`` and the caller falls back to the default
NUMA-node bind.
"""
sku = _pct_sku_from_cpuinfo()
if sku is None:
return None
try:
with open(_PCT_HIGHEST_PERF_PATH) as f:
actual = int(f.read().strip())
except (OSError, ValueError):
return None
if actual != sku.highest_perf:
return None
return sku
def _get_gpu_index(
parallel_config, local_rank: int, dp_local_rank: int | None = None
) -> int:
@@ -156,55 +267,183 @@ def _get_numa_node(parallel_config, gpu_index: int) -> int:
return numa_nodes[gpu_index]
def _get_cpu_binding(parallel_config, gpu_index: int) -> str | None:
def _maybe_get_pct_cpu_binding(numa_nodes: list[int]) -> list[int] | None:
"""Return the union of PCT priority cores across ``numa_nodes`` (or None).
PCT (Priority Core Turbo) lets a subset of cores boost above the rest;
we want workers and the EngineCore on those cores. The Linux kernel does
not expose PCT membership without root, so we use the empirical heuristic
documented above ``_PCT_CAPABLE_SKUS``: priority cores within each NUMA
node satisfy ``cpu_id % stride in (0, 1)`` intersected with the node's
``cpulist``, where ``stride`` is the SKU's logical CPUs per priority
group (16 on 64-core SKUs, 18 on 72-core SKUs). Only triggers on the
SKUs in ``_PCT_CAPABLE_SKUS`` with the expected CPPC ``highest_perf``
signal; on any other host it returns None and the caller falls back to
the default NUMA-node bind.
Returns the sorted CPU ids as a ``list[int]``; the caller is expected
to format them for the chosen tool (e.g. comma-joined for
``numactl --physcpubind``).
"""
sku = _pct_sku_config()
if sku is None:
return None
from vllm.utils.cpu_resource_utils import parse_id_list
stride = sku.priority_stride
union_cpus: set[int] = set()
for numa_node in numa_nodes:
cpulist_path = Path(f"/sys/devices/system/node/node{numa_node}/cpulist")
try:
cpulist_raw = cpulist_path.read_text().strip()
except OSError:
continue
if not cpulist_raw:
continue
try:
node_cpus = parse_id_list(cpulist_raw)
except ValueError:
continue
priority = [cpu for cpu in node_cpus if cpu % stride in (0, 1)]
if not priority:
continue
union_cpus.update(priority)
logger.info(
"Detected PCT-capable Granite Rapids Xeon (stride=%d); "
"NUMA node %d priority cores: %s",
stride,
numa_node,
",".join(str(c) for c in priority),
)
if not union_cpus:
return None
return sorted(union_cpus)
def _get_cpu_binding(
parallel_config, gpu_index: int, numa_nodes: list[int]
) -> str | None:
"""Return the CPU list a process should be pinned to (or None)."""
cpu_bindings = parallel_config.numa_bind_cpus
if cpu_bindings is None:
return None
pct_cpus = _maybe_get_pct_cpu_binding(numa_nodes)
if pct_cpus is None:
return None
return ",".join(str(c) for c in pct_cpus)
if gpu_index >= len(cpu_bindings):
raise ValueError(
f"GPU index {gpu_index} exceeds numa_bind_cpus size "
f"{len(cpu_bindings)}. Ensure the binding lists cover every visible GPU."
)
return cpu_bindings[gpu_index]
def _get_numactl_args(
vllm_config: "VllmConfig",
local_rank: int,
dp_local_rank: int | None = None,
process_kind: str = "worker",
) -> str | None:
parallel_config = vllm_config.parallel_config
if not parallel_config.numa_bind:
return None
def _get_numactl_worker_args(
parallel_config, local_rank: int, dp_local_rank: int | None = None
) -> str:
"""Compute the numactl args for a single TP/PP worker subprocess."""
gpu_index = _get_gpu_index(parallel_config, local_rank, dp_local_rank)
numa_node = _get_numa_node(parallel_config, gpu_index)
cpu_binding = _get_cpu_binding(parallel_config, gpu_index)
cpu_binding = _get_cpu_binding(parallel_config, gpu_index, [numa_node])
if cpu_binding is not None:
bind_arg = f"--physcpubind={cpu_binding}"
logger.info(
"Binding %s subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
process_kind,
"Binding worker subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
local_rank,
gpu_index,
cpu_binding,
numa_node,
)
else:
bind_arg = f"--cpunodebind={numa_node}"
logger.info(
"Binding %s subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
process_kind,
local_rank,
gpu_index,
numa_node,
)
return f"--physcpubind={cpu_binding} --membind={numa_node}"
return f"{bind_arg} --membind={numa_node}"
logger.info(
"Binding worker subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
local_rank,
gpu_index,
numa_node,
)
return f"--cpunodebind={numa_node} --membind={numa_node}"
def _get_enginecore_numa_nodes(
parallel_config, dp_local_rank: int | None = None
) -> list[int]:
"""Return the sorted, unique NUMA nodes of the EngineCore's DP shard."""
numa_nodes = parallel_config.numa_bind_nodes
if numa_nodes is None:
# Trigger auto-detection (it caches into parallel_config).
_get_numa_node(parallel_config, 0)
numa_nodes = parallel_config.numa_bind_nodes
if (
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
and parallel_config.data_parallel_backend != "ray"
and parallel_config.nnodes_within_dp == 1
):
if dp_local_rank is None:
dp_local_rank = parallel_config.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = parallel_config.data_parallel_index
tp_pp_world_size = (
parallel_config.pipeline_parallel_size
* parallel_config.tensor_parallel_size
)
shard_start = dp_local_rank * tp_pp_world_size
shard_end = min(shard_start + tp_pp_world_size, len(numa_nodes))
shard_indices: range | tuple[int, ...] = range(shard_start, shard_end)
else:
shard_indices = range(len(numa_nodes))
if not shard_indices:
return [numa_nodes[0]]
return sorted({numa_nodes[i] for i in shard_indices})
def _get_numactl_enginecore_args(
parallel_config, local_rank: int, dp_local_rank: int | None = None
) -> str:
"""Compute the numactl args for an EngineCore subprocess.
``--numa-bind-cpus`` is deliberately ignored here: the user provides a
per-worker CPU list, and binding EngineCore to any of those entries
would shrink its ``cpus_allowed`` below the strict-superset that the
workers' ``--physcpubind`` spawns require. We fall back to
``--cpunodebind=<shard nodes>`` instead, which is always a safe
superset. PCT auto-detection still applies when the user did not pass
``--numa-bind-cpus`` (its priority-core union across the shard nodes
is also a safe superset by construction).
"""
shard_nodes = _get_enginecore_numa_nodes(parallel_config, dp_local_rank)
membind_arg = ",".join(str(n) for n in shard_nodes)
pct_cpus = (
None
if parallel_config.numa_bind_cpus is not None
else _maybe_get_pct_cpu_binding(shard_nodes)
)
if pct_cpus is not None:
cpu_binding = ",".join(str(c) for c in pct_cpus)
logger.info(
"Binding EngineCore subprocess (local_rank=%s) to CPUs %s "
"and NUMA nodes %s",
local_rank,
cpu_binding,
membind_arg,
)
return f"--physcpubind={cpu_binding} --membind={membind_arg}"
logger.info(
"Binding EngineCore subprocess (local_rank=%s) to NUMA nodes %s",
local_rank,
membind_arg,
)
return f"--cpunodebind={membind_arg} --membind={membind_arg}"
def _log_numactl_show(label: str) -> bool:
@@ -242,13 +481,24 @@ def configure_subprocess(
process_kind: str = "worker",
):
"""Temporarily replace the multiprocessing executable with a numactl wrapper."""
numactl_args = _get_numactl_args(
vllm_config, local_rank, dp_local_rank, process_kind
)
if numactl_args is None:
parallel_config = vllm_config.parallel_config
if not parallel_config.numa_bind:
yield
return
if process_kind == "EngineCore":
numactl_args = _get_numactl_enginecore_args(
parallel_config, local_rank, dp_local_rank
)
elif process_kind == "worker":
numactl_args = _get_numactl_worker_args(
parallel_config, local_rank, dp_local_rank
)
else:
raise ValueError(
f"Unknown process_kind {process_kind!r}; expected 'worker' or 'EngineCore'."
)
executable, debug_str = _get_numactl_executable()
python_executable = os.fsdecode(multiprocessing.spawn.get_executable())
with (