[Misc][NUMA] Auto-bind to PCT priority cores on DGX B300 + widen EngineCore across shard NUMA nodes (#43270)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Cursor <noreply@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
1521173c17
commit
bf18d7e0b4
@@ -10,6 +10,33 @@ from vllm.config import ParallelConfig
|
||||
from vllm.utils import numa_utils
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _disable_pct_by_default(monkeypatch):
|
||||
"""Force PCT detection OFF unless a test opts in via ``_patch_pct_gates``.
|
||||
|
||||
The CI / dev machines themselves can be Xeon 6776P with PCT enabled, so a
|
||||
plain ``cache_clear`` would let the gate auto-detect ``True`` from the
|
||||
live filesystem and silently re-route "baseline" tests through the PCT
|
||||
path. Stub ``/proc/cpuinfo`` and ``acpi_cppc/highest_perf`` to a state
|
||||
that fails the gate; ``_patch_pct_gates`` re-stubs on top when needed.
|
||||
"""
|
||||
from io import StringIO
|
||||
|
||||
real_open = open
|
||||
|
||||
def _no_pct_open(path, *args, **kwargs):
|
||||
if path == numa_utils._PROC_CPUINFO_PATH:
|
||||
return StringIO("processor\t: 0\nmodel name\t: Generic Test CPU\n")
|
||||
if path == numa_utils._PCT_HIGHEST_PERF_PATH:
|
||||
raise OSError("PCT disabled by autouse fixture")
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr("builtins.open", _no_pct_open)
|
||||
numa_utils._pct_sku_config.cache_clear()
|
||||
yield
|
||||
numa_utils._pct_sku_config.cache_clear()
|
||||
|
||||
|
||||
def _make_config(**parallel_kwargs):
|
||||
parallel_defaults = dict(
|
||||
numa_bind=False,
|
||||
@@ -31,7 +58,7 @@ def _make_config(**parallel_kwargs):
|
||||
def test_get_numactl_args_with_node_binding():
|
||||
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
|
||||
assert (
|
||||
numa_utils._get_numactl_args(vllm_config, local_rank=1)
|
||||
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
|
||||
== "--cpunodebind=1 --membind=1"
|
||||
)
|
||||
|
||||
@@ -43,7 +70,300 @@ def test_get_numactl_args_with_cpu_binding():
|
||||
numa_bind_cpus=["0-3", "4-7"],
|
||||
)
|
||||
assert (
|
||||
numa_utils._get_numactl_args(vllm_config, local_rank=1)
|
||||
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
|
||||
== "--physcpubind=4-7 --membind=1"
|
||||
)
|
||||
|
||||
|
||||
def _patch_pct_gates(
|
||||
monkeypatch,
|
||||
*,
|
||||
model_match: bool,
|
||||
highest_perf: int | None,
|
||||
cpulist: str | None = "0-31,64-95",
|
||||
cpulist_by_node: dict[int, str | None] | None = None,
|
||||
sku: str = "6776P",
|
||||
):
|
||||
"""Force `_pct_sku_config` and node cpulist read to deterministic state.
|
||||
|
||||
``cpulist`` is the default returned for any node not present in
|
||||
``cpulist_by_node``. ``cpulist_by_node`` lets a test return different
|
||||
cpulists for different NUMA nodes (e.g. node 0 vs node 1). ``sku`` lets
|
||||
the test pick which Granite Rapids SKU appears in the fake
|
||||
``/proc/cpuinfo`` ``model name`` (only used when ``model_match=True``).
|
||||
"""
|
||||
import pathlib
|
||||
from io import StringIO
|
||||
|
||||
import regex as re
|
||||
|
||||
cpuinfo = (
|
||||
f"processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Platinum {sku} CPU @ 2.40GHz\n"
|
||||
if model_match
|
||||
else "processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Platinum 8480+\n"
|
||||
)
|
||||
|
||||
real_open = open
|
||||
|
||||
def fake_open(path, *args, **kwargs):
|
||||
if path == numa_utils._PROC_CPUINFO_PATH:
|
||||
return StringIO(cpuinfo)
|
||||
if path == numa_utils._PCT_HIGHEST_PERF_PATH:
|
||||
if highest_perf is None:
|
||||
raise OSError("missing")
|
||||
return StringIO(f"{highest_perf}\n")
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
real_read_text = pathlib.Path.read_text
|
||||
cpulist_by_node = cpulist_by_node or {}
|
||||
|
||||
def fake_read_text(self, *args, **kwargs):
|
||||
path_str = str(self)
|
||||
if path_str.endswith("/cpulist") and "/sys/devices/system/node" in path_str:
|
||||
match = re.search(r"/node(\d+)/cpulist$", path_str)
|
||||
if match:
|
||||
node_id = int(match.group(1))
|
||||
if node_id in cpulist_by_node:
|
||||
val = cpulist_by_node[node_id]
|
||||
if val is None:
|
||||
raise OSError(f"missing cpulist for node{node_id}")
|
||||
return val
|
||||
if cpulist is None:
|
||||
raise OSError("missing cpulist")
|
||||
return cpulist
|
||||
return real_read_text(self, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr("builtins.open", fake_open)
|
||||
monkeypatch.setattr("pathlib.Path.read_text", fake_read_text)
|
||||
numa_utils._pct_sku_config.cache_clear()
|
||||
|
||||
|
||||
def test_pct_binding_filters_cpus(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) == [0, 1, 16, 17, 64, 65, 80, 81]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sku,expected_cpus",
|
||||
[
|
||||
# 64-core SKUs (stride 16): cpus from "0-31,64-95" with cpu_id % 16
|
||||
# in (0, 1) -> 0, 1, 16, 17, 64, 65, 80, 81.
|
||||
("6776P", [0, 1, 16, 17, 64, 65, 80, 81]),
|
||||
("6774P", [0, 1, 16, 17, 64, 65, 80, 81]),
|
||||
# 72-core SKU (stride 18): cpus from "0-31,64-95" with cpu_id % 18
|
||||
# in (0, 1) -> 0, 1, 18, 19, 72, 73, 90, 91.
|
||||
("6962P", [0, 1, 18, 19, 72, 73, 90, 91]),
|
||||
],
|
||||
)
|
||||
def test_pct_binding_fires_on_every_capable_sku(monkeypatch, sku, expected_cpus):
|
||||
"""Each SKU in ``_PCT_CAPABLE_SKUS`` engages the gate at its own
|
||||
expected ``highest_perf`` and uses its own priority-core stride."""
|
||||
sku_config = numa_utils._PCT_CAPABLE_SKUS[sku]
|
||||
_patch_pct_gates(
|
||||
monkeypatch,
|
||||
model_match=True,
|
||||
highest_perf=sku_config.highest_perf,
|
||||
sku=sku,
|
||||
)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) == expected_cpus
|
||||
|
||||
|
||||
def test_pct_binding_fails_closed_when_sku_perf_mismatch(monkeypatch):
|
||||
"""6962P with 6776P's highest_perf (46 vs expected 44) must fail closed."""
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46, sku="6962P")
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
|
||||
|
||||
|
||||
def test_pct_binding_disabled_when_cpu_model_mismatch(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=False, highest_perf=46)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
|
||||
|
||||
|
||||
def test_pct_binding_disabled_when_highest_perf_does_not_match(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=42)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
|
||||
|
||||
|
||||
def test_pct_binding_disabled_when_files_missing(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=None)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
|
||||
|
||||
|
||||
def test_pct_binding_returns_none_when_node_cpulist_filter_empty(monkeypatch):
|
||||
_patch_pct_gates(
|
||||
monkeypatch,
|
||||
model_match=True,
|
||||
highest_perf=46,
|
||||
cpulist="2-15,18-31",
|
||||
)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
|
||||
|
||||
|
||||
def test_pct_binding_returns_none_when_node_cpulist_missing(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46, cpulist=None)
|
||||
assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
|
||||
|
||||
|
||||
def test_get_numactl_args_uses_pct_when_user_did_not_specify_cpus(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
|
||||
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
|
||||
assert (
|
||||
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
|
||||
== "--physcpubind=0,1,16,17,64,65,80,81 --membind=1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_baseline_single_node_shard():
|
||||
"""Baseline (no PCT): single-NUMA shard -> single-node bind."""
|
||||
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
|
||||
assert (
|
||||
numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
)
|
||||
== "--cpunodebind=0 --membind=0"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_baseline_spans_shard_numa_nodes():
|
||||
"""Baseline (no PCT): a TP=4 shard spanning both NUMA nodes -> bind to both."""
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 0, 1, 1],
|
||||
tensor_parallel_size=4,
|
||||
)
|
||||
assert (
|
||||
numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
)
|
||||
== "--cpunodebind=0,1 --membind=0,1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_pct_spans_shard_numa_nodes(monkeypatch):
|
||||
"""PCT: EngineCore for a multi-NUMA shard binds to the union of priority
|
||||
cores across all shard nodes, so worker `--physcpubind` is always a
|
||||
subset of EngineCore's `cpus_allowed`."""
|
||||
_patch_pct_gates(
|
||||
monkeypatch,
|
||||
model_match=True,
|
||||
highest_perf=46,
|
||||
cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
|
||||
)
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 0, 1, 1],
|
||||
tensor_parallel_size=4,
|
||||
)
|
||||
assert numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
) == (
|
||||
"--physcpubind="
|
||||
"0,1,16,17,64,65,80,81,128,129,144,145,192,193,208,209"
|
||||
" --membind=0,1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_pct_dp_shard_picks_local_nodes(monkeypatch):
|
||||
"""With DP=2, each shard's EngineCore binds only to its own NUMA nodes."""
|
||||
_patch_pct_gates(
|
||||
monkeypatch,
|
||||
model_match=True,
|
||||
highest_perf=46,
|
||||
cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
|
||||
)
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 0, 1, 1],
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_rank_local=1,
|
||||
)
|
||||
# Shard 1 owns gpu_indices 2 and 3 -> nodes [1, 1] -> {1}.
|
||||
assert (
|
||||
numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
)
|
||||
== "--physcpubind=64,65,80,81,192,193,208,209 --membind=1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_pct_external_launcher_spans_local_nodes(
|
||||
monkeypatch,
|
||||
):
|
||||
"""external_launcher (or multi-node-within-DP, or Ray) hits the
|
||||
fallback branch. EngineCore must still span every local NUMA node
|
||||
so it can mp-spawn its local workers without ``--physcpubind``
|
||||
strict-validation failures."""
|
||||
_patch_pct_gates(
|
||||
monkeypatch,
|
||||
model_match=True,
|
||||
highest_perf=46,
|
||||
cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
|
||||
)
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 0, 0, 0, 1, 1, 1, 1],
|
||||
distributed_executor_backend="external_launcher",
|
||||
tensor_parallel_size=8,
|
||||
)
|
||||
assert numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
) == (
|
||||
"--physcpubind="
|
||||
"0,1,16,17,64,65,80,81,128,129,144,145,192,193,208,209"
|
||||
" --membind=0,1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_baseline_multi_node_within_dp_spans_locals():
|
||||
"""Multi-node-within-DP fallback: bind EngineCore to all local NUMA
|
||||
nodes that the visible ``numa_bind_nodes`` reference."""
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 0, 0, 0, 1, 1, 1, 1],
|
||||
nnodes_within_dp=2,
|
||||
tensor_parallel_size=8,
|
||||
)
|
||||
assert (
|
||||
numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
)
|
||||
== "--cpunodebind=0,1 --membind=0,1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_engine_core_skips_user_cpu_list(monkeypatch):
|
||||
"""EngineCore ignores ``--numa-bind-cpus``.
|
||||
|
||||
Those are per-worker lists; binding EngineCore to any of them would
|
||||
shrink its ``cpus_allowed`` below the strict-superset workers'
|
||||
``--physcpubind`` spawns need. We fall back to ``--cpunodebind`` over
|
||||
the shard's NUMA nodes instead. PCT auto-detect is also bypassed when
|
||||
the user is explicit (its priority-core union may not be a superset
|
||||
of the user's per-worker cores)."""
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 0, 1, 1],
|
||||
numa_bind_cpus=["0-3", "4-7", "64-67", "68-71"],
|
||||
tensor_parallel_size=4,
|
||||
)
|
||||
assert (
|
||||
numa_utils._get_numactl_enginecore_args(
|
||||
vllm_config.parallel_config, local_rank=0
|
||||
)
|
||||
== "--cpunodebind=0,1 --membind=0,1"
|
||||
)
|
||||
|
||||
|
||||
def test_get_numactl_args_user_cpus_override_pct(monkeypatch):
|
||||
_patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
|
||||
vllm_config = _make_config(
|
||||
numa_bind=True,
|
||||
numa_bind_nodes=[0, 1],
|
||||
numa_bind_cpus=["0-3", "4-7"],
|
||||
)
|
||||
assert (
|
||||
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
|
||||
== "--physcpubind=4-7 --membind=1"
|
||||
)
|
||||
|
||||
@@ -57,7 +377,7 @@ def test_get_numactl_args_uses_dp_offset():
|
||||
tensor_parallel_size=2,
|
||||
)
|
||||
assert (
|
||||
numa_utils._get_numactl_args(vllm_config, local_rank=1)
|
||||
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
|
||||
== "--cpunodebind=1 --membind=1"
|
||||
)
|
||||
|
||||
@@ -66,7 +386,21 @@ def test_get_numactl_args_requires_detectable_nodes(monkeypatch):
|
||||
vllm_config = _make_config(numa_bind=True)
|
||||
monkeypatch.setattr(numa_utils, "get_auto_numa_nodes", lambda: None)
|
||||
with pytest.raises(RuntimeError):
|
||||
numa_utils._get_numactl_args(vllm_config, local_rank=0)
|
||||
numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=0)
|
||||
|
||||
|
||||
def test_configure_subprocess_rejects_unknown_process_kind():
|
||||
"""configure_subprocess only knows 'worker' and 'EngineCore'; anything
|
||||
else must raise ValueError instead of silently routing to the worker
|
||||
path."""
|
||||
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0])
|
||||
with (
|
||||
pytest.raises(ValueError, match="process_kind"),
|
||||
numa_utils.configure_subprocess(
|
||||
vllm_config, local_rank=0, process_kind="bogus"
|
||||
),
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def test_log_numactl_show(monkeypatch):
|
||||
|
||||
@@ -267,7 +267,12 @@ class ParallelConfig:
|
||||
"""num of nodes for multi-node distributed
|
||||
inference when distributed_executor_backend is mp."""
|
||||
numa_bind: bool = False
|
||||
"""Enable NUMA binding for GPU worker subprocesses."""
|
||||
"""Enable NUMA binding for GPU worker subprocesses.
|
||||
|
||||
By default, workers are pinned to their GPU's NUMA-local CPUs and
|
||||
memory; on PCT-capable Xeons they also auto-bind to the SKU's
|
||||
PCT priority cores.
|
||||
"""
|
||||
numa_bind_nodes: list[int] | None = None
|
||||
"""NUMA node to bind each GPU worker to.
|
||||
|
||||
|
||||
+282
-32
@@ -14,7 +14,7 @@ import subprocess
|
||||
from contextlib import contextmanager
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
import psutil
|
||||
|
||||
@@ -112,6 +112,117 @@ def get_auto_numa_nodes() -> list[int] | None:
|
||||
return numa_nodes
|
||||
|
||||
|
||||
# PCT (Priority Core Turbo) auto-detection workaround for Granite Rapids
|
||||
# Xeon SKUs.
|
||||
#
|
||||
# Background:
|
||||
# * The Linux kernel does not expose PCT priority-core membership via any
|
||||
# unprivileged sysfs path. The official interface
|
||||
# (/dev/isst_interface, used by `intel-speed-select`) is root-only,
|
||||
# which is a non-starter in most production deployments (shared
|
||||
# clusters, prebuilt containers, managed cloud).
|
||||
# * Even recent stable kernels (e.g. 6.14, March 2025) do not yet
|
||||
# preferentially schedule work on PCT priority cores, so vLLM cannot
|
||||
# just "let the scheduler handle it".
|
||||
#
|
||||
# Empirical heuristic (DGX B300 / Xeon 6776P, the SKU we measured):
|
||||
# * /proc/cpuinfo `model name` contains the SKU number.
|
||||
# * cpu0 is a PCT priority core on these SKUs, so it reports the
|
||||
# priority-cohort CPPC `highest_perf` (the value matches the SKU's
|
||||
# "Max PCT core frequency" in 100 MHz units, e.g. 4.6 GHz -> 46).
|
||||
# * Priority cores within each NUMA node satisfy `cpu_id % S in (0, 1)`
|
||||
# intersected with the node's cpulist, where `S` is the SKU's logical
|
||||
# CPUs per priority "group" (= total threads / 8 priority cores; 16 on
|
||||
# 64-core SKUs, 18 on 72-core SKUs).
|
||||
#
|
||||
# SKU table:
|
||||
# ``_PCT_CAPABLE_SKUS`` maps each known PCT-capable Granite Rapids part
|
||||
# to a ``_PctSku(highest_perf, priority_stride)`` config:
|
||||
# * highest_perf is the expected ``acpi_cppc/highest_perf`` on cpu0,
|
||||
# derived from Intel ARK's "Max PCT core frequency" * 10 (CPPC max
|
||||
# ratio reports in 100 MHz units).
|
||||
# * priority_stride is the SKU's "Total Cores" / 4 (= total HT threads
|
||||
# / 8 priority cores), used in the ``cpu_id % stride`` filter above.
|
||||
# Values:
|
||||
# * 6776P - 4.6 GHz, 64C/128T -> (46, 16) measured on DGX B300
|
||||
# * 6774P - 4.6 GHz, 64C/128T -> (46, 16) per Intel ARK, not measured
|
||||
# * 6962P - 4.4 GHz, 72C/144T -> (44, 18) per Intel ARK, not measured
|
||||
# The non-measured SKUs are listed best-effort: the gate fails closed
|
||||
# (no PCT engagement) if a host's actual highest_perf doesn't match the
|
||||
# table value, so adding entries is safe. If you have access to a 6962P
|
||||
# or 6774P box and find a different value or cpu-id pattern, update the
|
||||
# table below.
|
||||
#
|
||||
# This whole block is a stop-gap until the kernel exposes PCT membership
|
||||
# in an unprivileged way; see the tracking issue linked from the PR.
|
||||
|
||||
|
||||
class _PctSku(NamedTuple):
|
||||
"""Per-SKU config used by the PCT auto-detection gate."""
|
||||
|
||||
highest_perf: int
|
||||
priority_stride: int
|
||||
|
||||
|
||||
_PCT_CAPABLE_SKUS: dict[str, _PctSku] = {
|
||||
"6776P": _PctSku(highest_perf=46, priority_stride=16),
|
||||
"6774P": _PctSku(highest_perf=46, priority_stride=16),
|
||||
"6962P": _PctSku(highest_perf=44, priority_stride=18),
|
||||
}
|
||||
_PCT_HIGHEST_PERF_PATH = "/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf"
|
||||
_PROC_CPUINFO_PATH = "/proc/cpuinfo"
|
||||
|
||||
|
||||
def _pct_sku_from_cpuinfo() -> _PctSku | None:
|
||||
"""Return the ``_PctSku`` config for this host's SKU, or None.
|
||||
|
||||
Reads ``/proc/cpuinfo``'s ``model name`` and looks the SKU up in
|
||||
``_PCT_CAPABLE_SKUS``. Returns ``None`` when the host is not a known
|
||||
PCT-capable Granite Rapids Xeon (or when ``/proc/cpuinfo`` is
|
||||
unreadable).
|
||||
"""
|
||||
try:
|
||||
with open(_PROC_CPUINFO_PATH) as f:
|
||||
for line in f:
|
||||
if not line.lstrip().lower().startswith("model name"):
|
||||
continue
|
||||
for sku, config in _PCT_CAPABLE_SKUS.items():
|
||||
if sku in line:
|
||||
return config
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
@cache
|
||||
def _pct_sku_config() -> _PctSku | None:
|
||||
"""Detect a PCT-capable Granite Rapids Xeon with PCT enabled.
|
||||
|
||||
See the comment block above ``_PCT_CAPABLE_SKUS`` for the full context
|
||||
(why we hard-code SKUs, why we read CPPC ``highest_perf``, etc.).
|
||||
|
||||
Returns the matching ``_PctSku`` config when both gates hold:
|
||||
* ``/proc/cpuinfo`` ``model name`` contains an SKU listed in
|
||||
``_PCT_CAPABLE_SKUS``.
|
||||
* ``/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf`` matches
|
||||
that SKU's expected ``highest_perf``.
|
||||
Otherwise returns ``None`` and the caller falls back to the default
|
||||
NUMA-node bind.
|
||||
"""
|
||||
sku = _pct_sku_from_cpuinfo()
|
||||
if sku is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(_PCT_HIGHEST_PERF_PATH) as f:
|
||||
actual = int(f.read().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if actual != sku.highest_perf:
|
||||
return None
|
||||
return sku
|
||||
|
||||
|
||||
def _get_gpu_index(
|
||||
parallel_config, local_rank: int, dp_local_rank: int | None = None
|
||||
) -> int:
|
||||
@@ -156,55 +267,183 @@ def _get_numa_node(parallel_config, gpu_index: int) -> int:
|
||||
return numa_nodes[gpu_index]
|
||||
|
||||
|
||||
def _get_cpu_binding(parallel_config, gpu_index: int) -> str | None:
|
||||
def _maybe_get_pct_cpu_binding(numa_nodes: list[int]) -> list[int] | None:
|
||||
"""Return the union of PCT priority cores across ``numa_nodes`` (or None).
|
||||
|
||||
PCT (Priority Core Turbo) lets a subset of cores boost above the rest;
|
||||
we want workers and the EngineCore on those cores. The Linux kernel does
|
||||
not expose PCT membership without root, so we use the empirical heuristic
|
||||
documented above ``_PCT_CAPABLE_SKUS``: priority cores within each NUMA
|
||||
node satisfy ``cpu_id % stride in (0, 1)`` intersected with the node's
|
||||
``cpulist``, where ``stride`` is the SKU's logical CPUs per priority
|
||||
group (16 on 64-core SKUs, 18 on 72-core SKUs). Only triggers on the
|
||||
SKUs in ``_PCT_CAPABLE_SKUS`` with the expected CPPC ``highest_perf``
|
||||
signal; on any other host it returns None and the caller falls back to
|
||||
the default NUMA-node bind.
|
||||
|
||||
Returns the sorted CPU ids as a ``list[int]``; the caller is expected
|
||||
to format them for the chosen tool (e.g. comma-joined for
|
||||
``numactl --physcpubind``).
|
||||
"""
|
||||
sku = _pct_sku_config()
|
||||
if sku is None:
|
||||
return None
|
||||
|
||||
from vllm.utils.cpu_resource_utils import parse_id_list
|
||||
|
||||
stride = sku.priority_stride
|
||||
union_cpus: set[int] = set()
|
||||
for numa_node in numa_nodes:
|
||||
cpulist_path = Path(f"/sys/devices/system/node/node{numa_node}/cpulist")
|
||||
try:
|
||||
cpulist_raw = cpulist_path.read_text().strip()
|
||||
except OSError:
|
||||
continue
|
||||
if not cpulist_raw:
|
||||
continue
|
||||
try:
|
||||
node_cpus = parse_id_list(cpulist_raw)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
priority = [cpu for cpu in node_cpus if cpu % stride in (0, 1)]
|
||||
if not priority:
|
||||
continue
|
||||
union_cpus.update(priority)
|
||||
logger.info(
|
||||
"Detected PCT-capable Granite Rapids Xeon (stride=%d); "
|
||||
"NUMA node %d priority cores: %s",
|
||||
stride,
|
||||
numa_node,
|
||||
",".join(str(c) for c in priority),
|
||||
)
|
||||
|
||||
if not union_cpus:
|
||||
return None
|
||||
return sorted(union_cpus)
|
||||
|
||||
|
||||
def _get_cpu_binding(
|
||||
parallel_config, gpu_index: int, numa_nodes: list[int]
|
||||
) -> str | None:
|
||||
"""Return the CPU list a process should be pinned to (or None)."""
|
||||
cpu_bindings = parallel_config.numa_bind_cpus
|
||||
if cpu_bindings is None:
|
||||
return None
|
||||
pct_cpus = _maybe_get_pct_cpu_binding(numa_nodes)
|
||||
if pct_cpus is None:
|
||||
return None
|
||||
return ",".join(str(c) for c in pct_cpus)
|
||||
|
||||
if gpu_index >= len(cpu_bindings):
|
||||
raise ValueError(
|
||||
f"GPU index {gpu_index} exceeds numa_bind_cpus size "
|
||||
f"{len(cpu_bindings)}. Ensure the binding lists cover every visible GPU."
|
||||
)
|
||||
|
||||
return cpu_bindings[gpu_index]
|
||||
|
||||
|
||||
def _get_numactl_args(
|
||||
vllm_config: "VllmConfig",
|
||||
local_rank: int,
|
||||
dp_local_rank: int | None = None,
|
||||
process_kind: str = "worker",
|
||||
) -> str | None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not parallel_config.numa_bind:
|
||||
return None
|
||||
|
||||
def _get_numactl_worker_args(
|
||||
parallel_config, local_rank: int, dp_local_rank: int | None = None
|
||||
) -> str:
|
||||
"""Compute the numactl args for a single TP/PP worker subprocess."""
|
||||
gpu_index = _get_gpu_index(parallel_config, local_rank, dp_local_rank)
|
||||
numa_node = _get_numa_node(parallel_config, gpu_index)
|
||||
cpu_binding = _get_cpu_binding(parallel_config, gpu_index)
|
||||
cpu_binding = _get_cpu_binding(parallel_config, gpu_index, [numa_node])
|
||||
|
||||
if cpu_binding is not None:
|
||||
bind_arg = f"--physcpubind={cpu_binding}"
|
||||
logger.info(
|
||||
"Binding %s subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
|
||||
process_kind,
|
||||
"Binding worker subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
|
||||
local_rank,
|
||||
gpu_index,
|
||||
cpu_binding,
|
||||
numa_node,
|
||||
)
|
||||
else:
|
||||
bind_arg = f"--cpunodebind={numa_node}"
|
||||
logger.info(
|
||||
"Binding %s subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
|
||||
process_kind,
|
||||
local_rank,
|
||||
gpu_index,
|
||||
numa_node,
|
||||
)
|
||||
return f"--physcpubind={cpu_binding} --membind={numa_node}"
|
||||
|
||||
return f"{bind_arg} --membind={numa_node}"
|
||||
logger.info(
|
||||
"Binding worker subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
|
||||
local_rank,
|
||||
gpu_index,
|
||||
numa_node,
|
||||
)
|
||||
return f"--cpunodebind={numa_node} --membind={numa_node}"
|
||||
|
||||
|
||||
def _get_enginecore_numa_nodes(
|
||||
parallel_config, dp_local_rank: int | None = None
|
||||
) -> list[int]:
|
||||
"""Return the sorted, unique NUMA nodes of the EngineCore's DP shard."""
|
||||
numa_nodes = parallel_config.numa_bind_nodes
|
||||
if numa_nodes is None:
|
||||
# Trigger auto-detection (it caches into parallel_config).
|
||||
_get_numa_node(parallel_config, 0)
|
||||
numa_nodes = parallel_config.numa_bind_nodes
|
||||
|
||||
if (
|
||||
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
):
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = parallel_config.data_parallel_rank_local
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = parallel_config.data_parallel_index
|
||||
|
||||
tp_pp_world_size = (
|
||||
parallel_config.pipeline_parallel_size
|
||||
* parallel_config.tensor_parallel_size
|
||||
)
|
||||
shard_start = dp_local_rank * tp_pp_world_size
|
||||
shard_end = min(shard_start + tp_pp_world_size, len(numa_nodes))
|
||||
shard_indices: range | tuple[int, ...] = range(shard_start, shard_end)
|
||||
else:
|
||||
shard_indices = range(len(numa_nodes))
|
||||
|
||||
if not shard_indices:
|
||||
return [numa_nodes[0]]
|
||||
return sorted({numa_nodes[i] for i in shard_indices})
|
||||
|
||||
|
||||
def _get_numactl_enginecore_args(
|
||||
parallel_config, local_rank: int, dp_local_rank: int | None = None
|
||||
) -> str:
|
||||
"""Compute the numactl args for an EngineCore subprocess.
|
||||
|
||||
``--numa-bind-cpus`` is deliberately ignored here: the user provides a
|
||||
per-worker CPU list, and binding EngineCore to any of those entries
|
||||
would shrink its ``cpus_allowed`` below the strict-superset that the
|
||||
workers' ``--physcpubind`` spawns require. We fall back to
|
||||
``--cpunodebind=<shard nodes>`` instead, which is always a safe
|
||||
superset. PCT auto-detection still applies when the user did not pass
|
||||
``--numa-bind-cpus`` (its priority-core union across the shard nodes
|
||||
is also a safe superset by construction).
|
||||
"""
|
||||
shard_nodes = _get_enginecore_numa_nodes(parallel_config, dp_local_rank)
|
||||
membind_arg = ",".join(str(n) for n in shard_nodes)
|
||||
|
||||
pct_cpus = (
|
||||
None
|
||||
if parallel_config.numa_bind_cpus is not None
|
||||
else _maybe_get_pct_cpu_binding(shard_nodes)
|
||||
)
|
||||
|
||||
if pct_cpus is not None:
|
||||
cpu_binding = ",".join(str(c) for c in pct_cpus)
|
||||
logger.info(
|
||||
"Binding EngineCore subprocess (local_rank=%s) to CPUs %s "
|
||||
"and NUMA nodes %s",
|
||||
local_rank,
|
||||
cpu_binding,
|
||||
membind_arg,
|
||||
)
|
||||
return f"--physcpubind={cpu_binding} --membind={membind_arg}"
|
||||
|
||||
logger.info(
|
||||
"Binding EngineCore subprocess (local_rank=%s) to NUMA nodes %s",
|
||||
local_rank,
|
||||
membind_arg,
|
||||
)
|
||||
return f"--cpunodebind={membind_arg} --membind={membind_arg}"
|
||||
|
||||
|
||||
def _log_numactl_show(label: str) -> bool:
|
||||
@@ -242,13 +481,24 @@ def configure_subprocess(
|
||||
process_kind: str = "worker",
|
||||
):
|
||||
"""Temporarily replace the multiprocessing executable with a numactl wrapper."""
|
||||
numactl_args = _get_numactl_args(
|
||||
vllm_config, local_rank, dp_local_rank, process_kind
|
||||
)
|
||||
if numactl_args is None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not parallel_config.numa_bind:
|
||||
yield
|
||||
return
|
||||
|
||||
if process_kind == "EngineCore":
|
||||
numactl_args = _get_numactl_enginecore_args(
|
||||
parallel_config, local_rank, dp_local_rank
|
||||
)
|
||||
elif process_kind == "worker":
|
||||
numactl_args = _get_numactl_worker_args(
|
||||
parallel_config, local_rank, dp_local_rank
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown process_kind {process_kind!r}; expected 'worker' or 'EngineCore'."
|
||||
)
|
||||
|
||||
executable, debug_str = _get_numactl_executable()
|
||||
python_executable = os.fsdecode(multiprocessing.spawn.get_executable())
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user