[Feature] NUMA binding support for GPU workers (#38635)

Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
Co-authored-by: Jason Li <jasonlizhengjian@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Shengqi Chen
2026-04-08 09:55:24 -07:00
committed by GitHub
co-authored by Jason Li Roger Wang
parent 512c5eb455
commit 75e01a39a1
13 changed files with 817 additions and 7 deletions
+67
View File
@@ -6,6 +6,7 @@ import socket
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Literal, overload
import regex as re
import torch
from pydantic import Field, field_validator, model_validator
from torch.distributed import ProcessGroup, ReduceOp, Store
@@ -28,6 +29,7 @@ else:
Executor = Any
logger = init_logger(__name__)
_NUMACTL_CPUSET_PATTERN = re.compile(r"^\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*$")
ExpertPlacementStrategy = Literal["linear", "round_robin"]
DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]
@@ -259,6 +261,27 @@ class ParallelConfig:
nnodes: int = 1
"""num of nodes for multi-node distributed
inference when distributed_executor_backend is mp."""
numa_bind: bool = False
"""Enable NUMA binding for GPU worker subprocesses."""
numa_bind_nodes: list[int] | None = None
"""NUMA node to bind each GPU worker to.
Specify one NUMA node per visible GPU, for example `[0, 0, 1, 1]`
for a 4-GPU system with GPUs 0-1 on NUMA node 0 and GPUs 2-3 on
NUMA node 1. If unset and `numa_bind=True`, vLLM auto-detects the
GPU-to-NUMA topology. The values are passed to `numactl --membind`
and `--cpunodebind`, so they must be valid `numactl` NUMA node indices.
"""
numa_bind_cpus: list[str] | None = None
"""Optional CPU lists to bind each GPU worker to.
Specify one CPU list per visible GPU, for example
`["0-3", "4-7", "8-11", "12-15"]`. When set, vLLM uses
`numactl --physcpubind` instead of `--cpunodebind`. This is useful
for custom policies such as binding to PCT or other high-frequency cores.
Each entry must use `numactl --physcpubind` CPU-list syntax, for example
`"0-3"` or `"0,2,4-7"`.
"""
distributed_timeout_seconds: int | None = None
"""Timeout in seconds for distributed operations (e.g., init_process_group).
@@ -345,6 +368,43 @@ class ParallelConfig:
"""Skip validation if the value is `None` when initialisation is delayed."""
return None if value is None else handler(value)
@field_validator("numa_bind_nodes")
@classmethod
def _validate_numa_bind_nodes(cls, value: list[int] | None) -> list[int] | None:
if value is None:
return None
if not value:
raise ValueError("numa_bind_nodes must not be empty.")
if any(node < 0 for node in value):
raise ValueError("numa_bind_nodes must contain non-negative integers.")
return value
@field_validator("numa_bind_cpus")
@classmethod
def _validate_numa_bind_cpus(cls, value: list[str] | None) -> list[str] | None:
if value is None:
return None
if not value:
raise ValueError("numa_bind_cpus must not be empty.")
for cpuset in value:
if not cpuset:
raise ValueError("numa_bind_cpus entries must not be empty.")
if not _NUMACTL_CPUSET_PATTERN.fullmatch(cpuset):
raise ValueError(
"numa_bind_cpus entries must use numactl CPU list syntax, "
"for example '0-3' or '0,2,4-7'."
)
for part in cpuset.split(","):
if "-" not in part:
continue
start_str, end_str = part.split("-", 1)
if int(start_str) > int(end_str):
raise ValueError(
f"numa_bind_cpus ranges must be ascending, but got '{cpuset}'."
)
return value
@model_validator(mode="after")
def _validate_parallel_config(self) -> Self:
if self._api_process_rank >= self._api_process_count:
@@ -373,6 +433,13 @@ class ParallelConfig:
"data_parallel_external_lb can only be set when data_parallel_size > 1"
)
if not self.numa_bind and (
self.numa_bind_nodes is not None or self.numa_bind_cpus is not None
):
raise ValueError(
"numa_bind_nodes and numa_bind_cpus require numa_bind=True."
)
if self.enable_eplb:
if not current_platform.is_cuda_alike():
raise ValueError(