[Bugfix] Share PCP-aware local GPU indexing for NUMA binding (#61237)

This commit is contained in:
Sherif Waly
2026-10-14 10:54:03 +02:00
committed by GitHub
parent 937d6c19df
commit be4ce2e505
6 changed files with 134 additions and 85 deletions
+16
View File
@@ -838,6 +838,22 @@ class ParallelConfig:
def local_world_size(self) -> int:
return self.world_size // self.nnodes_within_dp
def get_local_gpu_index(
self, local_rank: int, dp_local_rank: int | None = None
) -> int:
"""Map a worker's local rank to its host-local logical GPU index."""
if (
self.distributed_executor_backend not in ("ray", "external_launcher")
and self.data_parallel_backend != "ray"
and self.nnodes_within_dp == 1
):
if dp_local_rank is None:
dp_local_rank = self.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = self.data_parallel_index
return local_rank + dp_local_rank * self.world_size
return local_rank
@staticmethod
def has_unfinished_dp(dp_group: ProcessGroup, has_unfinished: bool) -> bool:
tensor = torch.tensor([has_unfinished], dtype=torch.int32, device="cpu")
+3 -35
View File
@@ -223,29 +223,6 @@ def _pct_sku_config() -> _PctSku | None:
return sku
def _get_gpu_index(
parallel_config, local_rank: int, dp_local_rank: int | None = None
) -> int:
"""Compute the physical GPU index used for NUMA lookup."""
if (
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
and parallel_config.data_parallel_backend != "ray"
and parallel_config.nnodes_within_dp == 1
):
if dp_local_rank is None:
dp_local_rank = parallel_config.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = parallel_config.data_parallel_index
tp_pp_world_size = (
parallel_config.pipeline_parallel_size
* parallel_config.tensor_parallel_size
)
return local_rank + dp_local_rank * tp_pp_world_size
return local_rank
def _get_numa_node(parallel_config, gpu_index: int) -> int:
numa_nodes = parallel_config.numa_bind_nodes
if numa_nodes is None:
@@ -346,7 +323,7 @@ def _get_numactl_worker_args(
parallel_config, local_rank: int, dp_local_rank: int | None = None
) -> str:
"""Compute the numactl args for a single TP/PP worker subprocess."""
gpu_index = _get_gpu_index(parallel_config, local_rank, dp_local_rank)
gpu_index = parallel_config.get_local_gpu_index(local_rank, dp_local_rank)
numa_node = _get_numa_node(parallel_config, gpu_index)
cpu_binding = _get_cpu_binding(parallel_config, gpu_index, [numa_node])
@@ -384,17 +361,8 @@ def _get_enginecore_numa_nodes(
and parallel_config.data_parallel_backend != "ray"
and parallel_config.nnodes_within_dp == 1
):
if dp_local_rank is None:
dp_local_rank = parallel_config.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = parallel_config.data_parallel_index
tp_pp_world_size = (
parallel_config.pipeline_parallel_size
* parallel_config.tensor_parallel_size
)
shard_start = dp_local_rank * tp_pp_world_size
shard_end = min(shard_start + tp_pp_world_size, len(numa_nodes))
shard_start = parallel_config.get_local_gpu_index(0, dp_local_rank)
shard_end = min(shard_start + parallel_config.world_size, len(numa_nodes))
shard_indices: range | tuple[int, ...] = range(shard_start, shard_end)
else:
shard_indices = range(len(numa_nodes))
+3 -28
View File
@@ -191,32 +191,6 @@ def set_worker_gpu_nic_mapping(local_rank: int) -> None:
)
def _dp_adjusted_local_rank(tp_local_rank: int, vllm_config: VllmConfig) -> int:
"""Compute the node-wide GPU index accounting for data parallelism.
On CUDA-alike platforms without env-var device isolation (the common
MP-backend path), the worker sees *all* GPUs on the node and selects
its device via ``torch.accelerator.set_device_index()`` using::
dp_local_rank * tp_pp_world_size + tp_local_rank
This mirrors the adjustment in ``Worker.init_device()`` so we resolve
the correct GPU PCI address *before* the CUDA device is initialised.
"""
pc = vllm_config.parallel_config
if (
pc.distributed_executor_backend not in ("ray", "external_launcher")
and pc.data_parallel_backend != "ray"
and pc.nnodes_within_dp == 1
):
dp_local_rank = pc.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = pc.data_parallel_index
tp_pp_world_size = pc.pipeline_parallel_size * pc.tensor_parallel_size
return dp_local_rank * tp_pp_world_size + tp_local_rank
return tp_local_rank
def set_worker_net_device(local_rank: int, vllm_config: VllmConfig) -> None:
"""Top-level entry point for both UniProcExecutor and MultiprocExecutor.
@@ -239,5 +213,6 @@ def set_worker_net_device(local_rank: int, vllm_config: VllmConfig) -> None:
if not has_pcie_mapping and not has_selection_vars:
return
# Both env vars are present, so set the NIC selection env vars.
adjusted_rank = _dp_adjusted_local_rank(local_rank, vllm_config)
set_worker_gpu_nic_mapping(adjusted_rank)
# Resolve the node-wide GPU index before the CUDA device is initialised.
gpu_index = vllm_config.parallel_config.get_local_gpu_index(local_rank)
set_worker_gpu_nic_mapping(gpu_index)
+1 -19
View File
@@ -444,25 +444,7 @@ class Worker(WorkerBase):
# This env var set by Ray causes exceptions with graph building.
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
parallel_config = self.parallel_config
if (
parallel_config.distributed_executor_backend
not in ("ray", "external_launcher")
and parallel_config.data_parallel_backend != "ray"
and parallel_config.nnodes_within_dp == 1
):
# Use local DP rank if available, otherwise use global DP rank.
dp_local_rank = self.parallel_config.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = self.parallel_config.data_parallel_index
tp_pcp_pp_world_size = (
self.parallel_config.pipeline_parallel_size
* self.parallel_config.prefill_context_parallel_size
* self.parallel_config.tensor_parallel_size
)
# DP_LOCAL_RANK * TP_PCP_PP_WORLD_SIZE + TP_LOCAL_RANK
self.local_rank += dp_local_rank * tp_pcp_pp_world_size
self.local_rank = parallel_config.get_local_gpu_index(self.local_rank)
# Publish the logical-to-physical mapping for topology queries
# such as NIC affinity and P2P checks.