[Core] Bound UniProc EngineCore startup threads to available CPUs (#58946)
Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
This commit is contained in:
+66
-28
@@ -28,9 +28,9 @@ from vllm.utils.network_utils import (
|
||||
get_tcp_uri,
|
||||
zmq_socket_ctx,
|
||||
)
|
||||
from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.utils.system_utils import get_mp_context, set_env_var
|
||||
from vllm.v1.engine.coordinator import DPCoordinator
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.executor import Executor, UniProcExecutor
|
||||
from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS
|
||||
from vllm.v1.utils import _SubprocessWrapper, get_engine_client_zmq_addr, shutdown
|
||||
|
||||
@@ -166,6 +166,42 @@ def _node_ip_from_resources(node_resources: dict) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _configure_uniproc_startup_threads(
|
||||
executor_class: type[Executor], local_engine_count: int
|
||||
) -> Iterator[None]:
|
||||
if (
|
||||
not issubclass(executor_class, UniProcExecutor)
|
||||
or current_platform.is_cpu()
|
||||
or "OMP_NUM_THREADS" in os.environ
|
||||
):
|
||||
yield
|
||||
return
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import (
|
||||
OMP_NUM_THREADS_SET_BY_VLLM,
|
||||
set_default_torch_num_threads,
|
||||
startup_omp_num_threads,
|
||||
)
|
||||
|
||||
original_threads = torch.get_num_threads()
|
||||
num_threads = min(original_threads, startup_omp_num_threads(local_engine_count))
|
||||
torch_threads = (
|
||||
set_default_torch_num_threads(num_threads)
|
||||
if num_threads < original_threads
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
# Fork inherits Torch's setting; spawn reads OMP_NUM_THREADS on import.
|
||||
with (
|
||||
set_env_var("OMP_NUM_THREADS", str(num_threads)),
|
||||
set_env_var(OMP_NUM_THREADS_SET_BY_VLLM, "1"),
|
||||
torch_threads,
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
class CoreEngineProcManager:
|
||||
"""Utility class to handle creation, readiness, and shutdown
|
||||
of background processes used by the AsyncLLM and LLMEngine.
|
||||
@@ -229,34 +265,36 @@ class CoreEngineProcManager:
|
||||
# pickles process args at start() time, sequentially per rank.
|
||||
user_assigned_gpu_ids = vllm_config.parallel_config.assigned_physical_gpu_ids
|
||||
try:
|
||||
for proc, local_dp_rank in zip(self.processes, local_dp_ranks):
|
||||
# Populate the logical-to-physical GPU mapping in DP for
|
||||
# platforms that cannot rely on
|
||||
# torch.accelerator.set_device_index(), and for Ray.
|
||||
needs_device_env_isolation = not (
|
||||
current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
)
|
||||
if is_dp and (
|
||||
needs_device_env_isolation or vllm_config.parallel_config.use_ray
|
||||
):
|
||||
set_assigned_physical_gpu_ids_for_dp_rank(
|
||||
vllm_config, local_dp_rank, user_assigned_gpu_ids
|
||||
with _configure_uniproc_startup_threads(executor_class, local_engine_count):
|
||||
for proc, local_dp_rank in zip(self.processes, local_dp_ranks):
|
||||
# Populate the logical-to-physical GPU mapping in DP for
|
||||
# platforms that cannot rely on
|
||||
# torch.accelerator.set_device_index(), and for Ray.
|
||||
needs_device_env_isolation = not (
|
||||
current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
)
|
||||
if is_dp and (
|
||||
needs_device_env_isolation
|
||||
or vllm_config.parallel_config.use_ray
|
||||
):
|
||||
set_assigned_physical_gpu_ids_for_dp_rank(
|
||||
vllm_config, local_dp_rank, user_assigned_gpu_ids
|
||||
)
|
||||
|
||||
with numa_utils.configure_subprocess(
|
||||
# EngineCore itself does not have a TP/PP-local rank.
|
||||
# When DP is enabled, set_assigned_physical_gpu_ids_for_dp_rank()
|
||||
# populates the logical-to-physical mapping for this DP
|
||||
# shard, so local_rank=0 means "the first local GPU in
|
||||
# this shard". The actual TP/PP worker processes spawned
|
||||
# by the executor are bound separately with their own
|
||||
# local_rank values.
|
||||
vllm_config,
|
||||
local_rank=0,
|
||||
dp_local_rank=local_dp_rank,
|
||||
process_kind="EngineCore",
|
||||
):
|
||||
proc.start()
|
||||
with numa_utils.configure_subprocess(
|
||||
# EngineCore itself does not have a TP/PP-local rank.
|
||||
# set_assigned_physical_gpu_ids_for_dp_rank() populates
|
||||
# the logical-to-physical mapping for this DP
|
||||
# shard, so local_rank=0 means "the first local GPU in
|
||||
# this shard". The actual TP/PP worker processes spawned
|
||||
# by the executor are bound separately with their own
|
||||
# local_rank values.
|
||||
vllm_config,
|
||||
local_rank=0,
|
||||
dp_local_rank=local_dp_rank,
|
||||
process_kind="EngineCore",
|
||||
):
|
||||
proc.start()
|
||||
finally:
|
||||
# Kill other procs if not all are running.
|
||||
if self.finished_procs():
|
||||
|
||||
Reference in New Issue
Block a user