[Feature] NUMA binding support for GPU workers (#38635)
Signed-off-by: Shengqi Chen <harry-chen@outlook.com> Co-authored-by: Jason Li <jasonlizhengjian@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Jason Li
Roger Wang
parent
512c5eb455
commit
75e01a39a1
@@ -30,6 +30,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tracing import instrument, maybe_init_worker_tracer
|
||||
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
|
||||
from vllm.utils import numa_utils
|
||||
from vllm.utils.gc_utils import (
|
||||
freeze_gc_heap,
|
||||
maybe_attach_gc_debug_callback,
|
||||
@@ -1055,6 +1056,8 @@ class EngineCoreProc(EngineCore):
|
||||
set_process_title(process_title)
|
||||
maybe_init_worker_tracer("vllm.engine_core", "engine_core", process_title)
|
||||
decorate_logs()
|
||||
if parallel_config.numa_bind:
|
||||
numa_utils.log_current_affinity_state(process_title)
|
||||
|
||||
if data_parallel and vllm_config.kv_transfer_config is not None:
|
||||
# modify the engine_id and append the local_dp_rank to it to ensure
|
||||
@@ -1255,8 +1258,9 @@ class EngineCoreProc(EngineCore):
|
||||
return
|
||||
output = UtilityOutput(call_id)
|
||||
# Lazily look-up utility method so that failure will be handled/returned.
|
||||
get_result = lambda: (method := getattr(self, method_name)) and method(
|
||||
*self._convert_msgspec_args(method, args)
|
||||
get_result = lambda: (
|
||||
(method := getattr(self, method_name))
|
||||
and method(*self._convert_msgspec_args(method, args))
|
||||
)
|
||||
enqueue_output = lambda out: self.output_queue.put_nowait(
|
||||
(client_idx, EngineCoreOutputs(utility_output=out))
|
||||
|
||||
+24
-3
@@ -22,6 +22,7 @@ from vllm.config import CacheConfig, ParallelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.ray.ray_env import get_env_vars_to_copy
|
||||
from vllm.utils import numa_utils
|
||||
from vllm.utils.network_utils import get_open_zmq_ipc_path, zmq_socket_ctx
|
||||
from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.v1.engine.coordinator import DPCoordinator
|
||||
@@ -141,13 +142,33 @@ class CoreEngineProcManager:
|
||||
# Adjust device control in DP for non-CUDA platforms
|
||||
# as well as external and ray launchers
|
||||
# For CUDA platforms, we use torch.accelerator.set_device_index()()
|
||||
device_control_context: contextlib.AbstractContextManager[None] = (
|
||||
contextlib.nullcontext()
|
||||
)
|
||||
if is_dp and (
|
||||
not current_platform.is_cuda_alike()
|
||||
or vllm_config.parallel_config.use_ray
|
||||
):
|
||||
with set_device_control_env_var(vllm_config, local_dp_rank):
|
||||
proc.start()
|
||||
else:
|
||||
device_control_context = set_device_control_env_var(
|
||||
vllm_config, local_dp_rank
|
||||
)
|
||||
|
||||
with (
|
||||
device_control_context,
|
||||
numa_utils.configure_subprocess(
|
||||
# EngineCore itself does not have a TP/PP-local rank.
|
||||
# When DP is enabled, set_device_control_env_var()
|
||||
# narrows visible devices to this DP shard first, so
|
||||
# local_rank=0 means "the first local GPU in this
|
||||
# shard". The actual TP/PP worker processes spawned by
|
||||
# the executor are bound separately with their own
|
||||
# local_rank values.
|
||||
vllm_config,
|
||||
local_rank=0,
|
||||
dp_local_rank=local_dp_rank,
|
||||
process_kind="EngineCore",
|
||||
),
|
||||
):
|
||||
proc.start()
|
||||
finally:
|
||||
# Kill other procs if not all are running.
|
||||
|
||||
Reference in New Issue
Block a user