Add tensor IPC transfer mechanism for multimodal data (#32104)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com>
Signed-off-by: Brandon Pelfrey <brandonpelfrey@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Brandon Pelfrey
2026-03-21 20:10:20 +00:00
committed by GitHub
co-authored by Nick Hill
parent 61e381dcf0
commit 80b70884eb
13 changed files with 1430 additions and 25 deletions
+2 -1
View File
@@ -290,7 +290,7 @@ def run_multi_api_server(args: argparse.Namespace):
with launch_core_engines(
vllm_config, executor_class, log_stats, addresses, num_api_servers
) as (local_engine_manager, coordinator, addresses):
) as (local_engine_manager, coordinator, addresses, tensor_queue):
# Construct common args for the APIServerProcessManager up-front.
api_server_manager_kwargs = dict(
target_server_fn=run_api_server_worker_proc,
@@ -303,6 +303,7 @@ def run_multi_api_server(args: argparse.Namespace):
stats_update_address=coordinator.get_stats_publish_address()
if coordinator
else None,
tensor_queue=tensor_queue,
)
# For dp ranks > 0 in external/hybrid DP LB modes, we must delay the