Add tensor IPC transfer mechanism for multimodal data (#32104)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com>
Signed-off-by: Brandon Pelfrey <brandonpelfrey@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Brandon Pelfrey
2026-03-21 20:10:20 +00:00
committed by GitHub
co-authored by Nick Hill
parent 61e381dcf0
commit 80b70884eb
13 changed files with 1430 additions and 25 deletions
+6 -1
View File
@@ -79,7 +79,7 @@ from vllm.config.model import (
RunnerOption,
TokenizerMode,
)
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC
from vllm.config.observability import DetailedTraceModules
from vllm.config.parallel import (
All2AllBackend,
@@ -509,6 +509,7 @@ class EngineArgs:
io_processor_plugin: str | None = None
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
# LoRA fields
enable_lora: bool = False
max_loras: int = LoRAConfig.max_loras
@@ -1097,6 +1098,9 @@ class EngineArgs:
multimodal_group.add_argument(
"--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"]
)
multimodal_group.add_argument(
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
)
# LoRA related configs
lora_kwargs = get_kwargs(LoRAConfig)
@@ -1423,6 +1427,7 @@ class EngineArgs:
override_attention_dtype=self.override_attention_dtype,
logits_processors=self.logits_processors,
video_pruning_rate=self.video_pruning_rate,
mm_tensor_ipc=self.mm_tensor_ipc,
io_processor_plugin=self.io_processor_plugin,
)