Add tensor IPC transfer mechanism for multimodal data (#32104)
Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com> Signed-off-by: Brandon Pelfrey <brandonpelfrey@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
61e381dcf0
commit
80b70884eb
@@ -79,7 +79,7 @@ from vllm.config.model import (
|
||||
RunnerOption,
|
||||
TokenizerMode,
|
||||
)
|
||||
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode
|
||||
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC
|
||||
from vllm.config.observability import DetailedTraceModules
|
||||
from vllm.config.parallel import (
|
||||
All2AllBackend,
|
||||
@@ -509,6 +509,7 @@ class EngineArgs:
|
||||
io_processor_plugin: str | None = None
|
||||
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
|
||||
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
|
||||
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
|
||||
# LoRA fields
|
||||
enable_lora: bool = False
|
||||
max_loras: int = LoRAConfig.max_loras
|
||||
@@ -1097,6 +1098,9 @@ class EngineArgs:
|
||||
multimodal_group.add_argument(
|
||||
"--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
|
||||
)
|
||||
|
||||
# LoRA related configs
|
||||
lora_kwargs = get_kwargs(LoRAConfig)
|
||||
@@ -1423,6 +1427,7 @@ class EngineArgs:
|
||||
override_attention_dtype=self.override_attention_dtype,
|
||||
logits_processors=self.logits_processors,
|
||||
video_pruning_rate=self.video_pruning_rate,
|
||||
mm_tensor_ipc=self.mm_tensor_ipc,
|
||||
io_processor_plugin=self.io_processor_plugin,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user