[Refactor] Use dynamic MM cache in processor (#57674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-19 15:38:00 +00:00
committed by GitHub
parent 41c4a3ed4e
commit 751f6807d9
23 changed files with 343 additions and 401 deletions
+39 -4
View File
@@ -18,6 +18,7 @@ from typing import (
import torch
from typing_extensions import TypeVar, assert_never
from vllm.config import SchedulerConfig
from vllm.inputs import (
MultiModalEncDecInput,
MultiModalHashes,
@@ -1039,8 +1040,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
info: _I,
dummy_inputs: "BaseDummyInputsBuilder[_I]",
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> None:
super().__init__()
@@ -1053,16 +1052,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self.info = info
self.dummy_inputs = dummy_inputs
self.cache = cache
self.data_parser = self.info.get_data_parser()
def get_dummy_mm_inputs(
self,
mm_counts: Mapping[str, int],
*,
cache: BaseMultiModalProcessorCache | None = None,
scheduler_config: "SchedulerConfig | None" = None,
) -> MultiModalInput:
"""Create dummy data for profiling the memory usage of a model."""
model_config = self.info.ctx.model_config
seq_len = model_config.max_model_len
if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
mm_config = model_config.get_multimodal_config()
processor_inputs = self.dummy_inputs.get_dummy_processor_inputs(
seq_len=seq_len,
mm_counts=mm_counts,
mm_options=mm_config.limit_per_prompt,
)
processor_inputs.cache = cache
mm_inputs = self.apply(
processor_inputs,
timing_ctx=TimingContext(enabled=False),
)
prompt_token_ids = mm_inputs["prompt_token_ids"]
total_len = len(prompt_token_ids)
if total_len < seq_len:
prompt_token_ids.extend([0] * (seq_len - total_len))
return mm_inputs
def __call__(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
mm_uuid_items: MultiModalUUIDItems | None = None,
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
cache: BaseMultiModalProcessorCache | None = None,
) -> MultiModalInput:
if isinstance(prompt, str):
tokenizer = self.info.get_tokenizer()
@@ -1076,6 +1109,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_items,
mm_uuid_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
cache=cache,
)
return self.apply(processor_inputs, TimingContext(enabled=False))
@@ -1420,7 +1454,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""Apply the HF processor on the full prompt text,
caching the results and reusing cached results.
"""
cache = self.cache
cache = inputs.cache
has_passthrough_data = any(
len(items.get_passthrough_data()) > 0
for items in inputs.mm_data_items.values()
@@ -1827,6 +1861,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
inputs.mm_uuid_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
media_io_kwargs=inputs.media_io_kwargs,
cache=inputs.cache,
)
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)