[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -18,6 +18,7 @@ from typing import (
|
||||
import torch
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm.config import SchedulerConfig
|
||||
from vllm.inputs import (
|
||||
MultiModalEncDecInput,
|
||||
MultiModalHashes,
|
||||
@@ -1039,8 +1040,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: "BaseDummyInputsBuilder[_I]",
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1053,16 +1052,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
self.info = info
|
||||
self.dummy_inputs = dummy_inputs
|
||||
self.cache = cache
|
||||
|
||||
self.data_parser = self.info.get_data_parser()
|
||||
|
||||
def get_dummy_mm_inputs(
|
||||
self,
|
||||
mm_counts: Mapping[str, int],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
scheduler_config: "SchedulerConfig | None" = None,
|
||||
) -> MultiModalInput:
|
||||
"""Create dummy data for profiling the memory usage of a model."""
|
||||
model_config = self.info.ctx.model_config
|
||||
seq_len = model_config.max_model_len
|
||||
if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
|
||||
seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
|
||||
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
|
||||
processor_inputs = self.dummy_inputs.get_dummy_processor_inputs(
|
||||
seq_len=seq_len,
|
||||
mm_counts=mm_counts,
|
||||
mm_options=mm_config.limit_per_prompt,
|
||||
)
|
||||
processor_inputs.cache = cache
|
||||
|
||||
mm_inputs = self.apply(
|
||||
processor_inputs,
|
||||
timing_ctx=TimingContext(enabled=False),
|
||||
)
|
||||
|
||||
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
||||
total_len = len(prompt_token_ids)
|
||||
if total_len < seq_len:
|
||||
prompt_token_ids.extend([0] * (seq_len - total_len))
|
||||
|
||||
return mm_inputs
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> MultiModalInput:
|
||||
if isinstance(prompt, str):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -1076,6 +1109,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
return self.apply(processor_inputs, TimingContext(enabled=False))
|
||||
@@ -1420,7 +1454,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
"""
|
||||
cache = self.cache
|
||||
cache = inputs.cache
|
||||
has_passthrough_data = any(
|
||||
len(items.get_passthrough_data()) > 0
|
||||
for items in inputs.mm_data_items.values()
|
||||
@@ -1827,6 +1861,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
inputs.mm_uuid_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
media_io_kwargs=inputs.media_io_kwargs,
|
||||
cache=inputs.cache,
|
||||
)
|
||||
|
||||
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
|
||||
|
||||
Reference in New Issue
Block a user