[Refactor] Use dynamic MM cache in processor (#57674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-19 15:38:00 +00:00
committed by GitHub
parent 41c4a3ed4e
commit 751f6807d9
23 changed files with 343 additions and 401 deletions
-5
View File
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import BaseMultiModalProcessorCache
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
@@ -407,21 +406,17 @@ def _build_llava_or_pixtral_hf_info(
def _build_llava_or_pixtral_hf_processor(
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor:
if isinstance(info, PixtralHFProcessingInfo):
return PixtralHFMultiModalProcessor(
info,
dummy_inputs, # type: ignore
cache=cache,
)
if isinstance(info, LlavaProcessingInfo):
return LlavaMultiModalProcessor(
info,
dummy_inputs, # type: ignore
cache=cache,
)
raise NotImplementedError(type(info))
@@ -25,21 +25,10 @@ import torch
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.inputs import PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
SupportsRealtime,
)
from vllm.model_executor.models.qwen3_asr import (
Qwen3ASRDummyInputsBuilder,
Qwen3ASRForConditionalGeneration,
Qwen3ASRMultiModalProcessor,
Qwen3ASRProcessingInfo,
_get_feat_extract_output_lengths,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import ProcessorInputs, TimingContext
from vllm.multimodal.processing.processor import (
BaseDummyInputsBuilder,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
cached_encode,
@@ -47,6 +36,15 @@ from vllm.multimodal.processing.processor import (
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config
from .interfaces import SupportsRealtime
from .qwen3_asr import (
Qwen3ASRDummyInputsBuilder,
Qwen3ASRForConditionalGeneration,
Qwen3ASRMultiModalProcessor,
Qwen3ASRProcessingInfo,
_get_feat_extract_output_lengths,
)
logger = init_logger(__name__)
_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60
@@ -101,14 +99,13 @@ class Qwen3ASRRealtimeBuffer:
class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
def __init__(
def _cached_apply_hf_processor(
self,
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> None:
super().__init__(info, dummy_inputs, cache=None)
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingResult:
# realtime can't make use of a cache yet
return self._apply_hf_processor(inputs, timing_ctx)
def _maybe_apply_prompt_updates(
self,
+7 -12
View File
@@ -27,9 +27,8 @@ from vllm.model_executor.models.voxtral import (
VoxtralProcessingInfo,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing import ProcessorInputs, TimingContext
from vllm.multimodal.processing.processor import (
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
@@ -38,23 +37,19 @@ from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.utils.torch_utils import is_torch_equal_or_newer
from .utils import (
_flatten_embeddings,
)
from .utils import _flatten_embeddings
logger = init_logger(__name__)
class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
def __init__(
def _cached_apply_hf_processor(
self,
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> None:
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingResult:
# realtime can't make use of a cache yet
super().__init__(info, dummy_inputs, cache=None)
return self._apply_hf_processor(inputs, timing_ctx)
def _maybe_apply_prompt_updates(
self,
+33 -32
View File
@@ -2,8 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from vllm.config import ModelConfig, VllmConfig
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.multimodal.cache import BaseMultiModalProcessorCache
from vllm.multimodal.inputs import MultiModalKwargsItem
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.multimodal.registry import MultiModalRegistry
@@ -14,26 +15,21 @@ logger = init_logger(__name__)
def get_mm_max_toks_per_item(
model_config: ModelConfig,
mm_registry: MultiModalRegistry,
processor: BaseMultiModalProcessor,
mm_counts: Mapping[str, int],
cache: BaseMultiModalProcessorCache | None,
) -> Mapping[str, int]:
"""Get the maximum number of tokens per data item from each modality based
on underlying model configuration.
"""
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
seq_len=model_config.max_model_len,
seq_len=processor.info.ctx.model_config.max_model_len,
mm_counts=mm_counts,
)
if max_tokens_per_item is not None:
return max_tokens_per_item
mm_inputs = mm_registry.get_dummy_mm_inputs(
model_config,
mm_counts=mm_counts,
processor=processor,
)
mm_inputs = processor.get_dummy_mm_inputs(mm_counts, cache=cache)
return {
modality: sum(item.get_num_embeds() for item in placeholders)
@@ -65,7 +61,7 @@ class MultiModalBudget:
if enable_cache
else None
)
processor = mm_registry.create_processor(model_config, cache=cache)
processor = mm_registry.create_processor(model_config)
self.cache = cache
self.processor = processor
@@ -91,9 +87,8 @@ class MultiModalBudget:
active_modalities = tower_modalities | embed_only_modalities
all_mm_max_toks_per_item = get_mm_max_toks_per_item(
model_config,
mm_registry,
processor,
cache=cache,
mm_counts=dict.fromkeys(active_modalities, 1),
)
@@ -194,26 +189,32 @@ class MultiModalBudget:
def get_encoder_budget(self) -> int:
return min(self.encoder_compute_budget, self.encoder_cache_size)
def get_dummy_encoder_profile_inputs(
self,
modality: str | None = None,
max_items_per_batch: int | None = None,
) -> list[tuple[str, MultiModalKwargsItem]]:
if self.get_encoder_budget() <= 0 or not self.mm_max_toks_per_item:
return []
if modality is None:
modality = self.get_modality_with_max_tokens()
if max_items_per_batch is None:
max_items_per_batch = self.mm_max_items_per_batch[modality]
# Don't use `max_items_per_batch` here to avoid redundant computation
dummy_mm_inputs = self.processor.get_dummy_mm_inputs(
mm_counts={modality: 1},
cache=self.cache,
)
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
# We use the cache so that the item is saved to the cache,
# but not read from the cache
assert dummy_mm_item is not None, "Item should not already be cached"
return [(modality, dummy_mm_item)] * max_items_per_batch
def reset_cache(self) -> None:
if self.cache is not None:
self.cache.clear_cache()
def get_dummy_encoder_profile_inputs(
mm_registry: MultiModalRegistry,
budget: MultiModalBudget,
) -> list[tuple[str, MultiModalKwargsItem]]:
if budget.get_encoder_budget() <= 0 or not budget.mm_max_toks_per_item:
return []
modality = budget.get_modality_with_max_tokens()
max_items_per_batch = budget.mm_max_items_per_batch[modality]
dummy_mm_inputs = mm_registry.get_dummy_mm_inputs(
budget.model_config,
mm_counts={modality: 1},
processor=budget.processor,
)
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
assert dummy_mm_item is not None, "Dummy item should be generated"
return [(modality, dummy_mm_item)] * max_items_per_batch
+3
View File
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
from vllm.config.multimodal import MMHasherAlgorithm
from vllm.inputs import MultiModalHashes
from ..cache import BaseMultiModalProcessorCache
from ..hasher import MultiModalHasher
from ..parse import MultiModalDataItems, MultiModalUUIDItems
@@ -28,6 +29,8 @@ class ProcessorInputs:
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)
cache: BaseMultiModalProcessorCache | None = None
def get_mm_hashes(
self,
model_id: str,
+39 -4
View File
@@ -18,6 +18,7 @@ from typing import (
import torch
from typing_extensions import TypeVar, assert_never
from vllm.config import SchedulerConfig
from vllm.inputs import (
MultiModalEncDecInput,
MultiModalHashes,
@@ -1039,8 +1040,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
info: _I,
dummy_inputs: "BaseDummyInputsBuilder[_I]",
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> None:
super().__init__()
@@ -1053,16 +1052,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self.info = info
self.dummy_inputs = dummy_inputs
self.cache = cache
self.data_parser = self.info.get_data_parser()
def get_dummy_mm_inputs(
self,
mm_counts: Mapping[str, int],
*,
cache: BaseMultiModalProcessorCache | None = None,
scheduler_config: "SchedulerConfig | None" = None,
) -> MultiModalInput:
"""Create dummy data for profiling the memory usage of a model."""
model_config = self.info.ctx.model_config
seq_len = model_config.max_model_len
if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
mm_config = model_config.get_multimodal_config()
processor_inputs = self.dummy_inputs.get_dummy_processor_inputs(
seq_len=seq_len,
mm_counts=mm_counts,
mm_options=mm_config.limit_per_prompt,
)
processor_inputs.cache = cache
mm_inputs = self.apply(
processor_inputs,
timing_ctx=TimingContext(enabled=False),
)
prompt_token_ids = mm_inputs["prompt_token_ids"]
total_len = len(prompt_token_ids)
if total_len < seq_len:
prompt_token_ids.extend([0] * (seq_len - total_len))
return mm_inputs
def __call__(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
mm_uuid_items: MultiModalUUIDItems | None = None,
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
cache: BaseMultiModalProcessorCache | None = None,
) -> MultiModalInput:
if isinstance(prompt, str):
tokenizer = self.info.get_tokenizer()
@@ -1076,6 +1109,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_items,
mm_uuid_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
cache=cache,
)
return self.apply(processor_inputs, TimingContext(enabled=False))
@@ -1420,7 +1454,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""Apply the HF processor on the full prompt text,
caching the results and reusing cached results.
"""
cache = self.cache
cache = inputs.cache
has_passthrough_data = any(
len(items.get_passthrough_data()) > 0
for items in inputs.mm_data_items.values()
@@ -1827,6 +1861,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
inputs.mm_uuid_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
media_io_kwargs=inputs.media_io_kwargs,
cache=inputs.cache,
)
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
+3 -53
View File
@@ -2,12 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from collections import defaultdict
from collections.abc import Mapping
from dataclasses import dataclass
from multiprocessing.synchronize import Lock as LockType
from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
from vllm.inputs import MultiModalInput
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
@@ -29,12 +27,7 @@ from .processing import (
)
if TYPE_CHECKING:
from vllm.config import (
ModelConfig,
ObservabilityConfig,
SchedulerConfig,
VllmConfig,
)
from vllm.config import ModelConfig, ObservabilityConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
logger = init_logger(__name__)
@@ -75,8 +68,6 @@ class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
self,
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor[_I]: ...
@@ -89,12 +80,10 @@ class _ProcessorFactories(Generic[_I]):
def build_processor(
self,
ctx: InputProcessingContext,
*,
cache: BaseMultiModalProcessorCache | None = None,
):
info = self.info(ctx)
dummy_inputs_builder = self.dummy_inputs(info)
return self.processor(info, dummy_inputs_builder, cache=cache)
return self.processor(info, dummy_inputs_builder)
class MultiModalRegistry:
@@ -218,7 +207,6 @@ class MultiModalRegistry:
model_config: "ModelConfig",
*,
tokenizer: TokenizerLike | None = None,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
"""Create a multi-modal processor for a specific model and tokenizer."""
if not model_config.is_multimodal_model:
@@ -230,45 +218,7 @@ class MultiModalRegistry:
ctx = self._create_processing_ctx(model_config, tokenizer)
return factories.build_processor(ctx, cache=cache)
def get_dummy_mm_inputs(
self,
model_config: "ModelConfig",
mm_counts: Mapping[str, int],
*,
cache: BaseMultiModalProcessorCache | None = None,
processor: BaseMultiModalProcessor | None = None,
scheduler_config: "SchedulerConfig | None" = None,
) -> MultiModalInput:
"""Create dummy data for profiling the memory usage of a model.
The model is identified by `model_config`.
"""
seq_len = model_config.max_model_len
if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
if processor is None:
processor = self.create_processor(model_config, cache=cache)
mm_config = model_config.get_multimodal_config()
processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
seq_len=seq_len,
mm_counts=mm_counts,
mm_options=mm_config.limit_per_prompt,
)
mm_inputs = processor.apply(
processor_inputs,
timing_ctx=TimingContext(enabled=False),
)
prompt_token_ids = mm_inputs["prompt_token_ids"]
total_len = len(prompt_token_ids)
if total_len < seq_len:
prompt_token_ids.extend([0] * (seq_len - total_len))
return mm_inputs
return factories.build_processor(ctx)
def _get_cache_type(
self,
+42 -62
View File
@@ -127,8 +127,7 @@ class BaseRenderer(ABC, Generic[_T]):
self._async_tokenizer_decode = make_async(self._decode, executor=self._executor)
self.mm_processor: BaseMultiModalProcessor | None = None
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
self._mm_cache_stats: MultiModalCacheStats | None = None
self._clear_mm_cache_async = make_async(
self.clear_mm_cache, executor=self._mm_executor
)
@@ -138,6 +137,16 @@ class BaseRenderer(ABC, Generic[_T]):
self._safe_load_prompt_embeds_async = make_async(
safe_load_prompt_embeds, executor=self._executor
)
self._mm_processor_cache: BaseMultiModalProcessorCache | None = None
# A second processor with its own processor-only cache.
# Used by the tokenize endpoint so that tokenize-only
# requests don't pollute the sender cache.
self._mm_processor_only_cache: BaseMultiModalProcessorCache | None = None
self._mm_cache_stats: MultiModalCacheStats | None = None
if mm_registry.supports_multimodal_inputs(config.model_config):
# Install the process-global GPU memory pool used to gate
# frontend GPU-side multimodal decoding (no-op when the budget
@@ -149,30 +158,16 @@ class BaseRenderer(ABC, Generic[_T]):
config.parallel_config._api_process_count,
)
mm_processor_cache = mm_registry.processor_cache_from_config(config)
with set_default_torch_num_threads():
self.mm_processor = mm_registry.create_processor(
config.model_config,
tokenizer=self.tokenizer,
cache=mm_processor_cache,
)
if mm_processor_cache:
self._mm_cache_stats = MultiModalCacheStats()
self._resources.callback(mm_processor_cache.close)
# A second processor with its own processor-only cache.
# Used by the tokenize endpoint so that tokenize-only
# requests don't pollute the sender cache.
ro_cache = mm_registry.processor_only_cache_from_config(config)
if ro_cache is not None:
with set_default_torch_num_threads():
self._readonly_mm_processor = mm_registry.create_processor(
config.model_config,
tokenizer=self.tokenizer,
cache=ro_cache,
)
self._mm_processor_cache = mm_registry.processor_cache_from_config(config)
self._mm_processor_only_cache = (
mm_registry.processor_only_cache_from_config(config)
)
# This is used to generate internal request ID for MM processing
# It has no relation to the request ID for engine core
@@ -181,6 +176,12 @@ class BaseRenderer(ABC, Generic[_T]):
config.observability_config
)
if self._mm_processor_cache:
self._mm_cache_stats = MultiModalCacheStats()
self._resources.callback(self._mm_processor_cache.close)
if self._mm_processor_only_cache:
self._resources.callback(self._mm_processor_only_cache.close)
def get_tokenizer(self) -> _T:
tokenizer = self.tokenizer
if tokenizer is None:
@@ -198,11 +199,8 @@ class BaseRenderer(ABC, Generic[_T]):
return self.mm_processor
@property
def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None":
if self.mm_processor is None:
return None
return self.mm_processor.cache
def mm_processor_cache(self) -> BaseMultiModalProcessorCache | None:
return self._mm_processor_cache
def stat_mm_cache(self) -> MultiModalCacheStats | None:
mm_cache_stats = self._mm_cache_stats
@@ -214,7 +212,7 @@ class BaseRenderer(ABC, Generic[_T]):
return mm_cache_stats
def update_mm_cache_stats(self) -> None:
mm_processor_cache = self.mm_processor_cache
mm_processor_cache = self._mm_processor_cache
mm_cache_stats = self._mm_cache_stats
if mm_processor_cache and mm_cache_stats:
@@ -222,38 +220,26 @@ class BaseRenderer(ABC, Generic[_T]):
mm_cache_stats.record(delta.total, delta.hits)
def clear_mm_cache(self) -> None:
mm_processor_cache = self.mm_processor_cache
mm_processor_cache = self._mm_processor_cache
if mm_processor_cache is not None:
mm_processor_cache.clear_cache()
if self._mm_cache_stats is not None:
self._mm_cache_stats.reset = True
@staticmethod
def _clear_processor_cache(
processor: "BaseMultiModalProcessor | None",
) -> None:
if processor is None:
return
processor_cache = processor.cache
if processor_cache is not None:
processor_cache.clear_cache()
def _warmup_mm_processor(
self,
processor: "BaseMultiModalProcessor",
processor: BaseMultiModalProcessor,
cache: BaseMultiModalProcessorCache | None,
*,
log_prefix: str,
) -> None:
model_config = self.model_config
mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
start_time = time.perf_counter()
_ = mm_registry.get_dummy_mm_inputs(
model_config,
mm_counts=dict.fromkeys(mm_limits, 1),
processor=processor,
_ = processor.get_dummy_mm_inputs(
dict.fromkeys(mm_limits, 1),
cache=cache,
scheduler_config=self.config.scheduler_config,
)
@@ -274,28 +260,20 @@ class BaseRenderer(ABC, Generic[_T]):
# prevent MM processor hangs
with set_default_torch_num_threads(1):
if self.mm_processor:
mm_processor_only_cache = self._mm_processor_only_cache
try:
logger.debug("Warming up multi-modal processing...")
self._warmup_mm_processor(
self.mm_processor,
mm_processor_only_cache,
log_prefix="Multi-modal",
)
except Exception:
logger.warning("Multi-modal warmup failed")
finally:
self.clear_mm_cache()
if self._readonly_mm_processor is not None:
try:
logger.debug("Warming up readonly multi-modal processing...")
self._warmup_mm_processor(
self._readonly_mm_processor,
log_prefix="Readonly multi-modal",
)
except Exception:
logger.warning("Readonly multi-modal warmup failed")
finally:
self._clear_processor_cache(self._readonly_mm_processor)
if mm_processor_only_cache:
mm_processor_only_cache.clear_cache()
finally:
self._mm_warmup_done = True
@@ -323,7 +301,7 @@ class BaseRenderer(ABC, Generic[_T]):
# which is not designed for concurrent access (its get_and_update must
# stay ordered with the engine-core cache) — is never touched
# concurrently by the warmup and the serving path.
if self.mm_processor is None and self._readonly_mm_processor is None:
if self.mm_processor is None:
return # text-only model: nothing to warm up
if self._mm_warmup_future is not None:
return # already launched
@@ -853,10 +831,7 @@ class BaseRenderer(ABC, Generic[_T]):
*,
skip_mm_cache: bool = False,
) -> "MultiModalInput":
if skip_mm_cache and self._readonly_mm_processor is not None:
mm_processor = self._readonly_mm_processor
else:
mm_processor = self.get_mm_processor()
mm_processor = self.get_mm_processor()
mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
@@ -873,6 +848,11 @@ class BaseRenderer(ABC, Generic[_T]):
mm_uuid_items,
hf_processor_mm_kwargs=mm_processor_kwargs or {},
media_io_kwargs=media_io_kwargs or {},
cache=(
self._mm_processor_only_cache
if skip_mm_cache
else self._mm_processor_cache
),
)
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
+1 -5
View File
@@ -57,7 +57,6 @@ from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.encoder_budget import (
MultiModalBudget,
get_dummy_encoder_profile_inputs,
)
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
@@ -935,10 +934,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.mm_registry,
enable_cache=False,
)
dummy_mm_inputs = get_dummy_encoder_profile_inputs(
self.mm_registry,
mm_budget,
)
dummy_mm_inputs = mm_budget.get_dummy_encoder_profile_inputs()
self.model_state.encoder_runner.profile_encoder_cache(
dummy_mm_inputs, mm_budget
)
+4 -10
View File
@@ -5837,21 +5837,15 @@ class GPUModelRunner(
assert self.mm_budget is not None
# Don't use `max_items_per_batch` here to avoid redundant computation
dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs(
self.model_config,
mm_counts={modality: 1},
cache=self.mm_budget.cache,
dummy_mm_inputs = self.mm_budget.get_dummy_encoder_profile_inputs(
modality,
max_items_per_batch,
)
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
# We use the cache so that the item is saved to the cache,
# but not read from the cache
assert dummy_mm_item is not None, "Item should not already be cached"
return next(
mm_kwargs_batch
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
[(modality, dummy_mm_item)] * max_items_per_batch,
dummy_mm_inputs,
device=self.device,
pin_memory=PIN_MEMORY,
)