[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
@@ -407,21 +406,17 @@ def _build_llava_or_pixtral_hf_info(
|
||||
def _build_llava_or_pixtral_hf_processor(
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> BaseMultiModalProcessor:
|
||||
if isinstance(info, PixtralHFProcessingInfo):
|
||||
return PixtralHFMultiModalProcessor(
|
||||
info,
|
||||
dummy_inputs, # type: ignore
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
if isinstance(info, LlavaProcessingInfo):
|
||||
return LlavaMultiModalProcessor(
|
||||
info,
|
||||
dummy_inputs, # type: ignore
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
raise NotImplementedError(type(info))
|
||||
|
||||
@@ -25,21 +25,10 @@ import torch
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsRealtime,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_asr import (
|
||||
Qwen3ASRDummyInputsBuilder,
|
||||
Qwen3ASRForConditionalGeneration,
|
||||
Qwen3ASRMultiModalProcessor,
|
||||
Qwen3ASRProcessingInfo,
|
||||
_get_feat_extract_output_lengths,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import ProcessorInputs, TimingContext
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseDummyInputsBuilder,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
cached_encode,
|
||||
@@ -47,6 +36,15 @@ from vllm.multimodal.processing.processor import (
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
|
||||
from .interfaces import SupportsRealtime
|
||||
from .qwen3_asr import (
|
||||
Qwen3ASRDummyInputsBuilder,
|
||||
Qwen3ASRForConditionalGeneration,
|
||||
Qwen3ASRMultiModalProcessor,
|
||||
Qwen3ASRProcessingInfo,
|
||||
_get_feat_extract_output_lengths,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60
|
||||
@@ -101,14 +99,13 @@ class Qwen3ASRRealtimeBuffer:
|
||||
|
||||
|
||||
class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
|
||||
def __init__(
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
super().__init__(info, dummy_inputs, cache=None)
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingResult:
|
||||
# realtime can't make use of a cache yet
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -27,9 +27,8 @@ from vllm.model_executor.models.voxtral import (
|
||||
VoxtralProcessingInfo,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing import ProcessorInputs, TimingContext
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
@@ -38,23 +37,19 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from .utils import (
|
||||
_flatten_embeddings,
|
||||
)
|
||||
from .utils import _flatten_embeddings
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
def __init__(
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingResult:
|
||||
# realtime can't make use of a cache yet
|
||||
super().__init__(info, dummy_inputs, cache=None)
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.multimodal.registry import MultiModalRegistry
|
||||
@@ -14,26 +15,21 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_mm_max_toks_per_item(
|
||||
model_config: ModelConfig,
|
||||
mm_registry: MultiModalRegistry,
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_counts: Mapping[str, int],
|
||||
cache: BaseMultiModalProcessorCache | None,
|
||||
) -> Mapping[str, int]:
|
||||
"""Get the maximum number of tokens per data item from each modality based
|
||||
on underlying model configuration.
|
||||
"""
|
||||
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=model_config.max_model_len,
|
||||
seq_len=processor.info.ctx.model_config.max_model_len,
|
||||
mm_counts=mm_counts,
|
||||
)
|
||||
if max_tokens_per_item is not None:
|
||||
return max_tokens_per_item
|
||||
|
||||
mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
model_config,
|
||||
mm_counts=mm_counts,
|
||||
processor=processor,
|
||||
)
|
||||
mm_inputs = processor.get_dummy_mm_inputs(mm_counts, cache=cache)
|
||||
|
||||
return {
|
||||
modality: sum(item.get_num_embeds() for item in placeholders)
|
||||
@@ -65,7 +61,7 @@ class MultiModalBudget:
|
||||
if enable_cache
|
||||
else None
|
||||
)
|
||||
processor = mm_registry.create_processor(model_config, cache=cache)
|
||||
processor = mm_registry.create_processor(model_config)
|
||||
|
||||
self.cache = cache
|
||||
self.processor = processor
|
||||
@@ -91,9 +87,8 @@ class MultiModalBudget:
|
||||
active_modalities = tower_modalities | embed_only_modalities
|
||||
|
||||
all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
model_config,
|
||||
mm_registry,
|
||||
processor,
|
||||
cache=cache,
|
||||
mm_counts=dict.fromkeys(active_modalities, 1),
|
||||
)
|
||||
|
||||
@@ -194,26 +189,32 @@ class MultiModalBudget:
|
||||
def get_encoder_budget(self) -> int:
|
||||
return min(self.encoder_compute_budget, self.encoder_cache_size)
|
||||
|
||||
def get_dummy_encoder_profile_inputs(
|
||||
self,
|
||||
modality: str | None = None,
|
||||
max_items_per_batch: int | None = None,
|
||||
) -> list[tuple[str, MultiModalKwargsItem]]:
|
||||
if self.get_encoder_budget() <= 0 or not self.mm_max_toks_per_item:
|
||||
return []
|
||||
|
||||
if modality is None:
|
||||
modality = self.get_modality_with_max_tokens()
|
||||
if max_items_per_batch is None:
|
||||
max_items_per_batch = self.mm_max_items_per_batch[modality]
|
||||
|
||||
# Don't use `max_items_per_batch` here to avoid redundant computation
|
||||
dummy_mm_inputs = self.processor.get_dummy_mm_inputs(
|
||||
mm_counts={modality: 1},
|
||||
cache=self.cache,
|
||||
)
|
||||
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
|
||||
|
||||
# We use the cache so that the item is saved to the cache,
|
||||
# but not read from the cache
|
||||
assert dummy_mm_item is not None, "Item should not already be cached"
|
||||
|
||||
return [(modality, dummy_mm_item)] * max_items_per_batch
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
if self.cache is not None:
|
||||
self.cache.clear_cache()
|
||||
|
||||
|
||||
def get_dummy_encoder_profile_inputs(
|
||||
mm_registry: MultiModalRegistry,
|
||||
budget: MultiModalBudget,
|
||||
) -> list[tuple[str, MultiModalKwargsItem]]:
|
||||
if budget.get_encoder_budget() <= 0 or not budget.mm_max_toks_per_item:
|
||||
return []
|
||||
|
||||
modality = budget.get_modality_with_max_tokens()
|
||||
max_items_per_batch = budget.mm_max_items_per_batch[modality]
|
||||
dummy_mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
budget.model_config,
|
||||
mm_counts={modality: 1},
|
||||
processor=budget.processor,
|
||||
)
|
||||
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
|
||||
assert dummy_mm_item is not None, "Dummy item should be generated"
|
||||
|
||||
return [(modality, dummy_mm_item)] * max_items_per_batch
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.inputs import MultiModalHashes
|
||||
|
||||
from ..cache import BaseMultiModalProcessorCache
|
||||
from ..hasher import MultiModalHasher
|
||||
from ..parse import MultiModalDataItems, MultiModalUUIDItems
|
||||
|
||||
@@ -28,6 +29,8 @@ class ProcessorInputs:
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)
|
||||
|
||||
cache: BaseMultiModalProcessorCache | None = None
|
||||
|
||||
def get_mm_hashes(
|
||||
self,
|
||||
model_id: str,
|
||||
|
||||
@@ -18,6 +18,7 @@ from typing import (
|
||||
import torch
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm.config import SchedulerConfig
|
||||
from vllm.inputs import (
|
||||
MultiModalEncDecInput,
|
||||
MultiModalHashes,
|
||||
@@ -1039,8 +1040,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: "BaseDummyInputsBuilder[_I]",
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1053,16 +1052,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
self.info = info
|
||||
self.dummy_inputs = dummy_inputs
|
||||
self.cache = cache
|
||||
|
||||
self.data_parser = self.info.get_data_parser()
|
||||
|
||||
def get_dummy_mm_inputs(
|
||||
self,
|
||||
mm_counts: Mapping[str, int],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
scheduler_config: "SchedulerConfig | None" = None,
|
||||
) -> MultiModalInput:
|
||||
"""Create dummy data for profiling the memory usage of a model."""
|
||||
model_config = self.info.ctx.model_config
|
||||
seq_len = model_config.max_model_len
|
||||
if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
|
||||
seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
|
||||
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
|
||||
processor_inputs = self.dummy_inputs.get_dummy_processor_inputs(
|
||||
seq_len=seq_len,
|
||||
mm_counts=mm_counts,
|
||||
mm_options=mm_config.limit_per_prompt,
|
||||
)
|
||||
processor_inputs.cache = cache
|
||||
|
||||
mm_inputs = self.apply(
|
||||
processor_inputs,
|
||||
timing_ctx=TimingContext(enabled=False),
|
||||
)
|
||||
|
||||
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
||||
total_len = len(prompt_token_ids)
|
||||
if total_len < seq_len:
|
||||
prompt_token_ids.extend([0] * (seq_len - total_len))
|
||||
|
||||
return mm_inputs
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> MultiModalInput:
|
||||
if isinstance(prompt, str):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -1076,6 +1109,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
return self.apply(processor_inputs, TimingContext(enabled=False))
|
||||
@@ -1420,7 +1454,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
"""
|
||||
cache = self.cache
|
||||
cache = inputs.cache
|
||||
has_passthrough_data = any(
|
||||
len(items.get_passthrough_data()) > 0
|
||||
for items in inputs.mm_data_items.values()
|
||||
@@ -1827,6 +1861,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
inputs.mm_uuid_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
media_io_kwargs=inputs.media_io_kwargs,
|
||||
cache=inputs.cache,
|
||||
)
|
||||
|
||||
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
|
||||
|
||||
@@ -2,12 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.synchronize import Lock as LockType
|
||||
from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
|
||||
|
||||
from vllm.inputs import MultiModalInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
|
||||
@@ -29,12 +27,7 @@ from .processing import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import (
|
||||
ModelConfig,
|
||||
ObservabilityConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config import ModelConfig, ObservabilityConfig, VllmConfig
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -75,8 +68,6 @@ class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> BaseMultiModalProcessor[_I]: ...
|
||||
|
||||
|
||||
@@ -89,12 +80,10 @@ class _ProcessorFactories(Generic[_I]):
|
||||
def build_processor(
|
||||
self,
|
||||
ctx: InputProcessingContext,
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
):
|
||||
info = self.info(ctx)
|
||||
dummy_inputs_builder = self.dummy_inputs(info)
|
||||
return self.processor(info, dummy_inputs_builder, cache=cache)
|
||||
return self.processor(info, dummy_inputs_builder)
|
||||
|
||||
|
||||
class MultiModalRegistry:
|
||||
@@ -218,7 +207,6 @@ class MultiModalRegistry:
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
|
||||
"""Create a multi-modal processor for a specific model and tokenizer."""
|
||||
if not model_config.is_multimodal_model:
|
||||
@@ -230,45 +218,7 @@ class MultiModalRegistry:
|
||||
|
||||
ctx = self._create_processing_ctx(model_config, tokenizer)
|
||||
|
||||
return factories.build_processor(ctx, cache=cache)
|
||||
|
||||
def get_dummy_mm_inputs(
|
||||
self,
|
||||
model_config: "ModelConfig",
|
||||
mm_counts: Mapping[str, int],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
processor: BaseMultiModalProcessor | None = None,
|
||||
scheduler_config: "SchedulerConfig | None" = None,
|
||||
) -> MultiModalInput:
|
||||
"""Create dummy data for profiling the memory usage of a model.
|
||||
|
||||
The model is identified by `model_config`.
|
||||
"""
|
||||
seq_len = model_config.max_model_len
|
||||
if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
|
||||
seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
|
||||
|
||||
if processor is None:
|
||||
processor = self.create_processor(model_config, cache=cache)
|
||||
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
|
||||
seq_len=seq_len,
|
||||
mm_counts=mm_counts,
|
||||
mm_options=mm_config.limit_per_prompt,
|
||||
)
|
||||
mm_inputs = processor.apply(
|
||||
processor_inputs,
|
||||
timing_ctx=TimingContext(enabled=False),
|
||||
)
|
||||
|
||||
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
||||
total_len = len(prompt_token_ids)
|
||||
if total_len < seq_len:
|
||||
prompt_token_ids.extend([0] * (seq_len - total_len))
|
||||
|
||||
return mm_inputs
|
||||
return factories.build_processor(ctx)
|
||||
|
||||
def _get_cache_type(
|
||||
self,
|
||||
|
||||
+42
-62
@@ -127,8 +127,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self._async_tokenizer_decode = make_async(self._decode, executor=self._executor)
|
||||
|
||||
self.mm_processor: BaseMultiModalProcessor | None = None
|
||||
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
|
||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||
|
||||
self._clear_mm_cache_async = make_async(
|
||||
self.clear_mm_cache, executor=self._mm_executor
|
||||
)
|
||||
@@ -138,6 +137,16 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self._safe_load_prompt_embeds_async = make_async(
|
||||
safe_load_prompt_embeds, executor=self._executor
|
||||
)
|
||||
|
||||
self._mm_processor_cache: BaseMultiModalProcessorCache | None = None
|
||||
|
||||
# A second processor with its own processor-only cache.
|
||||
# Used by the tokenize endpoint so that tokenize-only
|
||||
# requests don't pollute the sender cache.
|
||||
self._mm_processor_only_cache: BaseMultiModalProcessorCache | None = None
|
||||
|
||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||
|
||||
if mm_registry.supports_multimodal_inputs(config.model_config):
|
||||
# Install the process-global GPU memory pool used to gate
|
||||
# frontend GPU-side multimodal decoding (no-op when the budget
|
||||
@@ -149,30 +158,16 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
config.parallel_config._api_process_count,
|
||||
)
|
||||
|
||||
mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
|
||||
with set_default_torch_num_threads():
|
||||
self.mm_processor = mm_registry.create_processor(
|
||||
config.model_config,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=mm_processor_cache,
|
||||
)
|
||||
|
||||
if mm_processor_cache:
|
||||
self._mm_cache_stats = MultiModalCacheStats()
|
||||
self._resources.callback(mm_processor_cache.close)
|
||||
|
||||
# A second processor with its own processor-only cache.
|
||||
# Used by the tokenize endpoint so that tokenize-only
|
||||
# requests don't pollute the sender cache.
|
||||
ro_cache = mm_registry.processor_only_cache_from_config(config)
|
||||
if ro_cache is not None:
|
||||
with set_default_torch_num_threads():
|
||||
self._readonly_mm_processor = mm_registry.create_processor(
|
||||
config.model_config,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=ro_cache,
|
||||
)
|
||||
self._mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
self._mm_processor_only_cache = (
|
||||
mm_registry.processor_only_cache_from_config(config)
|
||||
)
|
||||
|
||||
# This is used to generate internal request ID for MM processing
|
||||
# It has no relation to the request ID for engine core
|
||||
@@ -181,6 +176,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
config.observability_config
|
||||
)
|
||||
|
||||
if self._mm_processor_cache:
|
||||
self._mm_cache_stats = MultiModalCacheStats()
|
||||
self._resources.callback(self._mm_processor_cache.close)
|
||||
if self._mm_processor_only_cache:
|
||||
self._resources.callback(self._mm_processor_only_cache.close)
|
||||
|
||||
def get_tokenizer(self) -> _T:
|
||||
tokenizer = self.tokenizer
|
||||
if tokenizer is None:
|
||||
@@ -198,11 +199,8 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return self.mm_processor
|
||||
|
||||
@property
|
||||
def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None":
|
||||
if self.mm_processor is None:
|
||||
return None
|
||||
|
||||
return self.mm_processor.cache
|
||||
def mm_processor_cache(self) -> BaseMultiModalProcessorCache | None:
|
||||
return self._mm_processor_cache
|
||||
|
||||
def stat_mm_cache(self) -> MultiModalCacheStats | None:
|
||||
mm_cache_stats = self._mm_cache_stats
|
||||
@@ -214,7 +212,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return mm_cache_stats
|
||||
|
||||
def update_mm_cache_stats(self) -> None:
|
||||
mm_processor_cache = self.mm_processor_cache
|
||||
mm_processor_cache = self._mm_processor_cache
|
||||
mm_cache_stats = self._mm_cache_stats
|
||||
|
||||
if mm_processor_cache and mm_cache_stats:
|
||||
@@ -222,38 +220,26 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
mm_cache_stats.record(delta.total, delta.hits)
|
||||
|
||||
def clear_mm_cache(self) -> None:
|
||||
mm_processor_cache = self.mm_processor_cache
|
||||
mm_processor_cache = self._mm_processor_cache
|
||||
if mm_processor_cache is not None:
|
||||
mm_processor_cache.clear_cache()
|
||||
|
||||
if self._mm_cache_stats is not None:
|
||||
self._mm_cache_stats.reset = True
|
||||
|
||||
@staticmethod
|
||||
def _clear_processor_cache(
|
||||
processor: "BaseMultiModalProcessor | None",
|
||||
) -> None:
|
||||
if processor is None:
|
||||
return
|
||||
|
||||
processor_cache = processor.cache
|
||||
if processor_cache is not None:
|
||||
processor_cache.clear_cache()
|
||||
|
||||
def _warmup_mm_processor(
|
||||
self,
|
||||
processor: "BaseMultiModalProcessor",
|
||||
processor: BaseMultiModalProcessor,
|
||||
cache: BaseMultiModalProcessorCache | None,
|
||||
*,
|
||||
log_prefix: str,
|
||||
) -> None:
|
||||
model_config = self.model_config
|
||||
mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
_ = mm_registry.get_dummy_mm_inputs(
|
||||
model_config,
|
||||
mm_counts=dict.fromkeys(mm_limits, 1),
|
||||
processor=processor,
|
||||
_ = processor.get_dummy_mm_inputs(
|
||||
dict.fromkeys(mm_limits, 1),
|
||||
cache=cache,
|
||||
scheduler_config=self.config.scheduler_config,
|
||||
)
|
||||
|
||||
@@ -274,28 +260,20 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# prevent MM processor hangs
|
||||
with set_default_torch_num_threads(1):
|
||||
if self.mm_processor:
|
||||
mm_processor_only_cache = self._mm_processor_only_cache
|
||||
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
mm_processor_only_cache,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
if mm_processor_only_cache:
|
||||
mm_processor_only_cache.clear_cache()
|
||||
finally:
|
||||
self._mm_warmup_done = True
|
||||
|
||||
@@ -323,7 +301,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# which is not designed for concurrent access (its get_and_update must
|
||||
# stay ordered with the engine-core cache) — is never touched
|
||||
# concurrently by the warmup and the serving path.
|
||||
if self.mm_processor is None and self._readonly_mm_processor is None:
|
||||
if self.mm_processor is None:
|
||||
return # text-only model: nothing to warm up
|
||||
if self._mm_warmup_future is not None:
|
||||
return # already launched
|
||||
@@ -853,10 +831,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> "MultiModalInput":
|
||||
if skip_mm_cache and self._readonly_mm_processor is not None:
|
||||
mm_processor = self._readonly_mm_processor
|
||||
else:
|
||||
mm_processor = self.get_mm_processor()
|
||||
mm_processor = self.get_mm_processor()
|
||||
|
||||
mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
|
||||
|
||||
@@ -873,6 +848,11 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs or {},
|
||||
media_io_kwargs=media_io_kwargs or {},
|
||||
cache=(
|
||||
self._mm_processor_only_cache
|
||||
if skip_mm_cache
|
||||
else self._mm_processor_cache
|
||||
),
|
||||
)
|
||||
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
|
||||
|
||||
|
||||
@@ -57,7 +57,6 @@ from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.encoder_budget import (
|
||||
MultiModalBudget,
|
||||
get_dummy_encoder_profile_inputs,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
@@ -935,10 +934,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.mm_registry,
|
||||
enable_cache=False,
|
||||
)
|
||||
dummy_mm_inputs = get_dummy_encoder_profile_inputs(
|
||||
self.mm_registry,
|
||||
mm_budget,
|
||||
)
|
||||
dummy_mm_inputs = mm_budget.get_dummy_encoder_profile_inputs()
|
||||
self.model_state.encoder_runner.profile_encoder_cache(
|
||||
dummy_mm_inputs, mm_budget
|
||||
)
|
||||
|
||||
@@ -5837,21 +5837,15 @@ class GPUModelRunner(
|
||||
assert self.mm_budget is not None
|
||||
|
||||
# Don't use `max_items_per_batch` here to avoid redundant computation
|
||||
dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs(
|
||||
self.model_config,
|
||||
mm_counts={modality: 1},
|
||||
cache=self.mm_budget.cache,
|
||||
dummy_mm_inputs = self.mm_budget.get_dummy_encoder_profile_inputs(
|
||||
modality,
|
||||
max_items_per_batch,
|
||||
)
|
||||
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
|
||||
|
||||
# We use the cache so that the item is saved to the cache,
|
||||
# but not read from the cache
|
||||
assert dummy_mm_item is not None, "Item should not already be cached"
|
||||
|
||||
return next(
|
||||
mm_kwargs_batch
|
||||
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
[(modality, dummy_mm_item)] * max_items_per_batch,
|
||||
dummy_mm_inputs,
|
||||
device=self.device,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user