[Refactor] Use dynamic MM cache in processor (#57674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-19 15:38:00 +00:00
committed by GitHub
parent 41c4a3ed4e
commit 751f6807d9
23 changed files with 343 additions and 401 deletions
-5
View File
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import BaseMultiModalProcessorCache
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
@@ -407,21 +406,17 @@ def _build_llava_or_pixtral_hf_info(
def _build_llava_or_pixtral_hf_processor(
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor:
if isinstance(info, PixtralHFProcessingInfo):
return PixtralHFMultiModalProcessor(
info,
dummy_inputs, # type: ignore
cache=cache,
)
if isinstance(info, LlavaProcessingInfo):
return LlavaMultiModalProcessor(
info,
dummy_inputs, # type: ignore
cache=cache,
)
raise NotImplementedError(type(info))
@@ -25,21 +25,10 @@ import torch
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.inputs import PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
SupportsRealtime,
)
from vllm.model_executor.models.qwen3_asr import (
Qwen3ASRDummyInputsBuilder,
Qwen3ASRForConditionalGeneration,
Qwen3ASRMultiModalProcessor,
Qwen3ASRProcessingInfo,
_get_feat_extract_output_lengths,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import ProcessorInputs, TimingContext
from vllm.multimodal.processing.processor import (
BaseDummyInputsBuilder,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
cached_encode,
@@ -47,6 +36,15 @@ from vllm.multimodal.processing.processor import (
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config
from .interfaces import SupportsRealtime
from .qwen3_asr import (
Qwen3ASRDummyInputsBuilder,
Qwen3ASRForConditionalGeneration,
Qwen3ASRMultiModalProcessor,
Qwen3ASRProcessingInfo,
_get_feat_extract_output_lengths,
)
logger = init_logger(__name__)
_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60
@@ -101,14 +99,13 @@ class Qwen3ASRRealtimeBuffer:
class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
def __init__(
def _cached_apply_hf_processor(
self,
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> None:
super().__init__(info, dummy_inputs, cache=None)
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingResult:
# realtime can't make use of a cache yet
return self._apply_hf_processor(inputs, timing_ctx)
def _maybe_apply_prompt_updates(
self,
+7 -12
View File
@@ -27,9 +27,8 @@ from vllm.model_executor.models.voxtral import (
VoxtralProcessingInfo,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing import ProcessorInputs, TimingContext
from vllm.multimodal.processing.processor import (
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
@@ -38,23 +37,19 @@ from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.utils.torch_utils import is_torch_equal_or_newer
from .utils import (
_flatten_embeddings,
)
from .utils import _flatten_embeddings
logger = init_logger(__name__)
class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
def __init__(
def _cached_apply_hf_processor(
self,
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> None:
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingResult:
# realtime can't make use of a cache yet
super().__init__(info, dummy_inputs, cache=None)
return self._apply_hf_processor(inputs, timing_ctx)
def _maybe_apply_prompt_updates(
self,