[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
@@ -407,21 +406,17 @@ def _build_llava_or_pixtral_hf_info(
|
||||
def _build_llava_or_pixtral_hf_processor(
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> BaseMultiModalProcessor:
|
||||
if isinstance(info, PixtralHFProcessingInfo):
|
||||
return PixtralHFMultiModalProcessor(
|
||||
info,
|
||||
dummy_inputs, # type: ignore
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
if isinstance(info, LlavaProcessingInfo):
|
||||
return LlavaMultiModalProcessor(
|
||||
info,
|
||||
dummy_inputs, # type: ignore
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
raise NotImplementedError(type(info))
|
||||
|
||||
@@ -25,21 +25,10 @@ import torch
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsRealtime,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_asr import (
|
||||
Qwen3ASRDummyInputsBuilder,
|
||||
Qwen3ASRForConditionalGeneration,
|
||||
Qwen3ASRMultiModalProcessor,
|
||||
Qwen3ASRProcessingInfo,
|
||||
_get_feat_extract_output_lengths,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import ProcessorInputs, TimingContext
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseDummyInputsBuilder,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
cached_encode,
|
||||
@@ -47,6 +36,15 @@ from vllm.multimodal.processing.processor import (
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
|
||||
from .interfaces import SupportsRealtime
|
||||
from .qwen3_asr import (
|
||||
Qwen3ASRDummyInputsBuilder,
|
||||
Qwen3ASRForConditionalGeneration,
|
||||
Qwen3ASRMultiModalProcessor,
|
||||
Qwen3ASRProcessingInfo,
|
||||
_get_feat_extract_output_lengths,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60
|
||||
@@ -101,14 +99,13 @@ class Qwen3ASRRealtimeBuffer:
|
||||
|
||||
|
||||
class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
|
||||
def __init__(
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
super().__init__(info, dummy_inputs, cache=None)
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingResult:
|
||||
# realtime can't make use of a cache yet
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -27,9 +27,8 @@ from vllm.model_executor.models.voxtral import (
|
||||
VoxtralProcessingInfo,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing import ProcessorInputs, TimingContext
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
@@ -38,23 +37,19 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from .utils import (
|
||||
_flatten_embeddings,
|
||||
)
|
||||
from .utils import _flatten_embeddings
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
def __init__(
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingResult:
|
||||
# realtime can't make use of a cache yet
|
||||
super().__init__(info, dummy_inputs, cache=None)
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user