[Renderer] Move InputPreprocessor into Renderer (1/2) (#34510)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
+23
-91
@@ -9,13 +9,11 @@ from typing_extensions import assert_never
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalInputs,
|
||||
MultiModalUUIDDict,
|
||||
)
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.renderers import BaseRenderer, renderer_from_config
|
||||
from vllm.renderers.inputs import (
|
||||
DecoderDictPrompt,
|
||||
@@ -28,8 +26,6 @@ from vllm.renderers.inputs import (
|
||||
)
|
||||
from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.jsontree import json_iter_leaves
|
||||
from vllm.v1.metrics.stats import MultiModalCacheStats
|
||||
|
||||
from .data import (
|
||||
DecoderInputs,
|
||||
@@ -57,17 +53,12 @@ class InputPreprocessor:
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer | None = None,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
mm_processor_cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.model_config = vllm_config.model_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
self.renderer = renderer or renderer_from_config(vllm_config)
|
||||
self.mm_registry = mm_registry
|
||||
self.mm_processor_cache = mm_processor_cache
|
||||
|
||||
self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> TokenizerLike | None:
|
||||
@@ -124,23 +115,6 @@ class InputPreprocessor:
|
||||
|
||||
return decoder_input_ids
|
||||
|
||||
def _get_tokenization_kw(
|
||||
self,
|
||||
overrides: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
kwargs = dict[str, Any]()
|
||||
|
||||
if self.model_config.is_encoder_decoder:
|
||||
# For Whisper, special tokens should be provided by the user based
|
||||
# on the task and language of their request. Also needed to avoid
|
||||
# appending an EOS token to the prompt which disrupts generation.
|
||||
kwargs["add_special_tokens"] = False
|
||||
|
||||
if overrides:
|
||||
kwargs.update(overrides)
|
||||
|
||||
return kwargs
|
||||
|
||||
def _tokenize_prompt(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -150,26 +124,18 @@ class InputPreprocessor:
|
||||
Apply the model's tokenizer to a text prompt, returning the
|
||||
corresponding token IDs.
|
||||
"""
|
||||
tokenizer = self.get_tokenizer()
|
||||
tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
|
||||
renderer = self.renderer
|
||||
|
||||
encoder_config = self.model_config.encoder_config
|
||||
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(tokenization_kwargs or {})
|
||||
)
|
||||
|
||||
if encoder_config and encoder_config.get("do_lower_case", False):
|
||||
prompt = prompt.lower()
|
||||
tok_prompt = renderer.tokenize_prompt(
|
||||
TextPrompt(prompt=prompt),
|
||||
tok_params,
|
||||
)
|
||||
|
||||
return tokenizer.encode(prompt, **tokenization_kwargs)
|
||||
|
||||
def _get_mm_processor(self) -> BaseMultiModalProcessor:
|
||||
if not hasattr(self, "_mm_processor"):
|
||||
self._mm_processor = self.mm_registry.create_processor(
|
||||
self.model_config,
|
||||
self.observability_config,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=self.mm_processor_cache,
|
||||
)
|
||||
|
||||
return self._mm_processor
|
||||
return tok_prompt["prompt_token_ids"]
|
||||
|
||||
def _process_multimodal(
|
||||
self,
|
||||
@@ -184,33 +150,20 @@ class InputPreprocessor:
|
||||
Apply the model's multi-modal processor to a multi-modal prompt,
|
||||
returning the corresponding token IDs and metadata.
|
||||
"""
|
||||
mm_processor = self._get_mm_processor()
|
||||
mm_processor = self.renderer.get_mm_processor()
|
||||
|
||||
if mm_processor_kwargs is None:
|
||||
mm_processor_kwargs = {}
|
||||
|
||||
mm_items = mm_processor.info.parse_mm_data(mm_data)
|
||||
mm_input = mm_processor.apply(
|
||||
|
||||
return mm_processor.apply(
|
||||
prompt,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
mm_hashes = mm_input["mm_hashes"]
|
||||
|
||||
# Validate that all mm items have a string as their hash
|
||||
contains_only_strings = all(
|
||||
isinstance(leaf, str) for leaf in json_iter_leaves(mm_hashes)
|
||||
)
|
||||
if not contains_only_strings:
|
||||
raise ValueError(
|
||||
f"mm_hashes must contain only strings, got: {mm_hashes}. "
|
||||
"This is likely due to an incorrect custom implementation of "
|
||||
"MultiModalProcessor.apply method."
|
||||
)
|
||||
|
||||
return mm_input
|
||||
|
||||
def _process_embeds(
|
||||
self,
|
||||
@@ -245,19 +198,18 @@ class InputPreprocessor:
|
||||
def _truncate_inputs(
|
||||
self, inputs: list[int], tokenization_kwargs: dict[str, Any] | None = None
|
||||
) -> list[int]:
|
||||
if (
|
||||
not tokenization_kwargs
|
||||
or "truncation" not in tokenization_kwargs
|
||||
or self.tokenizer is None
|
||||
):
|
||||
return inputs
|
||||
renderer = self.renderer
|
||||
|
||||
max_length = tokenization_kwargs["max_length"]
|
||||
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(tokenization_kwargs or {})
|
||||
)
|
||||
|
||||
if self.tokenizer.truncation_side == "left":
|
||||
return inputs[-max_length:]
|
||||
else:
|
||||
return inputs[:max_length]
|
||||
tok_prompt = renderer.tokenize_prompt(
|
||||
TokensPrompt(prompt_token_ids=inputs),
|
||||
tok_params,
|
||||
)
|
||||
|
||||
return tok_prompt["prompt_token_ids"]
|
||||
|
||||
def _process_tokens(
|
||||
self,
|
||||
@@ -539,26 +491,6 @@ class InputPreprocessor:
|
||||
"""Preprocess the input prompt."""
|
||||
res = self._preprocess(prompt, tokenization_kwargs, mm_uuids=mm_uuids)
|
||||
|
||||
if self.mm_processor_cache and self.mm_cache_stats is not None:
|
||||
delta = self.mm_processor_cache.make_stats(delta=True)
|
||||
self.mm_cache_stats.requests += 1
|
||||
self.mm_cache_stats.queries += delta.total
|
||||
self.mm_cache_stats.hits += delta.hits
|
||||
self.renderer.update_mm_cache_stats()
|
||||
|
||||
return res
|
||||
|
||||
def stat_mm_cache(self) -> MultiModalCacheStats | None:
|
||||
mm_cache_stats = self.mm_cache_stats
|
||||
if mm_cache_stats is None:
|
||||
return None
|
||||
|
||||
self.mm_cache_stats = MultiModalCacheStats()
|
||||
|
||||
return mm_cache_stats
|
||||
|
||||
def clear_mm_cache(self) -> None:
|
||||
if self.mm_processor_cache is not None:
|
||||
self.mm_processor_cache.clear_cache()
|
||||
|
||||
if self.mm_cache_stats is not None:
|
||||
self.mm_cache_stats.reset = True
|
||||
|
||||
Reference in New Issue
Block a user