[Renderer] Move InputPreprocessor into Renderer (1/2) (#34510)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
@@ -208,14 +208,23 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
|
||||
*,
|
||||
mm_uuids: MultiModalUUIDDict | None = None,
|
||||
) -> MultiModalInputs:
|
||||
if prompt and mm_items:
|
||||
raise ValueError(
|
||||
"CLIP accepts text-only or image-only inputs, not both! "
|
||||
"Image-only inputs means passing an image with an empty text "
|
||||
"prompt."
|
||||
)
|
||||
|
||||
if mm_items:
|
||||
if isinstance(prompt, str):
|
||||
if len(prompt) > 0:
|
||||
raise ValueError(
|
||||
"CLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty text prompt."
|
||||
)
|
||||
else:
|
||||
special_tokens = self.info.get_tokenizer().all_special_ids
|
||||
if all(tok in special_tokens for tok in prompt):
|
||||
prompt = []
|
||||
else:
|
||||
raise ValueError(
|
||||
"CLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty token prompt."
|
||||
)
|
||||
|
||||
# For multi-modal data, the prompt after processing should
|
||||
# only contain the dummy image tokens
|
||||
tokenization_kwargs = {
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -90,6 +91,9 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
|
||||
def get_image_processor(self, **kwargs: object) -> Lfm2VlImageProcessorFast:
|
||||
return self.get_hf_processor(**kwargs).image_processor
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -554,6 +555,9 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
|
||||
Llama4Processor, use_fast=kwargs.pop("use_fast", True), **kwargs
|
||||
)
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# Although vLLM can support more images from an infra capability
|
||||
# perspective, we do not recommend using >10 images in practice.
|
||||
|
||||
@@ -76,6 +76,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdateDetails,
|
||||
_seq2tokens,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.radio import RadioConfig
|
||||
@@ -1093,6 +1094,9 @@ class BaseNanoNemotronVLProcessingInfo(BaseProcessingInfo):
|
||||
) -> BaseNanoNemotronVLProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.transformers_utils.configs.radio import RadioConfig
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -608,6 +609,9 @@ class NemotronParseProcessingInfo(BaseProcessingInfo):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
@property
|
||||
def skip_prompt_length_check(self) -> bool:
|
||||
return True # Because the encoder prompt is padded
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.ovis import OvisProcessor
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -264,6 +265,9 @@ class OvisProcessingInfo(BaseProcessingInfo):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_image_segment_len(self) -> int:
|
||||
visual_tokenizer_config = self.get_hf_config().visual_tokenizer_config
|
||||
image_size = visual_tokenizer_config.backbone_config.image_size
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -183,6 +184,9 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
temporal_patch_size=vit_config.temporal_patch_size,
|
||||
)
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_image_processor(self) -> BaseImageProcessor:
|
||||
return self.get_hf_processor().image_processor # type: ignore
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -102,6 +103,9 @@ class PaliGemmaProcessingInfo(BaseProcessingInfo):
|
||||
def get_vision_encoder_info(self):
|
||||
return get_vision_encoder_info(self.get_hf_config())
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": 1}
|
||||
|
||||
|
||||
@@ -194,14 +194,23 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
|
||||
*,
|
||||
mm_uuids: MultiModalUUIDDict | None = None,
|
||||
) -> MultiModalInputs:
|
||||
if prompt and mm_items:
|
||||
raise ValueError(
|
||||
"Siglip accepts text-only or image-only inputs, not both! "
|
||||
"Image-only inputs means passing an image with an empty text "
|
||||
"prompt."
|
||||
)
|
||||
|
||||
if mm_items:
|
||||
if isinstance(prompt, str):
|
||||
if len(prompt) > 0:
|
||||
raise ValueError(
|
||||
"SigLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty text prompt."
|
||||
)
|
||||
else:
|
||||
special_tokens = self.info.get_tokenizer().all_special_ids
|
||||
if all(tok in special_tokens for tok in prompt):
|
||||
prompt = []
|
||||
else:
|
||||
raise ValueError(
|
||||
"SigLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty token prompt."
|
||||
)
|
||||
|
||||
# For multi-modal data, the prompt after processing should
|
||||
# only contain the image token
|
||||
tokenization_kwargs = {
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -133,6 +134,9 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
|
||||
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
||||
return feature_extractor
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_data_parser(self):
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
|
||||
|
||||
@@ -17,8 +17,9 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S
|
||||
from vllm.inputs.data import PromptType, StreamingInput, TokensPrompt
|
||||
from vllm.inputs.data import PromptType, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime
|
||||
from vllm.model_executor.models.voxtral import (
|
||||
|
||||
@@ -55,6 +55,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.jsontree import json_map_leaves
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -644,6 +645,12 @@ class WhisperProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self) -> WhisperConfig:
|
||||
return self.ctx.get_hf_config(WhisperConfig)
|
||||
|
||||
def get_default_tok_params(self) -> TokenizeParams:
|
||||
# Special tokens should be provided by the user based on the
|
||||
# task and language of their request. Also needed to avoid
|
||||
# appending an EOS token to the prompt which disrupts generation.
|
||||
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
|
||||
|
||||
def get_data_parser(self):
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user