[Refactor] Remove InputPreprocessor (#53064)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -172,7 +172,6 @@ steps:
|
|||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/
|
- vllm/
|
||||||
- tests/test_envs.py
|
- tests/test_envs.py
|
||||||
- tests/test_inputs.py
|
|
||||||
- tests/test_outputs.py
|
- tests/test_outputs.py
|
||||||
- tests/test_pooling_params.py
|
- tests/test_pooling_params.py
|
||||||
- tests/test_ray_env.py
|
- tests/test_ray_env.py
|
||||||
@@ -189,7 +188,6 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- python3 standalone_tests/lazy_imports.py
|
- python3 standalone_tests/lazy_imports.py
|
||||||
- pytest -v -s test_envs.py
|
- pytest -v -s test_envs.py
|
||||||
- pytest -v -s test_inputs.py
|
|
||||||
- pytest -v -s test_outputs.py
|
- pytest -v -s test_outputs.py
|
||||||
- pytest -v -s test_pooling_params.py
|
- pytest -v -s test_pooling_params.py
|
||||||
- pytest -v -s test_ray_env.py
|
- pytest -v -s test_ray_env.py
|
||||||
|
|||||||
@@ -358,7 +358,6 @@ steps:
|
|||||||
- vllm/utils/
|
- vllm/utils/
|
||||||
- vllm/v1/
|
- vllm/v1/
|
||||||
- tests/test_envs.py
|
- tests/test_envs.py
|
||||||
- tests/test_inputs.py
|
|
||||||
- tests/test_outputs.py
|
- tests/test_outputs.py
|
||||||
- tests/test_pooling_params.py
|
- tests/test_pooling_params.py
|
||||||
- tests/test_ray_env.py
|
- tests/test_ray_env.py
|
||||||
@@ -376,7 +375,6 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- python3 standalone_tests/lazy_imports.py
|
- python3 standalone_tests/lazy_imports.py
|
||||||
- pytest -v -s test_envs.py
|
- pytest -v -s test_envs.py
|
||||||
- pytest -v -s test_inputs.py
|
|
||||||
- pytest -v -s test_outputs.py
|
- pytest -v -s test_outputs.py
|
||||||
- pytest -v -s test_pooling_params.py
|
- pytest -v -s test_pooling_params.py
|
||||||
- pytest -v -s test_ray_env.py
|
- pytest -v -s test_ray_env.py
|
||||||
|
|||||||
@@ -107,7 +107,6 @@
|
|||||||
/tests/multimodal @DarkLight1337 @ywang96 @NickLucche @shen-shanshan
|
/tests/multimodal @DarkLight1337 @ywang96 @NickLucche @shen-shanshan
|
||||||
/tests/multimodal/media @Isotr0py
|
/tests/multimodal/media @Isotr0py
|
||||||
/tests/quantization @mgoin @robertgshaw2-redhat @yewentao256 @pavanimajety @zyongye @AndreasKaratzas
|
/tests/quantization @mgoin @robertgshaw2-redhat @yewentao256 @pavanimajety @zyongye @AndreasKaratzas
|
||||||
/tests/test_inputs.py @DarkLight1337 @ywang96
|
|
||||||
/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
|
/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
|
||||||
/tests/v1/structured_output @mgoin @russellb @aarnphm
|
/tests/v1/structured_output @mgoin @russellb @aarnphm
|
||||||
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
|
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
|
||||||
|
|||||||
@@ -54,11 +54,10 @@ def test_text_only_model_mm_data_maps_to_bad_request():
|
|||||||
|
|
||||||
with pytest.raises(ValueError, match="text-only") as exc_info:
|
with pytest.raises(ValueError, match="text-only") as exc_info:
|
||||||
renderer._process_multimodal(
|
renderer._process_multimodal(
|
||||||
prompt="What is in this image?",
|
prompt=[1],
|
||||||
mm_data={"image": [cherry_pil_image]},
|
mm_data={"image": [cherry_pil_image]},
|
||||||
mm_uuids=None,
|
mm_uuids=None,
|
||||||
mm_processor_kwargs=None,
|
mm_processor_kwargs=None,
|
||||||
tokenization_kwargs=None,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
error_response = create_error_response(exc_info.value)
|
error_response = create_error_response(exc_info.value)
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from vllm.config import ModelConfig, VllmConfig
|
|
||||||
from vllm.inputs.preprocess import InputPreprocessor
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.cpu_test
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model_id", ["facebook/chameleon-7b"])
|
|
||||||
@pytest.mark.parametrize("prompt", ["", {"prompt_token_ids": []}])
|
|
||||||
@pytest.mark.skip(
|
|
||||||
reason=(
|
|
||||||
"Applying huggingface processor on text inputs results in "
|
|
||||||
"significant performance regression for multimodal models. "
|
|
||||||
"See https://github.com/vllm-project/vllm/issues/26320"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
def test_preprocessor_always_mm_code_path(model_id, prompt):
|
|
||||||
model_config = ModelConfig(model=model_id)
|
|
||||||
vllm_config = VllmConfig(model_config=model_config)
|
|
||||||
input_preprocessor = InputPreprocessor(vllm_config)
|
|
||||||
|
|
||||||
# HF processor adds sep token
|
|
||||||
tokenizer = input_preprocessor.get_tokenizer()
|
|
||||||
sep_token_id = tokenizer.vocab[tokenizer.sep_token]
|
|
||||||
|
|
||||||
processed_inputs = input_preprocessor.preprocess(prompt)
|
|
||||||
assert sep_token_id in processed_inputs["prompt_token_ids"]
|
|
||||||
@@ -1,291 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from collections.abc import Mapping
|
|
||||||
from typing import Any, overload
|
|
||||||
|
|
||||||
from typing_extensions import assert_never
|
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
|
||||||
from vllm.inputs import build_enc_dec_input
|
|
||||||
from vllm.logger import init_logger
|
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
|
||||||
from vllm.renderers import BaseRenderer, renderer_from_config
|
|
||||||
from vllm.renderers.inputs import (
|
|
||||||
DecoderDictPrompt,
|
|
||||||
DecoderOnlyDictPrompt,
|
|
||||||
EncoderDecoderDictPrompt,
|
|
||||||
EncoderDictPrompt,
|
|
||||||
SingletonDictPrompt,
|
|
||||||
)
|
|
||||||
from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt
|
|
||||||
from vllm.tokenizers import TokenizerLike
|
|
||||||
|
|
||||||
from .engine import (
|
|
||||||
DecoderEngineInput,
|
|
||||||
DecoderOnlyEngineInput,
|
|
||||||
EmbedsInput,
|
|
||||||
EncoderDecoderInput,
|
|
||||||
EncoderInput,
|
|
||||||
EngineInput,
|
|
||||||
MultiModalInput,
|
|
||||||
SingletonInput,
|
|
||||||
TokensInput,
|
|
||||||
tokens_input,
|
|
||||||
)
|
|
||||||
from .llm import (
|
|
||||||
EmbedsPrompt,
|
|
||||||
MultiModalDataDict,
|
|
||||||
MultiModalUUIDDict,
|
|
||||||
PromptType,
|
|
||||||
TextPrompt,
|
|
||||||
TokensPrompt,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class InputPreprocessor:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
vllm_config: VllmConfig,
|
|
||||||
renderer: BaseRenderer | None = None,
|
|
||||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
|
||||||
) -> None:
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
self.model_config = vllm_config.model_config
|
|
||||||
self.renderer = renderer or renderer_from_config(vllm_config)
|
|
||||||
self.mm_registry = mm_registry
|
|
||||||
|
|
||||||
@property
|
|
||||||
def tokenizer(self) -> TokenizerLike | None:
|
|
||||||
return self.renderer.tokenizer
|
|
||||||
|
|
||||||
def get_tokenizer(self) -> TokenizerLike:
|
|
||||||
return self.renderer.get_tokenizer()
|
|
||||||
|
|
||||||
def _tokenize_prompt(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> list[int]:
|
|
||||||
"""
|
|
||||||
Apply the model's tokenizer to a text prompt, returning the
|
|
||||||
corresponding token IDs.
|
|
||||||
"""
|
|
||||||
renderer = self.renderer
|
|
||||||
|
|
||||||
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
|
|
||||||
**(tokenization_kwargs or {})
|
|
||||||
)
|
|
||||||
|
|
||||||
tok_prompt = renderer._tokenize_singleton_prompt(
|
|
||||||
TextPrompt(prompt=prompt),
|
|
||||||
tok_params,
|
|
||||||
)
|
|
||||||
|
|
||||||
return tok_prompt["prompt_token_ids"]
|
|
||||||
|
|
||||||
def _process_multimodal(
|
|
||||||
self,
|
|
||||||
prompt: str | list[int],
|
|
||||||
mm_data: MultiModalDataDict,
|
|
||||||
mm_processor_kwargs: Mapping[str, object] | None = None,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
*,
|
|
||||||
mm_uuids: MultiModalUUIDDict | None = None,
|
|
||||||
) -> MultiModalInput:
|
|
||||||
"""
|
|
||||||
Apply the model's multi-modal processor to a multi-modal prompt,
|
|
||||||
returning the corresponding token IDs and metadata.
|
|
||||||
"""
|
|
||||||
return self.renderer._process_multimodal(
|
|
||||||
prompt,
|
|
||||||
mm_data,
|
|
||||||
mm_uuids=mm_uuids,
|
|
||||||
mm_processor_kwargs=mm_processor_kwargs,
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _process_embeds(
|
|
||||||
self,
|
|
||||||
parsed_content: EmbedsPrompt,
|
|
||||||
) -> EmbedsInput:
|
|
||||||
return self.renderer._process_embeds(parsed_content)
|
|
||||||
|
|
||||||
def _truncate_inputs(
|
|
||||||
self, inputs: list[int], tokenization_kwargs: dict[str, Any] | None = None
|
|
||||||
) -> list[int]:
|
|
||||||
renderer = self.renderer
|
|
||||||
|
|
||||||
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
|
|
||||||
**(tokenization_kwargs or {})
|
|
||||||
)
|
|
||||||
|
|
||||||
tok_prompt = renderer._tokenize_singleton_prompt(
|
|
||||||
TokensPrompt(prompt_token_ids=inputs),
|
|
||||||
tok_params,
|
|
||||||
)
|
|
||||||
|
|
||||||
return tok_prompt["prompt_token_ids"]
|
|
||||||
|
|
||||||
def _process_tokens(
|
|
||||||
self,
|
|
||||||
parsed_content: TokensPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> TokensInput | MultiModalInput:
|
|
||||||
prompt_token_ids = self._truncate_inputs(
|
|
||||||
parsed_content["prompt_token_ids"], tokenization_kwargs
|
|
||||||
)
|
|
||||||
|
|
||||||
inputs: TokensInput | MultiModalInput
|
|
||||||
if multi_modal_data := parsed_content.get("multi_modal_data"):
|
|
||||||
inputs = self._process_multimodal(
|
|
||||||
prompt_token_ids,
|
|
||||||
multi_modal_data,
|
|
||||||
parsed_content.get("mm_processor_kwargs"),
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
mm_uuids=parsed_content.get("multi_modal_uuids"),
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
inputs = tokens_input(prompt_token_ids)
|
|
||||||
|
|
||||||
if prompt_text := parsed_content.get("prompt"):
|
|
||||||
inputs["prompt"] = prompt_text
|
|
||||||
if cache_salt := parsed_content.get("cache_salt"):
|
|
||||||
inputs["cache_salt"] = cache_salt
|
|
||||||
|
|
||||||
return inputs
|
|
||||||
|
|
||||||
def _process_text(
|
|
||||||
self,
|
|
||||||
parsed_content: TextPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> TokensInput | MultiModalInput:
|
|
||||||
prompt_text = parsed_content["prompt"]
|
|
||||||
|
|
||||||
inputs: TokensInput | MultiModalInput
|
|
||||||
if multi_modal_data := parsed_content.get("multi_modal_data"):
|
|
||||||
inputs = self._process_multimodal(
|
|
||||||
prompt_text,
|
|
||||||
multi_modal_data,
|
|
||||||
parsed_content.get("mm_processor_kwargs") or {},
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
prompt_token_ids = self._tokenize_prompt(
|
|
||||||
prompt_text,
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
inputs = tokens_input(prompt_token_ids)
|
|
||||||
|
|
||||||
inputs["prompt"] = prompt_text
|
|
||||||
|
|
||||||
if cache_salt := parsed_content.get("cache_salt"):
|
|
||||||
inputs["cache_salt"] = cache_salt
|
|
||||||
|
|
||||||
return inputs
|
|
||||||
|
|
||||||
@overload
|
|
||||||
def _prompt_to_llm_inputs(
|
|
||||||
self,
|
|
||||||
prompt: EncoderDictPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> EncoderInput: ...
|
|
||||||
|
|
||||||
@overload
|
|
||||||
def _prompt_to_llm_inputs( # type: ignore[misc]
|
|
||||||
self,
|
|
||||||
prompt: DecoderDictPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> DecoderEngineInput: ...
|
|
||||||
|
|
||||||
@overload
|
|
||||||
def _prompt_to_llm_inputs( # type: ignore[misc]
|
|
||||||
self,
|
|
||||||
prompt: DecoderOnlyDictPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> DecoderOnlyEngineInput: ...
|
|
||||||
|
|
||||||
def _prompt_to_llm_inputs(
|
|
||||||
self,
|
|
||||||
prompt: SingletonDictPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> SingletonInput:
|
|
||||||
if "prompt_embeds" in prompt:
|
|
||||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
|
||||||
|
|
||||||
if "prompt_token_ids" in prompt:
|
|
||||||
return self._process_tokens(prompt) # type: ignore[arg-type]
|
|
||||||
|
|
||||||
if "prompt" in prompt:
|
|
||||||
return self._process_text(
|
|
||||||
prompt, # type: ignore[arg-type]
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert_never(prompt) # type: ignore[arg-type]
|
|
||||||
|
|
||||||
def _process_encoder_decoder_prompt(
|
|
||||||
self,
|
|
||||||
prompt: EncoderDecoderDictPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> EncoderDecoderInput:
|
|
||||||
encoder_prompt = prompt["encoder_prompt"]
|
|
||||||
decoder_prompt = prompt["decoder_prompt"]
|
|
||||||
|
|
||||||
skip_decoder_start_token = False
|
|
||||||
if self.renderer.mm_processor is not None:
|
|
||||||
from vllm.multimodal.processing import EncDecMultiModalProcessor
|
|
||||||
|
|
||||||
if isinstance(self.renderer.mm_processor, EncDecMultiModalProcessor):
|
|
||||||
skip_decoder_start_token = (
|
|
||||||
self.renderer.mm_processor.skip_decoder_start_token
|
|
||||||
)
|
|
||||||
|
|
||||||
return build_enc_dec_input(
|
|
||||||
encoder_input=self._prompt_to_llm_inputs(
|
|
||||||
encoder_prompt,
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
),
|
|
||||||
decoder_input=(
|
|
||||||
None
|
|
||||||
if decoder_prompt is None
|
|
||||||
else self._prompt_to_llm_inputs(
|
|
||||||
decoder_prompt,
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
decoder_start_token_id=self.renderer.get_dec_start_token_id(),
|
|
||||||
skip_decoder_start_token=skip_decoder_start_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _process_decoder_only_prompt(
|
|
||||||
self,
|
|
||||||
prompt: DecoderOnlyDictPrompt,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> DecoderOnlyEngineInput:
|
|
||||||
return self._prompt_to_llm_inputs(
|
|
||||||
prompt,
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def preprocess(
|
|
||||||
self,
|
|
||||||
prompt: PromptType,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> EngineInput:
|
|
||||||
"""Preprocess the input prompt."""
|
|
||||||
if self.model_config.is_encoder_decoder:
|
|
||||||
# Encoder-decoder model requires special mapping of
|
|
||||||
# input prompts to encoder & decoder.
|
|
||||||
return self._process_encoder_decoder_prompt(
|
|
||||||
parse_enc_dec_prompt(prompt),
|
|
||||||
tokenization_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
return self._process_decoder_only_prompt(
|
|
||||||
parse_dec_only_prompt(prompt),
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
@@ -725,14 +725,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
|
|
||||||
return mm_uuid_items
|
return mm_uuid_items
|
||||||
|
|
||||||
# TODO: Remove str and tokenization_kwargs after deprecating InputPreprocessor
|
|
||||||
def _process_multimodal(
|
def _process_multimodal(
|
||||||
self,
|
self,
|
||||||
prompt: list[int] | str,
|
prompt: list[int],
|
||||||
mm_data: MultiModalDataDict,
|
mm_data: MultiModalDataDict,
|
||||||
mm_uuids: MultiModalUUIDDict | None,
|
mm_uuids: MultiModalUUIDDict | None,
|
||||||
mm_processor_kwargs: Mapping[str, object] | None,
|
mm_processor_kwargs: Mapping[str, object] | None,
|
||||||
tokenization_kwargs: dict[str, Any] | None,
|
|
||||||
*,
|
*,
|
||||||
skip_mm_cache: bool = False,
|
skip_mm_cache: bool = False,
|
||||||
) -> "MultiModalInput":
|
) -> "MultiModalInput":
|
||||||
@@ -755,7 +753,6 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
mm_data_items,
|
mm_data_items,
|
||||||
mm_uuid_items,
|
mm_uuid_items,
|
||||||
hf_processor_mm_kwargs=mm_processor_kwargs or {},
|
hf_processor_mm_kwargs=mm_processor_kwargs or {},
|
||||||
tokenization_kwargs=tokenization_kwargs or {},
|
|
||||||
)
|
)
|
||||||
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
|
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
|
||||||
|
|
||||||
@@ -783,7 +780,6 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
prompt_token_ids,
|
prompt_token_ids,
|
||||||
multi_modal_data,
|
multi_modal_data,
|
||||||
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
|
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
|
||||||
tokenization_kwargs=None, # Tokenization already done in Step 2
|
|
||||||
mm_uuids=prompt.get("multi_modal_uuids"),
|
mm_uuids=prompt.get("multi_modal_uuids"),
|
||||||
skip_mm_cache=skip_mm_cache,
|
skip_mm_cache=skip_mm_cache,
|
||||||
)
|
)
|
||||||
@@ -846,7 +842,6 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
prompt_token_ids,
|
prompt_token_ids,
|
||||||
multi_modal_data,
|
multi_modal_data,
|
||||||
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
|
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
|
||||||
tokenization_kwargs=None,
|
|
||||||
mm_uuids=prompt.get("multi_modal_uuids"),
|
mm_uuids=prompt.get("multi_modal_uuids"),
|
||||||
skip_mm_cache=skip_mm_cache,
|
skip_mm_cache=skip_mm_cache,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ from vllm.inputs import (
|
|||||||
SingletonInput,
|
SingletonInput,
|
||||||
split_enc_dec_input,
|
split_enc_dec_input,
|
||||||
)
|
)
|
||||||
from vllm.inputs.preprocess import InputPreprocessor
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||||
@@ -24,6 +23,7 @@ from vllm.multimodal.utils import argsort_mm_positions
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.pooling_params import PoolingParams
|
from vllm.pooling_params import PoolingParams
|
||||||
from vllm.renderers import BaseRenderer, renderer_from_config
|
from vllm.renderers import BaseRenderer, renderer_from_config
|
||||||
|
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
|
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
@@ -67,12 +67,6 @@ class InputProcessor:
|
|||||||
)
|
)
|
||||||
mm_budget.reset_cache() # Not used anymore
|
mm_budget.reset_cache() # Not used anymore
|
||||||
|
|
||||||
self.input_preprocessor = InputPreprocessor(
|
|
||||||
vllm_config,
|
|
||||||
renderer=renderer,
|
|
||||||
mm_registry=mm_registry,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Raw-prompt preprocessing (tokenization and multimodal processing)
|
# Raw-prompt preprocessing (tokenization and multimodal processing)
|
||||||
# is blocking, so async callers should run it on the renderer's
|
# is blocking, so async callers should run it on the renderer's
|
||||||
# thread pool to keep their event loop responsive.
|
# thread pool to keep their event loop responsive.
|
||||||
@@ -313,44 +307,45 @@ class InputProcessor:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if isinstance(prompt, dict) and "type" in prompt:
|
if isinstance(prompt, dict) and "type" in prompt:
|
||||||
if tokenization_kwargs:
|
|
||||||
logger.warning_once(
|
|
||||||
"Passing tokenization_kwargs to InputProcessor is deprecated "
|
|
||||||
"and will be removed in v0.18. You should instead pass "
|
|
||||||
"them to Renderer.render_cmpl() or Renderer.render_chat()."
|
|
||||||
)
|
|
||||||
|
|
||||||
if arrival_time is None:
|
if arrival_time is None:
|
||||||
arrival_time = prompt.get("arrival_time", time.time()) # type: ignore[assignment]
|
arrival_time = prompt.get("arrival_time", time.time()) # type: ignore[assignment]
|
||||||
|
|
||||||
processed_inputs: EngineInput = prompt # type: ignore[assignment]
|
engine_input: EngineInput = prompt # type: ignore[assignment]
|
||||||
else:
|
else:
|
||||||
logger.warning_once(
|
logger.warning_once(
|
||||||
"Passing raw prompts to InputProcessor is deprecated "
|
"Passing raw prompts to InputProcessor is deprecated "
|
||||||
"and will be removed in v0.18. You should instead pass "
|
"and will be removed in the future. You should instead pass "
|
||||||
"the outputs of Renderer.render_cmpl() or Renderer.render_chat()."
|
"the outputs of Renderer.render_cmpl() or Renderer.render_chat()."
|
||||||
)
|
)
|
||||||
|
|
||||||
if arrival_time is None:
|
if arrival_time is None:
|
||||||
arrival_time = time.time()
|
arrival_time = time.time()
|
||||||
|
|
||||||
processed_inputs = self.input_preprocessor.preprocess(
|
renderer = self.renderer
|
||||||
prompt,
|
model_config = self.model_config
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
|
parsed_prompt = parse_model_prompt(model_config, prompt)
|
||||||
|
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
|
||||||
|
**(tokenization_kwargs or {})
|
||||||
)
|
)
|
||||||
|
|
||||||
current_platform.validate_request(processed_inputs, params)
|
(engine_input,) = renderer.render_cmpl(
|
||||||
|
[parsed_prompt],
|
||||||
|
tok_params,
|
||||||
|
)
|
||||||
|
|
||||||
encoder_inputs, decoder_inputs = split_enc_dec_input(processed_inputs)
|
current_platform.validate_request(engine_input, params)
|
||||||
self._validate_model_inputs(encoder_inputs, decoder_inputs)
|
|
||||||
|
encoder_input, decoder_input = split_enc_dec_input(engine_input)
|
||||||
|
self._validate_model_inputs(encoder_input, decoder_input)
|
||||||
|
|
||||||
# Mypy can be conservative for TypedDict unions; normalize access.
|
# Mypy can be conservative for TypedDict unions; normalize access.
|
||||||
if decoder_inputs["type"] == "embeds":
|
if decoder_input["type"] == "embeds":
|
||||||
prompt_embeds = decoder_inputs["prompt_embeds"]
|
prompt_embeds = decoder_input["prompt_embeds"]
|
||||||
prompt_token_ids = decoder_inputs.get("prompt_token_ids")
|
prompt_token_ids = decoder_input.get("prompt_token_ids")
|
||||||
prompt_is_token_ids = decoder_inputs.get("is_token_ids")
|
prompt_is_token_ids = decoder_input.get("is_token_ids")
|
||||||
else:
|
else:
|
||||||
prompt_token_ids = decoder_inputs["prompt_token_ids"]
|
prompt_token_ids = decoder_input["prompt_token_ids"]
|
||||||
prompt_embeds = None
|
prompt_embeds = None
|
||||||
prompt_is_token_ids = None
|
prompt_is_token_ids = None
|
||||||
|
|
||||||
@@ -385,10 +380,10 @@ class InputProcessor:
|
|||||||
# Multimodal related.
|
# Multimodal related.
|
||||||
mm_features: list[MultiModalFeatureSpec] | None = None
|
mm_features: list[MultiModalFeatureSpec] | None = None
|
||||||
|
|
||||||
if decoder_inputs["type"] == "multimodal":
|
if decoder_input["type"] == "multimodal":
|
||||||
decoder_mm_inputs = decoder_inputs["mm_kwargs"]
|
decoder_mm_inputs = decoder_input["mm_kwargs"]
|
||||||
decoder_mm_positions = decoder_inputs["mm_placeholders"]
|
decoder_mm_positions = decoder_input["mm_placeholders"]
|
||||||
decoder_mm_hashes = decoder_inputs["mm_hashes"]
|
decoder_mm_hashes = decoder_input["mm_hashes"]
|
||||||
|
|
||||||
if not all(
|
if not all(
|
||||||
isinstance(leaf, str) for leaf in json_iter_leaves(decoder_mm_hashes)
|
isinstance(leaf, str) for leaf in json_iter_leaves(decoder_mm_hashes)
|
||||||
@@ -430,7 +425,7 @@ class InputProcessor:
|
|||||||
pooling_params=pooling_params,
|
pooling_params=pooling_params,
|
||||||
arrival_time=arrival_time,
|
arrival_time=arrival_time,
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
cache_salt=decoder_inputs.get("cache_salt"),
|
cache_salt=decoder_input.get("cache_salt"),
|
||||||
priority=priority,
|
priority=priority,
|
||||||
data_parallel_rank=data_parallel_rank,
|
data_parallel_rank=data_parallel_rank,
|
||||||
trace_headers=trace_headers,
|
trace_headers=trace_headers,
|
||||||
|
|||||||
Reference in New Issue
Block a user