diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index e21c8f59e6..e789829fe9 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -172,7 +172,6 @@ steps: source_file_dependencies: - vllm/ - tests/test_envs.py - - tests/test_inputs.py - tests/test_outputs.py - tests/test_pooling_params.py - tests/test_ray_env.py @@ -189,7 +188,6 @@ steps: commands: - python3 standalone_tests/lazy_imports.py - pytest -v -s test_envs.py - - pytest -v -s test_inputs.py - pytest -v -s test_outputs.py - pytest -v -s test_pooling_params.py - pytest -v -s test_ray_env.py diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index a73ca9bb81..b564896fbd 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -358,7 +358,6 @@ steps: - vllm/utils/ - vllm/v1/ - tests/test_envs.py - - tests/test_inputs.py - tests/test_outputs.py - tests/test_pooling_params.py - tests/test_ray_env.py @@ -376,7 +375,6 @@ steps: commands: - python3 standalone_tests/lazy_imports.py - pytest -v -s test_envs.py - - pytest -v -s test_inputs.py - pytest -v -s test_outputs.py - pytest -v -s test_pooling_params.py - pytest -v -s test_ray_env.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a9a21b5ef1..95ea9046d5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -107,7 +107,6 @@ /tests/multimodal @DarkLight1337 @ywang96 @NickLucche @shen-shanshan /tests/multimodal/media @Isotr0py /tests/quantization @mgoin @robertgshaw2-redhat @yewentao256 @pavanimajety @zyongye @AndreasKaratzas -/tests/test_inputs.py @DarkLight1337 @ywang96 /tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm /tests/v1/structured_output @mgoin @russellb @aarnphm /tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index c811630c85..76b394897e 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -54,11 +54,10 @@ def test_text_only_model_mm_data_maps_to_bad_request(): with pytest.raises(ValueError, match="text-only") as exc_info: renderer._process_multimodal( - prompt="What is in this image?", + prompt=[1], mm_data={"image": [cherry_pil_image]}, mm_uuids=None, mm_processor_kwargs=None, - tokenization_kwargs=None, ) error_response = create_error_response(exc_info.value) diff --git a/tests/test_inputs.py b/tests/test_inputs.py deleted file mode 100644 index fb1bbd21ea..0000000000 --- a/tests/test_inputs.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from vllm.config import ModelConfig, VllmConfig -from vllm.inputs.preprocess import InputPreprocessor - -pytestmark = pytest.mark.cpu_test - - -@pytest.mark.parametrize("model_id", ["facebook/chameleon-7b"]) -@pytest.mark.parametrize("prompt", ["", {"prompt_token_ids": []}]) -@pytest.mark.skip( - reason=( - "Applying huggingface processor on text inputs results in " - "significant performance regression for multimodal models. " - "See https://github.com/vllm-project/vllm/issues/26320" - ) -) -def test_preprocessor_always_mm_code_path(model_id, prompt): - model_config = ModelConfig(model=model_id) - vllm_config = VllmConfig(model_config=model_config) - input_preprocessor = InputPreprocessor(vllm_config) - - # HF processor adds sep token - tokenizer = input_preprocessor.get_tokenizer() - sep_token_id = tokenizer.vocab[tokenizer.sep_token] - - processed_inputs = input_preprocessor.preprocess(prompt) - assert sep_token_id in processed_inputs["prompt_token_ids"] diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py deleted file mode 100644 index 7722014f9c..0000000000 --- a/vllm/inputs/preprocess.py +++ /dev/null @@ -1,291 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Mapping -from typing import Any, overload - -from typing_extensions import assert_never - -from vllm.config import VllmConfig -from vllm.inputs import build_enc_dec_input -from vllm.logger import init_logger -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry -from vllm.renderers import BaseRenderer, renderer_from_config -from vllm.renderers.inputs import ( - DecoderDictPrompt, - DecoderOnlyDictPrompt, - EncoderDecoderDictPrompt, - EncoderDictPrompt, - SingletonDictPrompt, -) -from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt -from vllm.tokenizers import TokenizerLike - -from .engine import ( - DecoderEngineInput, - DecoderOnlyEngineInput, - EmbedsInput, - EncoderDecoderInput, - EncoderInput, - EngineInput, - MultiModalInput, - SingletonInput, - TokensInput, - tokens_input, -) -from .llm import ( - EmbedsPrompt, - MultiModalDataDict, - MultiModalUUIDDict, - PromptType, - TextPrompt, - TokensPrompt, -) - -logger = init_logger(__name__) - - -class InputPreprocessor: - def __init__( - self, - vllm_config: VllmConfig, - renderer: BaseRenderer | None = None, - mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, - ) -> None: - super().__init__() - - self.model_config = vllm_config.model_config - self.renderer = renderer or renderer_from_config(vllm_config) - self.mm_registry = mm_registry - - @property - def tokenizer(self) -> TokenizerLike | None: - return self.renderer.tokenizer - - def get_tokenizer(self) -> TokenizerLike: - return self.renderer.get_tokenizer() - - def _tokenize_prompt( - self, - prompt: str, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> list[int]: - """ - Apply the model's tokenizer to a text prompt, returning the - corresponding token IDs. - """ - renderer = self.renderer - - tok_params = renderer.default_cmpl_tok_params.with_kwargs( - **(tokenization_kwargs or {}) - ) - - tok_prompt = renderer._tokenize_singleton_prompt( - TextPrompt(prompt=prompt), - tok_params, - ) - - return tok_prompt["prompt_token_ids"] - - def _process_multimodal( - self, - prompt: str | list[int], - mm_data: MultiModalDataDict, - mm_processor_kwargs: Mapping[str, object] | None = None, - tokenization_kwargs: dict[str, Any] | None = None, - *, - mm_uuids: MultiModalUUIDDict | None = None, - ) -> MultiModalInput: - """ - Apply the model's multi-modal processor to a multi-modal prompt, - returning the corresponding token IDs and metadata. - """ - return self.renderer._process_multimodal( - prompt, - mm_data, - mm_uuids=mm_uuids, - mm_processor_kwargs=mm_processor_kwargs, - tokenization_kwargs=tokenization_kwargs, - ) - - def _process_embeds( - self, - parsed_content: EmbedsPrompt, - ) -> EmbedsInput: - return self.renderer._process_embeds(parsed_content) - - def _truncate_inputs( - self, inputs: list[int], tokenization_kwargs: dict[str, Any] | None = None - ) -> list[int]: - renderer = self.renderer - - tok_params = renderer.default_cmpl_tok_params.with_kwargs( - **(tokenization_kwargs or {}) - ) - - tok_prompt = renderer._tokenize_singleton_prompt( - TokensPrompt(prompt_token_ids=inputs), - tok_params, - ) - - return tok_prompt["prompt_token_ids"] - - def _process_tokens( - self, - parsed_content: TokensPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> TokensInput | MultiModalInput: - prompt_token_ids = self._truncate_inputs( - parsed_content["prompt_token_ids"], tokenization_kwargs - ) - - inputs: TokensInput | MultiModalInput - if multi_modal_data := parsed_content.get("multi_modal_data"): - inputs = self._process_multimodal( - prompt_token_ids, - multi_modal_data, - parsed_content.get("mm_processor_kwargs"), - tokenization_kwargs=tokenization_kwargs, - mm_uuids=parsed_content.get("multi_modal_uuids"), - ) - else: - inputs = tokens_input(prompt_token_ids) - - if prompt_text := parsed_content.get("prompt"): - inputs["prompt"] = prompt_text - if cache_salt := parsed_content.get("cache_salt"): - inputs["cache_salt"] = cache_salt - - return inputs - - def _process_text( - self, - parsed_content: TextPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> TokensInput | MultiModalInput: - prompt_text = parsed_content["prompt"] - - inputs: TokensInput | MultiModalInput - if multi_modal_data := parsed_content.get("multi_modal_data"): - inputs = self._process_multimodal( - prompt_text, - multi_modal_data, - parsed_content.get("mm_processor_kwargs") or {}, - tokenization_kwargs=tokenization_kwargs, - ) - else: - prompt_token_ids = self._tokenize_prompt( - prompt_text, - tokenization_kwargs=tokenization_kwargs, - ) - inputs = tokens_input(prompt_token_ids) - - inputs["prompt"] = prompt_text - - if cache_salt := parsed_content.get("cache_salt"): - inputs["cache_salt"] = cache_salt - - return inputs - - @overload - def _prompt_to_llm_inputs( - self, - prompt: EncoderDictPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> EncoderInput: ... - - @overload - def _prompt_to_llm_inputs( # type: ignore[misc] - self, - prompt: DecoderDictPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> DecoderEngineInput: ... - - @overload - def _prompt_to_llm_inputs( # type: ignore[misc] - self, - prompt: DecoderOnlyDictPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> DecoderOnlyEngineInput: ... - - def _prompt_to_llm_inputs( - self, - prompt: SingletonDictPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> SingletonInput: - if "prompt_embeds" in prompt: - return self._process_embeds(prompt) # type: ignore[arg-type] - - if "prompt_token_ids" in prompt: - return self._process_tokens(prompt) # type: ignore[arg-type] - - if "prompt" in prompt: - return self._process_text( - prompt, # type: ignore[arg-type] - tokenization_kwargs=tokenization_kwargs, - ) - - assert_never(prompt) # type: ignore[arg-type] - - def _process_encoder_decoder_prompt( - self, - prompt: EncoderDecoderDictPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> EncoderDecoderInput: - encoder_prompt = prompt["encoder_prompt"] - decoder_prompt = prompt["decoder_prompt"] - - skip_decoder_start_token = False - if self.renderer.mm_processor is not None: - from vllm.multimodal.processing import EncDecMultiModalProcessor - - if isinstance(self.renderer.mm_processor, EncDecMultiModalProcessor): - skip_decoder_start_token = ( - self.renderer.mm_processor.skip_decoder_start_token - ) - - return build_enc_dec_input( - encoder_input=self._prompt_to_llm_inputs( - encoder_prompt, - tokenization_kwargs=tokenization_kwargs, - ), - decoder_input=( - None - if decoder_prompt is None - else self._prompt_to_llm_inputs( - decoder_prompt, - tokenization_kwargs=tokenization_kwargs, - ) - ), - decoder_start_token_id=self.renderer.get_dec_start_token_id(), - skip_decoder_start_token=skip_decoder_start_token, - ) - - def _process_decoder_only_prompt( - self, - prompt: DecoderOnlyDictPrompt, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> DecoderOnlyEngineInput: - return self._prompt_to_llm_inputs( - prompt, - tokenization_kwargs=tokenization_kwargs, - ) - - def preprocess( - self, - prompt: PromptType, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> EngineInput: - """Preprocess the input prompt.""" - if self.model_config.is_encoder_decoder: - # Encoder-decoder model requires special mapping of - # input prompts to encoder & decoder. - return self._process_encoder_decoder_prompt( - parse_enc_dec_prompt(prompt), - tokenization_kwargs, - ) - - return self._process_decoder_only_prompt( - parse_dec_only_prompt(prompt), - tokenization_kwargs=tokenization_kwargs, - ) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 832f954eaf..aaf89e6edd 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -725,14 +725,12 @@ class BaseRenderer(ABC, Generic[_T]): return mm_uuid_items - # TODO: Remove str and tokenization_kwargs after deprecating InputPreprocessor def _process_multimodal( self, - prompt: list[int] | str, + prompt: list[int], mm_data: MultiModalDataDict, mm_uuids: MultiModalUUIDDict | None, mm_processor_kwargs: Mapping[str, object] | None, - tokenization_kwargs: dict[str, Any] | None, *, skip_mm_cache: bool = False, ) -> "MultiModalInput": @@ -755,7 +753,6 @@ class BaseRenderer(ABC, Generic[_T]): mm_data_items, mm_uuid_items, hf_processor_mm_kwargs=mm_processor_kwargs or {}, - tokenization_kwargs=tokenization_kwargs or {}, ) mm_timing_ctx = self._mm_timing_registry.get(mm_req_id) @@ -783,7 +780,6 @@ class BaseRenderer(ABC, Generic[_T]): prompt_token_ids, multi_modal_data, mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - tokenization_kwargs=None, # Tokenization already done in Step 2 mm_uuids=prompt.get("multi_modal_uuids"), skip_mm_cache=skip_mm_cache, ) @@ -846,7 +842,6 @@ class BaseRenderer(ABC, Generic[_T]): prompt_token_ids, multi_modal_data, mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - tokenization_kwargs=None, mm_uuids=prompt.get("multi_modal_uuids"), skip_mm_cache=skip_mm_cache, ) diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 744da670a0..2b2ff4bab8 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -14,7 +14,6 @@ from vllm.inputs import ( SingletonInput, split_enc_dec_input, ) -from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry @@ -24,6 +23,7 @@ from vllm.multimodal.utils import argsort_mm_positions from vllm.platforms import current_platform from vllm.pooling_params import PoolingParams from vllm.renderers import BaseRenderer, renderer_from_config +from vllm.renderers.inputs.preprocess import parse_model_prompt from vllm.sampling_params import SamplingParams from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask from vllm.tokenizers import TokenizerLike @@ -67,12 +67,6 @@ class InputProcessor: ) mm_budget.reset_cache() # Not used anymore - self.input_preprocessor = InputPreprocessor( - vllm_config, - renderer=renderer, - mm_registry=mm_registry, - ) - # Raw-prompt preprocessing (tokenization and multimodal processing) # is blocking, so async callers should run it on the renderer's # thread pool to keep their event loop responsive. @@ -313,44 +307,45 @@ class InputProcessor: ) if isinstance(prompt, dict) and "type" in prompt: - if tokenization_kwargs: - logger.warning_once( - "Passing tokenization_kwargs to InputProcessor is deprecated " - "and will be removed in v0.18. You should instead pass " - "them to Renderer.render_cmpl() or Renderer.render_chat()." - ) - if arrival_time is None: arrival_time = prompt.get("arrival_time", time.time()) # type: ignore[assignment] - processed_inputs: EngineInput = prompt # type: ignore[assignment] + engine_input: EngineInput = prompt # type: ignore[assignment] else: logger.warning_once( "Passing raw prompts to InputProcessor is deprecated " - "and will be removed in v0.18. You should instead pass " + "and will be removed in the future. You should instead pass " "the outputs of Renderer.render_cmpl() or Renderer.render_chat()." ) if arrival_time is None: arrival_time = time.time() - processed_inputs = self.input_preprocessor.preprocess( - prompt, - tokenization_kwargs=tokenization_kwargs, + renderer = self.renderer + model_config = self.model_config + + parsed_prompt = parse_model_prompt(model_config, prompt) + tok_params = renderer.default_cmpl_tok_params.with_kwargs( + **(tokenization_kwargs or {}) ) - current_platform.validate_request(processed_inputs, params) + (engine_input,) = renderer.render_cmpl( + [parsed_prompt], + tok_params, + ) - encoder_inputs, decoder_inputs = split_enc_dec_input(processed_inputs) - self._validate_model_inputs(encoder_inputs, decoder_inputs) + current_platform.validate_request(engine_input, params) + + encoder_input, decoder_input = split_enc_dec_input(engine_input) + self._validate_model_inputs(encoder_input, decoder_input) # Mypy can be conservative for TypedDict unions; normalize access. - if decoder_inputs["type"] == "embeds": - prompt_embeds = decoder_inputs["prompt_embeds"] - prompt_token_ids = decoder_inputs.get("prompt_token_ids") - prompt_is_token_ids = decoder_inputs.get("is_token_ids") + if decoder_input["type"] == "embeds": + prompt_embeds = decoder_input["prompt_embeds"] + prompt_token_ids = decoder_input.get("prompt_token_ids") + prompt_is_token_ids = decoder_input.get("is_token_ids") else: - prompt_token_ids = decoder_inputs["prompt_token_ids"] + prompt_token_ids = decoder_input["prompt_token_ids"] prompt_embeds = None prompt_is_token_ids = None @@ -385,10 +380,10 @@ class InputProcessor: # Multimodal related. mm_features: list[MultiModalFeatureSpec] | None = None - if decoder_inputs["type"] == "multimodal": - decoder_mm_inputs = decoder_inputs["mm_kwargs"] - decoder_mm_positions = decoder_inputs["mm_placeholders"] - decoder_mm_hashes = decoder_inputs["mm_hashes"] + if decoder_input["type"] == "multimodal": + decoder_mm_inputs = decoder_input["mm_kwargs"] + decoder_mm_positions = decoder_input["mm_placeholders"] + decoder_mm_hashes = decoder_input["mm_hashes"] if not all( isinstance(leaf, str) for leaf in json_iter_leaves(decoder_mm_hashes) @@ -430,7 +425,7 @@ class InputProcessor: pooling_params=pooling_params, arrival_time=arrival_time, lora_request=lora_request, - cache_salt=decoder_inputs.get("cache_salt"), + cache_salt=decoder_input.get("cache_salt"), priority=priority, data_parallel_rank=data_parallel_rank, trace_headers=trace_headers,