From 27b7757f6364be5b0b714793ccdc3e3931443bca Mon Sep 17 00:00:00 2001 From: Dilber <44312867+dilberx@users.noreply.github.com> Date: Sun, 20 Sep 2026 14:22:27 +0530 Subject: [PATCH] [Bugfix][Frontend] Bound the prompt after multimodal expansion (#57076) Signed-off-by: Dilber P Shakir --- .../test_truncate_after_mm_expansion.py | 166 ++++++++++++++++++ vllm/renderers/base.py | 102 ++++++++++- vllm/renderers/hf.py | 52 +++--- 3 files changed, 285 insertions(+), 35 deletions(-) create mode 100644 tests/renderers/test_truncate_after_mm_expansion.py diff --git a/tests/renderers/test_truncate_after_mm_expansion.py b/tests/renderers/test_truncate_after_mm_expansion.py new file mode 100644 index 0000000000..96cdfb12c2 --- /dev/null +++ b/tests/renderers/test_truncate_after_mm_expansion.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Truncation must bound the prompt *after* multimodal placeholders expand. + +``TokenizeParams.apply_post_tokenization`` truncates the unexpanded prompt, so +one placeholder token can expand into many and push the result back over +``truncate_prompt_tokens`` (vllm-project/vllm#57038). +""" + +import numpy as np +import pytest +import torch + +from vllm.config import ModelConfig, VllmConfig +from vllm.exceptions import VLLMValidationError +from vllm.multimodal.inputs import PlaceholderRange +from vllm.renderers import renderer_from_config +from vllm.renderers.base import BaseRenderer +from vllm.renderers.hf import _ensure_prompt_embeds_placeholder_token +from vllm.renderers.params import TokenizeParams + + +def _renderer(): + """A BaseRenderer stub: only the tokenizer attribute is needed here.""" + + class _StubRenderer(BaseRenderer): + def __init__(self): + self.tokenizer = None + + def render_messages(self, *args, **kwargs): + raise NotImplementedError + + return _StubRenderer() + + +def _mm_input(num_tokens: int, offset: int, length: int, **extra): + return { + "type": "multimodal", + "prompt_token_ids": list(range(num_tokens)), + "mm_kwargs": {}, + "mm_hashes": {}, + "mm_placeholders": {"audio": [PlaceholderRange(offset=offset, length=length)]}, + **extra, + } + + +def _params(limit: int | None, side: str = "right"): + return TokenizeParams( + max_total_tokens=4096, + truncate_prompt_tokens=limit, + truncation_side=side, + ) + + +def test_expanded_prompt_is_bounded(): + """The reported case: audio fits, trailing text is cut to the limit.""" + engine_input = _mm_input(num_tokens=147, offset=1, length=52) + + _renderer()._truncate_expanded_prompt(engine_input, _params(96)) + + assert engine_input["prompt_token_ids"] == list(range(96)) + assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange( + offset=1, length=52 + ) + + +def test_prompt_within_limit_is_untouched(): + engine_input = _mm_input(num_tokens=64, offset=1, length=52) + + _renderer()._truncate_expanded_prompt(engine_input, _params(96)) + + assert engine_input["prompt_token_ids"] == list(range(64)) + + +def test_split_multimodal_span_is_rejected(): + """Cutting mid-span would desync features from token ids, so reject.""" + engine_input = _mm_input(num_tokens=147, offset=80, length=50) + + with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"): + _renderer()._truncate_expanded_prompt(engine_input, _params(96)) + + +def test_dropped_multimodal_span_is_rejected(): + """A span entirely past the cut would be discarded silently.""" + engine_input = _mm_input(num_tokens=147, offset=100, length=20) + + with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"): + _renderer()._truncate_expanded_prompt(engine_input, _params(96)) + + +def test_left_truncation_shifts_placeholder_offsets(): + engine_input = _mm_input(num_tokens=147, offset=100, length=20) + + _renderer()._truncate_expanded_prompt(engine_input, _params(96, side="left")) + + assert engine_input["prompt_token_ids"] == list(range(51, 147)) + # The span started at 100 and 51 leading tokens were dropped. + assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange( + offset=49, length=20 + ) + + +def test_text_only_input_is_untouched(): + """Only multimodal prompts grow after tokenization, so nothing else runs.""" + engine_input = { + "type": "token", + "prompt_token_ids": list(range(147)), + "prompt_token_offsets": [(i, i + 1) for i in range(147)], + } + + _renderer()._truncate_expanded_prompt(engine_input, _params(96)) + + assert engine_input["prompt_token_ids"] == list(range(147)) + assert len(engine_input["prompt_token_offsets"]) == 147 + + +def test_no_truncation_requested_is_noop(): + engine_input = _mm_input(num_tokens=147, offset=1, length=52) + + _renderer()._truncate_expanded_prompt(engine_input, _params(None)) + + assert engine_input["prompt_token_ids"] == list(range(147)) + + +def test_mixed_prompt_embeds_truncation_covers_embeds_spans(): + """Bound in _process_singleton, where prompt_embeds spans are already attached.""" + model_config = ModelConfig( + "Qwen/Qwen3-ASR-0.6B-hf", + revision="7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c", + dtype="float32", + max_model_len=4096, + mm_processor_cache_gb=0, + enable_prompt_embeds=True, + ) + renderer = renderer_from_config(VllmConfig(model_config=model_config)) + tok = renderer.tokenizer + pe_id = _ensure_prompt_embeds_placeholder_token(tok) + hidden = model_config.get_hidden_size() + audio = np.zeros(4 * 16000, dtype=np.float32) + + pe_len = 20 + tensors = [torch.zeros(pe_len, hidden, dtype=torch.float32)] + + def _build_prompt(): + text = "<|audio_start|><|audio_pad|><|audio_end|>" + ids = tok.encode(text) + [pe_id] + tok.encode(" tail" * 40) + return { + "prompt_token_ids": ids, + "multi_modal_data": {"audio": (audio, 16000)}, + "_prompt_embeds": (tensors, pe_id), + } + + # A limit past every span (audio ends at 52, prompt_embeds at 74): + # truncation should succeed and bound the prompt. + out = renderer._process_singleton( + _build_prompt(), + tok_params=_params(79), + ) + assert len(out["prompt_token_ids"]) == 79 + + # A limit inside the prompt_embeds span (54..74): must be rejected. + with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"): + renderer._process_singleton( + _build_prompt(), + tok_params=_params(64), + ) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index f4916faf6b..9b769686a7 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -7,11 +7,13 @@ from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack +from dataclasses import replace from functools import cached_property from typing import TYPE_CHECKING, Any, Generic, overload from typing_extensions import TypeVar +from vllm.exceptions import VLLMValidationError from vllm.inputs import ( EmbedsInput, EmbedsPrompt, @@ -962,27 +964,103 @@ class BaseRenderer(ABC, Generic[_T]): return engine_input + def _truncate_expanded_prompt( + self, + engine_input: TokensInput | MultiModalInput, + params: TokenizeParams | None, + ) -> None: + """Re-apply `truncate_prompt_tokens` once placeholders are expanded. + + `TokenizeParams.apply_post_tokenization` truncates the prompt before + multimodal expansion, so a single placeholder token can expand into + many and push the result back over the requested bound. + + Multimodal spans are never cut: a request whose spans would not all + survive intact is rejected, matching the post-expansion rejection in + `vllm.v1.engine.input_processor._validate_model_input`. + + Only multimodal inputs can grow after `apply_post_tokenization`, so + everything else returns immediately. + """ + if engine_input["type"] != "multimodal": + return + + if params is None or params.truncate_prompt_tokens is None: + return + + prompt_token_ids = engine_input.get("prompt_token_ids") + if prompt_token_ids is None: + return + + num_tokens = len(prompt_token_ids) + truncation = params._truncation_slice(self.tokenizer, num_tokens) + if truncation is None: + return + + start, stop, _ = truncation.indices(num_tokens) + + mm_placeholders = engine_input["mm_placeholders"] + for modality, positions in mm_placeholders.items(): + for position in positions: + end = position.offset + position.length + if position.offset >= start and end <= stop: + continue + raise VLLMValidationError( + f"truncate_prompt_tokens={params.truncate_prompt_tokens} " + f"would split or drop a(n) {modality} item, which spans " + f"tokens {position.offset} to {end} of the " + f"{num_tokens}-token prompt. Multimodal placeholders " + f"expand after truncation is applied, so the limit must " + f"leave every multimodal item intact. Please raise " + f"truncate_prompt_tokens or shorten the multimodal input.", + parameter="truncate_prompt_tokens", + value=params.truncate_prompt_tokens, + ) + + if start: + engine_input["mm_placeholders"] = { + modality: [ + replace(position, offset=position.offset - start) + for position in positions + ] + for modality, positions in mm_placeholders.items() + } + + engine_input["prompt_token_ids"] = prompt_token_ids[truncation] + def _process_singleton( self, prompt: SingletonTokPrompt, *, skip_mm_cache: bool = False, + tok_params: TokenizeParams | None = None, ) -> SingletonInput: if "prompt_embeds" in prompt: return self._process_embeds(prompt) # type: ignore[arg-type] - return self._process_tokens(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type] + engine_input = self._process_tokens( # type: ignore[arg-type] + prompt, # type: ignore[arg-type] + skip_mm_cache=skip_mm_cache, + ) + self._truncate_expanded_prompt(engine_input, tok_params) + return engine_input async def _process_singleton_async( self, prompt: SingletonTokPrompt, *, skip_mm_cache: bool = False, + tok_params: TokenizeParams | None = None, ) -> SingletonInput: if "prompt_embeds" in prompt: return self._process_embeds(prompt) # type: ignore[arg-type] - return await self._process_tokens_async(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type] + engine_input = await self._process_tokens_async( # type: ignore[arg-type] + prompt, # type: ignore[arg-type] + skip_mm_cache=skip_mm_cache, + ) + self._truncate_expanded_prompt(engine_input, tok_params) + return engine_input def _get_skip_decoder_start_token(self) -> bool: """Whether the multimodal processor supplies a complete decoder prefix.""" @@ -1048,12 +1126,15 @@ class BaseRenderer(ABC, Generic[_T]): arrival_time: float, *, skip_mm_cache: bool = False, + tok_params: TokenizeParams | None = None, ) -> EngineInput: engine_input: EngineInput if "encoder_prompt" in prompt: engine_input = self._process_enc_dec(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type] else: - engine_input = self._process_singleton(prompt, skip_mm_cache=skip_mm_cache) + engine_input = self._process_singleton( + prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params + ) engine_input["arrival_time"] = arrival_time @@ -1065,6 +1146,7 @@ class BaseRenderer(ABC, Generic[_T]): arrival_time: float, *, skip_mm_cache: bool = False, + tok_params: TokenizeParams | None = None, ) -> EngineInput: engine_input: EngineInput if "encoder_prompt" in prompt: @@ -1074,7 +1156,7 @@ class BaseRenderer(ABC, Generic[_T]): ) else: engine_input = await self._process_singleton_async( - prompt, skip_mm_cache=skip_mm_cache + prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params ) engine_input["arrival_time"] = arrival_time @@ -1101,7 +1183,9 @@ class BaseRenderer(ABC, Generic[_T]): self._apply_prompt_extras(tok_prompts, prompt_extras) return [ - self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache) + self.process_for_engine( + prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params + ) for prompt in tok_prompts ] @@ -1126,7 +1210,7 @@ class BaseRenderer(ABC, Generic[_T]): return await asyncio.gather( *( self.process_for_engine_async( - p, arrival_time, skip_mm_cache=skip_mm_cache + p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params ) for p in tok_prompts ) @@ -1164,7 +1248,9 @@ class BaseRenderer(ABC, Generic[_T]): self._apply_prompt_extras(tok_prompts, prompt_extras) eng_prompts = [ - self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache) + self.process_for_engine( + prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params + ) for prompt in tok_prompts ] @@ -1204,7 +1290,7 @@ class BaseRenderer(ABC, Generic[_T]): eng_prompts = await asyncio.gather( *( self.process_for_engine_async( - p, arrival_time, skip_mm_cache=skip_mm_cache + p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params ) for p in tok_prompts ) diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 1c35802d0d..36605c62b9 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -1194,22 +1194,20 @@ class HfRenderer(BaseRenderer[HfTokenizer]): coordinate space, no offset shifting needed afterwards. """ prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None) - if prompt_embeds_info is not None: - tensors, placeholder_token_id = prompt_embeds_info - mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id) - cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders( - list(prompt["prompt_token_ids"]), mm_updates - ) + if prompt_embeds_info is None: + return super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache) + tensors, placeholder_token_id = prompt_embeds_info + mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id) + cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders( + list(prompt["prompt_token_ids"]), mm_updates + ) engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache) - - if prompt_embeds_info is not None: - tensors, _ = prompt_embeds_info - self._apply_prompt_embeds_to_engine_input( - cast(MultiModalInput, engine_input), - tensors, - mm_updates, - ) + self._apply_prompt_embeds_to_engine_input( + cast(MultiModalInput, engine_input), + tensors, + mm_updates, + ) return engine_input @@ -1222,24 +1220,24 @@ class HfRenderer(BaseRenderer[HfTokenizer]): ) -> TokensInput | MultiModalInput: """Async equivalent of `_process_tokens`.""" prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None) - if prompt_embeds_info is not None: - tensors, placeholder_token_id = prompt_embeds_info - mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id) - cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders( - list(prompt["prompt_token_ids"]), mm_updates + if prompt_embeds_info is None: + return await super()._process_tokens_async( + prompt, skip_mm_cache=skip_mm_cache ) + tensors, placeholder_token_id = prompt_embeds_info + mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id) + cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders( + list(prompt["prompt_token_ids"]), mm_updates + ) engine_input = await super()._process_tokens_async( prompt, skip_mm_cache=skip_mm_cache ) - - if prompt_embeds_info is not None: - tensors, _ = prompt_embeds_info - self._apply_prompt_embeds_to_engine_input( - cast(MultiModalInput, engine_input), - tensors, - mm_updates, - ) + self._apply_prompt_embeds_to_engine_input( + cast(MultiModalInput, engine_input), + tensors, + mm_updates, + ) return engine_input