[Bugfix][Frontend] Bound the prompt after multimodal expansion (#57076)

Signed-off-by: Dilber P Shakir <dilbersha@gmail.com>
This commit is contained in:
Dilber
2026-09-20 08:52:27 +00:00
committed by GitHub
parent e378275a8f
commit 27b7757f63
3 changed files with 285 additions and 35 deletions
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Truncation must bound the prompt *after* multimodal placeholders expand.
``TokenizeParams.apply_post_tokenization`` truncates the unexpanded prompt, so
one placeholder token can expand into many and push the result back over
``truncate_prompt_tokens`` (vllm-project/vllm#57038).
"""
import numpy as np
import pytest
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal.inputs import PlaceholderRange
from vllm.renderers import renderer_from_config
from vllm.renderers.base import BaseRenderer
from vllm.renderers.hf import _ensure_prompt_embeds_placeholder_token
from vllm.renderers.params import TokenizeParams
def _renderer():
"""A BaseRenderer stub: only the tokenizer attribute is needed here."""
class _StubRenderer(BaseRenderer):
def __init__(self):
self.tokenizer = None
def render_messages(self, *args, **kwargs):
raise NotImplementedError
return _StubRenderer()
def _mm_input(num_tokens: int, offset: int, length: int, **extra):
return {
"type": "multimodal",
"prompt_token_ids": list(range(num_tokens)),
"mm_kwargs": {},
"mm_hashes": {},
"mm_placeholders": {"audio": [PlaceholderRange(offset=offset, length=length)]},
**extra,
}
def _params(limit: int | None, side: str = "right"):
return TokenizeParams(
max_total_tokens=4096,
truncate_prompt_tokens=limit,
truncation_side=side,
)
def test_expanded_prompt_is_bounded():
"""The reported case: audio fits, trailing text is cut to the limit."""
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
assert engine_input["prompt_token_ids"] == list(range(96))
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
offset=1, length=52
)
def test_prompt_within_limit_is_untouched():
engine_input = _mm_input(num_tokens=64, offset=1, length=52)
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
assert engine_input["prompt_token_ids"] == list(range(64))
def test_split_multimodal_span_is_rejected():
"""Cutting mid-span would desync features from token ids, so reject."""
engine_input = _mm_input(num_tokens=147, offset=80, length=50)
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
def test_dropped_multimodal_span_is_rejected():
"""A span entirely past the cut would be discarded silently."""
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
def test_left_truncation_shifts_placeholder_offsets():
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
_renderer()._truncate_expanded_prompt(engine_input, _params(96, side="left"))
assert engine_input["prompt_token_ids"] == list(range(51, 147))
# The span started at 100 and 51 leading tokens were dropped.
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
offset=49, length=20
)
def test_text_only_input_is_untouched():
"""Only multimodal prompts grow after tokenization, so nothing else runs."""
engine_input = {
"type": "token",
"prompt_token_ids": list(range(147)),
"prompt_token_offsets": [(i, i + 1) for i in range(147)],
}
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
assert engine_input["prompt_token_ids"] == list(range(147))
assert len(engine_input["prompt_token_offsets"]) == 147
def test_no_truncation_requested_is_noop():
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
_renderer()._truncate_expanded_prompt(engine_input, _params(None))
assert engine_input["prompt_token_ids"] == list(range(147))
def test_mixed_prompt_embeds_truncation_covers_embeds_spans():
"""Bound in _process_singleton, where prompt_embeds spans are already attached."""
model_config = ModelConfig(
"Qwen/Qwen3-ASR-0.6B-hf",
revision="7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c",
dtype="float32",
max_model_len=4096,
mm_processor_cache_gb=0,
enable_prompt_embeds=True,
)
renderer = renderer_from_config(VllmConfig(model_config=model_config))
tok = renderer.tokenizer
pe_id = _ensure_prompt_embeds_placeholder_token(tok)
hidden = model_config.get_hidden_size()
audio = np.zeros(4 * 16000, dtype=np.float32)
pe_len = 20
tensors = [torch.zeros(pe_len, hidden, dtype=torch.float32)]
def _build_prompt():
text = "<|audio_start|><|audio_pad|><|audio_end|>"
ids = tok.encode(text) + [pe_id] + tok.encode(" tail" * 40)
return {
"prompt_token_ids": ids,
"multi_modal_data": {"audio": (audio, 16000)},
"_prompt_embeds": (tensors, pe_id),
}
# A limit past every span (audio ends at 52, prompt_embeds at 74):
# truncation should succeed and bound the prompt.
out = renderer._process_singleton(
_build_prompt(),
tok_params=_params(79),
)
assert len(out["prompt_token_ids"]) == 79
# A limit inside the prompt_embeds span (54..74): must be rejected.
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
renderer._process_singleton(
_build_prompt(),
tok_params=_params(64),
)
+94 -8
View File
@@ -7,11 +7,13 @@ from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import ExitStack
from dataclasses import replace
from functools import cached_property
from typing import TYPE_CHECKING, Any, Generic, overload
from typing_extensions import TypeVar
from vllm.exceptions import VLLMValidationError
from vllm.inputs import (
EmbedsInput,
EmbedsPrompt,
@@ -962,27 +964,103 @@ class BaseRenderer(ABC, Generic[_T]):
return engine_input
def _truncate_expanded_prompt(
self,
engine_input: TokensInput | MultiModalInput,
params: TokenizeParams | None,
) -> None:
"""Re-apply `truncate_prompt_tokens` once placeholders are expanded.
`TokenizeParams.apply_post_tokenization` truncates the prompt before
multimodal expansion, so a single placeholder token can expand into
many and push the result back over the requested bound.
Multimodal spans are never cut: a request whose spans would not all
survive intact is rejected, matching the post-expansion rejection in
`vllm.v1.engine.input_processor._validate_model_input`.
Only multimodal inputs can grow after `apply_post_tokenization`, so
everything else returns immediately.
"""
if engine_input["type"] != "multimodal":
return
if params is None or params.truncate_prompt_tokens is None:
return
prompt_token_ids = engine_input.get("prompt_token_ids")
if prompt_token_ids is None:
return
num_tokens = len(prompt_token_ids)
truncation = params._truncation_slice(self.tokenizer, num_tokens)
if truncation is None:
return
start, stop, _ = truncation.indices(num_tokens)
mm_placeholders = engine_input["mm_placeholders"]
for modality, positions in mm_placeholders.items():
for position in positions:
end = position.offset + position.length
if position.offset >= start and end <= stop:
continue
raise VLLMValidationError(
f"truncate_prompt_tokens={params.truncate_prompt_tokens} "
f"would split or drop a(n) {modality} item, which spans "
f"tokens {position.offset} to {end} of the "
f"{num_tokens}-token prompt. Multimodal placeholders "
f"expand after truncation is applied, so the limit must "
f"leave every multimodal item intact. Please raise "
f"truncate_prompt_tokens or shorten the multimodal input.",
parameter="truncate_prompt_tokens",
value=params.truncate_prompt_tokens,
)
if start:
engine_input["mm_placeholders"] = {
modality: [
replace(position, offset=position.offset - start)
for position in positions
]
for modality, positions in mm_placeholders.items()
}
engine_input["prompt_token_ids"] = prompt_token_ids[truncation]
def _process_singleton(
self,
prompt: SingletonTokPrompt,
*,
skip_mm_cache: bool = False,
tok_params: TokenizeParams | None = None,
) -> SingletonInput:
if "prompt_embeds" in prompt:
return self._process_embeds(prompt) # type: ignore[arg-type]
return self._process_tokens(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
engine_input = self._process_tokens( # type: ignore[arg-type]
prompt, # type: ignore[arg-type]
skip_mm_cache=skip_mm_cache,
)
self._truncate_expanded_prompt(engine_input, tok_params)
return engine_input
async def _process_singleton_async(
self,
prompt: SingletonTokPrompt,
*,
skip_mm_cache: bool = False,
tok_params: TokenizeParams | None = None,
) -> SingletonInput:
if "prompt_embeds" in prompt:
return self._process_embeds(prompt) # type: ignore[arg-type]
return await self._process_tokens_async(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
engine_input = await self._process_tokens_async( # type: ignore[arg-type]
prompt, # type: ignore[arg-type]
skip_mm_cache=skip_mm_cache,
)
self._truncate_expanded_prompt(engine_input, tok_params)
return engine_input
def _get_skip_decoder_start_token(self) -> bool:
"""Whether the multimodal processor supplies a complete decoder prefix."""
@@ -1048,12 +1126,15 @@ class BaseRenderer(ABC, Generic[_T]):
arrival_time: float,
*,
skip_mm_cache: bool = False,
tok_params: TokenizeParams | None = None,
) -> EngineInput:
engine_input: EngineInput
if "encoder_prompt" in prompt:
engine_input = self._process_enc_dec(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
else:
engine_input = self._process_singleton(prompt, skip_mm_cache=skip_mm_cache)
engine_input = self._process_singleton(
prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params
)
engine_input["arrival_time"] = arrival_time
@@ -1065,6 +1146,7 @@ class BaseRenderer(ABC, Generic[_T]):
arrival_time: float,
*,
skip_mm_cache: bool = False,
tok_params: TokenizeParams | None = None,
) -> EngineInput:
engine_input: EngineInput
if "encoder_prompt" in prompt:
@@ -1074,7 +1156,7 @@ class BaseRenderer(ABC, Generic[_T]):
)
else:
engine_input = await self._process_singleton_async(
prompt, skip_mm_cache=skip_mm_cache
prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params
)
engine_input["arrival_time"] = arrival_time
@@ -1101,7 +1183,9 @@ class BaseRenderer(ABC, Generic[_T]):
self._apply_prompt_extras(tok_prompts, prompt_extras)
return [
self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)
self.process_for_engine(
prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
)
for prompt in tok_prompts
]
@@ -1126,7 +1210,7 @@ class BaseRenderer(ABC, Generic[_T]):
return await asyncio.gather(
*(
self.process_for_engine_async(
p, arrival_time, skip_mm_cache=skip_mm_cache
p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
)
for p in tok_prompts
)
@@ -1164,7 +1248,9 @@ class BaseRenderer(ABC, Generic[_T]):
self._apply_prompt_extras(tok_prompts, prompt_extras)
eng_prompts = [
self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)
self.process_for_engine(
prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
)
for prompt in tok_prompts
]
@@ -1204,7 +1290,7 @@ class BaseRenderer(ABC, Generic[_T]):
eng_prompts = await asyncio.gather(
*(
self.process_for_engine_async(
p, arrival_time, skip_mm_cache=skip_mm_cache
p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
)
for p in tok_prompts
)
+25 -27
View File
@@ -1194,22 +1194,20 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
coordinate space, no offset shifting needed afterwards.
"""
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
if prompt_embeds_info is None:
return super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
tensors, placeholder_token_id = prompt_embeds_info
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
if prompt_embeds_info is not None:
tensors, _ = prompt_embeds_info
self._apply_prompt_embeds_to_engine_input(
cast(MultiModalInput, engine_input),
tensors,
mm_updates,
)
self._apply_prompt_embeds_to_engine_input(
cast(MultiModalInput, engine_input),
tensors,
mm_updates,
)
return engine_input
@@ -1222,24 +1220,24 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
) -> TokensInput | MultiModalInput:
"""Async equivalent of `_process_tokens`."""
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
if prompt_embeds_info is None:
return await super()._process_tokens_async(
prompt, skip_mm_cache=skip_mm_cache
)
tensors, placeholder_token_id = prompt_embeds_info
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
engine_input = await super()._process_tokens_async(
prompt, skip_mm_cache=skip_mm_cache
)
if prompt_embeds_info is not None:
tensors, _ = prompt_embeds_info
self._apply_prompt_embeds_to_engine_input(
cast(MultiModalInput, engine_input),
tensors,
mm_updates,
)
self._apply_prompt_embeds_to_engine_input(
cast(MultiModalInput, engine_input),
tensors,
mm_updates,
)
return engine_input