[Bugfix][Frontend] Bound the prompt after multimodal expansion (#57076)
Signed-off-by: Dilber P Shakir <dilbersha@gmail.com>
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Truncation must bound the prompt *after* multimodal placeholders expand.
|
||||
|
||||
``TokenizeParams.apply_post_tokenization`` truncates the unexpanded prompt, so
|
||||
one placeholder token can expand into many and push the result back over
|
||||
``truncate_prompt_tokens`` (vllm-project/vllm#57038).
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.hf import _ensure_prompt_embeds_placeholder_token
|
||||
from vllm.renderers.params import TokenizeParams
|
||||
|
||||
|
||||
def _renderer():
|
||||
"""A BaseRenderer stub: only the tokenizer attribute is needed here."""
|
||||
|
||||
class _StubRenderer(BaseRenderer):
|
||||
def __init__(self):
|
||||
self.tokenizer = None
|
||||
|
||||
def render_messages(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
return _StubRenderer()
|
||||
|
||||
|
||||
def _mm_input(num_tokens: int, offset: int, length: int, **extra):
|
||||
return {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": list(range(num_tokens)),
|
||||
"mm_kwargs": {},
|
||||
"mm_hashes": {},
|
||||
"mm_placeholders": {"audio": [PlaceholderRange(offset=offset, length=length)]},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _params(limit: int | None, side: str = "right"):
|
||||
return TokenizeParams(
|
||||
max_total_tokens=4096,
|
||||
truncate_prompt_tokens=limit,
|
||||
truncation_side=side,
|
||||
)
|
||||
|
||||
|
||||
def test_expanded_prompt_is_bounded():
|
||||
"""The reported case: audio fits, trailing text is cut to the limit."""
|
||||
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(96))
|
||||
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
|
||||
offset=1, length=52
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_within_limit_is_untouched():
|
||||
engine_input = _mm_input(num_tokens=64, offset=1, length=52)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(64))
|
||||
|
||||
|
||||
def test_split_multimodal_span_is_rejected():
|
||||
"""Cutting mid-span would desync features from token ids, so reject."""
|
||||
engine_input = _mm_input(num_tokens=147, offset=80, length=50)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
|
||||
def test_dropped_multimodal_span_is_rejected():
|
||||
"""A span entirely past the cut would be discarded silently."""
|
||||
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
|
||||
def test_left_truncation_shifts_placeholder_offsets():
|
||||
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96, side="left"))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(51, 147))
|
||||
# The span started at 100 and 51 leading tokens were dropped.
|
||||
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
|
||||
offset=49, length=20
|
||||
)
|
||||
|
||||
|
||||
def test_text_only_input_is_untouched():
|
||||
"""Only multimodal prompts grow after tokenization, so nothing else runs."""
|
||||
engine_input = {
|
||||
"type": "token",
|
||||
"prompt_token_ids": list(range(147)),
|
||||
"prompt_token_offsets": [(i, i + 1) for i in range(147)],
|
||||
}
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(147))
|
||||
assert len(engine_input["prompt_token_offsets"]) == 147
|
||||
|
||||
|
||||
def test_no_truncation_requested_is_noop():
|
||||
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(None))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(147))
|
||||
|
||||
|
||||
def test_mixed_prompt_embeds_truncation_covers_embeds_spans():
|
||||
"""Bound in _process_singleton, where prompt_embeds spans are already attached."""
|
||||
model_config = ModelConfig(
|
||||
"Qwen/Qwen3-ASR-0.6B-hf",
|
||||
revision="7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c",
|
||||
dtype="float32",
|
||||
max_model_len=4096,
|
||||
mm_processor_cache_gb=0,
|
||||
enable_prompt_embeds=True,
|
||||
)
|
||||
renderer = renderer_from_config(VllmConfig(model_config=model_config))
|
||||
tok = renderer.tokenizer
|
||||
pe_id = _ensure_prompt_embeds_placeholder_token(tok)
|
||||
hidden = model_config.get_hidden_size()
|
||||
audio = np.zeros(4 * 16000, dtype=np.float32)
|
||||
|
||||
pe_len = 20
|
||||
tensors = [torch.zeros(pe_len, hidden, dtype=torch.float32)]
|
||||
|
||||
def _build_prompt():
|
||||
text = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
ids = tok.encode(text) + [pe_id] + tok.encode(" tail" * 40)
|
||||
return {
|
||||
"prompt_token_ids": ids,
|
||||
"multi_modal_data": {"audio": (audio, 16000)},
|
||||
"_prompt_embeds": (tensors, pe_id),
|
||||
}
|
||||
|
||||
# A limit past every span (audio ends at 52, prompt_embeds at 74):
|
||||
# truncation should succeed and bound the prompt.
|
||||
out = renderer._process_singleton(
|
||||
_build_prompt(),
|
||||
tok_params=_params(79),
|
||||
)
|
||||
assert len(out["prompt_token_ids"]) == 79
|
||||
|
||||
# A limit inside the prompt_embeds span (54..74): must be rejected.
|
||||
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
|
||||
renderer._process_singleton(
|
||||
_build_prompt(),
|
||||
tok_params=_params(64),
|
||||
)
|
||||
+94
-8
@@ -7,11 +7,13 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import replace
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, Generic, overload
|
||||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import (
|
||||
EmbedsInput,
|
||||
EmbedsPrompt,
|
||||
@@ -962,27 +964,103 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
return engine_input
|
||||
|
||||
def _truncate_expanded_prompt(
|
||||
self,
|
||||
engine_input: TokensInput | MultiModalInput,
|
||||
params: TokenizeParams | None,
|
||||
) -> None:
|
||||
"""Re-apply `truncate_prompt_tokens` once placeholders are expanded.
|
||||
|
||||
`TokenizeParams.apply_post_tokenization` truncates the prompt before
|
||||
multimodal expansion, so a single placeholder token can expand into
|
||||
many and push the result back over the requested bound.
|
||||
|
||||
Multimodal spans are never cut: a request whose spans would not all
|
||||
survive intact is rejected, matching the post-expansion rejection in
|
||||
`vllm.v1.engine.input_processor._validate_model_input`.
|
||||
|
||||
Only multimodal inputs can grow after `apply_post_tokenization`, so
|
||||
everything else returns immediately.
|
||||
"""
|
||||
if engine_input["type"] != "multimodal":
|
||||
return
|
||||
|
||||
if params is None or params.truncate_prompt_tokens is None:
|
||||
return
|
||||
|
||||
prompt_token_ids = engine_input.get("prompt_token_ids")
|
||||
if prompt_token_ids is None:
|
||||
return
|
||||
|
||||
num_tokens = len(prompt_token_ids)
|
||||
truncation = params._truncation_slice(self.tokenizer, num_tokens)
|
||||
if truncation is None:
|
||||
return
|
||||
|
||||
start, stop, _ = truncation.indices(num_tokens)
|
||||
|
||||
mm_placeholders = engine_input["mm_placeholders"]
|
||||
for modality, positions in mm_placeholders.items():
|
||||
for position in positions:
|
||||
end = position.offset + position.length
|
||||
if position.offset >= start and end <= stop:
|
||||
continue
|
||||
raise VLLMValidationError(
|
||||
f"truncate_prompt_tokens={params.truncate_prompt_tokens} "
|
||||
f"would split or drop a(n) {modality} item, which spans "
|
||||
f"tokens {position.offset} to {end} of the "
|
||||
f"{num_tokens}-token prompt. Multimodal placeholders "
|
||||
f"expand after truncation is applied, so the limit must "
|
||||
f"leave every multimodal item intact. Please raise "
|
||||
f"truncate_prompt_tokens or shorten the multimodal input.",
|
||||
parameter="truncate_prompt_tokens",
|
||||
value=params.truncate_prompt_tokens,
|
||||
)
|
||||
|
||||
if start:
|
||||
engine_input["mm_placeholders"] = {
|
||||
modality: [
|
||||
replace(position, offset=position.offset - start)
|
||||
for position in positions
|
||||
]
|
||||
for modality, positions in mm_placeholders.items()
|
||||
}
|
||||
|
||||
engine_input["prompt_token_ids"] = prompt_token_ids[truncation]
|
||||
|
||||
def _process_singleton(
|
||||
self,
|
||||
prompt: SingletonTokPrompt,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
) -> SingletonInput:
|
||||
if "prompt_embeds" in prompt:
|
||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
||||
|
||||
return self._process_tokens(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
|
||||
engine_input = self._process_tokens( # type: ignore[arg-type]
|
||||
prompt, # type: ignore[arg-type]
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
self._truncate_expanded_prompt(engine_input, tok_params)
|
||||
return engine_input
|
||||
|
||||
async def _process_singleton_async(
|
||||
self,
|
||||
prompt: SingletonTokPrompt,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
) -> SingletonInput:
|
||||
if "prompt_embeds" in prompt:
|
||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
||||
|
||||
return await self._process_tokens_async(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
|
||||
engine_input = await self._process_tokens_async( # type: ignore[arg-type]
|
||||
prompt, # type: ignore[arg-type]
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
self._truncate_expanded_prompt(engine_input, tok_params)
|
||||
return engine_input
|
||||
|
||||
def _get_skip_decoder_start_token(self) -> bool:
|
||||
"""Whether the multimodal processor supplies a complete decoder prefix."""
|
||||
@@ -1048,12 +1126,15 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
arrival_time: float,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
) -> EngineInput:
|
||||
engine_input: EngineInput
|
||||
if "encoder_prompt" in prompt:
|
||||
engine_input = self._process_enc_dec(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]
|
||||
else:
|
||||
engine_input = self._process_singleton(prompt, skip_mm_cache=skip_mm_cache)
|
||||
engine_input = self._process_singleton(
|
||||
prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params
|
||||
)
|
||||
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
|
||||
@@ -1065,6 +1146,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
arrival_time: float,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
) -> EngineInput:
|
||||
engine_input: EngineInput
|
||||
if "encoder_prompt" in prompt:
|
||||
@@ -1074,7 +1156,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
)
|
||||
else:
|
||||
engine_input = await self._process_singleton_async(
|
||||
prompt, skip_mm_cache=skip_mm_cache
|
||||
prompt, skip_mm_cache=skip_mm_cache, tok_params=tok_params
|
||||
)
|
||||
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
@@ -1101,7 +1183,9 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self._apply_prompt_extras(tok_prompts, prompt_extras)
|
||||
|
||||
return [
|
||||
self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)
|
||||
self.process_for_engine(
|
||||
prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
|
||||
)
|
||||
for prompt in tok_prompts
|
||||
]
|
||||
|
||||
@@ -1126,7 +1210,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return await asyncio.gather(
|
||||
*(
|
||||
self.process_for_engine_async(
|
||||
p, arrival_time, skip_mm_cache=skip_mm_cache
|
||||
p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
|
||||
)
|
||||
for p in tok_prompts
|
||||
)
|
||||
@@ -1164,7 +1248,9 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self._apply_prompt_extras(tok_prompts, prompt_extras)
|
||||
|
||||
eng_prompts = [
|
||||
self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)
|
||||
self.process_for_engine(
|
||||
prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
|
||||
)
|
||||
for prompt in tok_prompts
|
||||
]
|
||||
|
||||
@@ -1204,7 +1290,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
eng_prompts = await asyncio.gather(
|
||||
*(
|
||||
self.process_for_engine_async(
|
||||
p, arrival_time, skip_mm_cache=skip_mm_cache
|
||||
p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params
|
||||
)
|
||||
for p in tok_prompts
|
||||
)
|
||||
|
||||
+25
-27
@@ -1194,22 +1194,20 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
coordinate space, no offset shifting needed afterwards.
|
||||
"""
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
if prompt_embeds_info is None:
|
||||
return super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
|
||||
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
|
||||
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
|
||||
return engine_input
|
||||
|
||||
@@ -1222,24 +1220,24 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Async equivalent of `_process_tokens`."""
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
if prompt_embeds_info is None:
|
||||
return await super()._process_tokens_async(
|
||||
prompt, skip_mm_cache=skip_mm_cache
|
||||
)
|
||||
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
engine_input = await super()._process_tokens_async(
|
||||
prompt, skip_mm_cache=skip_mm_cache
|
||||
)
|
||||
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
|
||||
return engine_input
|
||||
|
||||
|
||||
Reference in New Issue
Block a user