[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,626 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline unit tests for `prompt_embeds` chat-completion content parts."""
from __future__ import annotations
import inspect
import io
from typing import Final
from unittest import mock
import pybase64 as base64
import pytest
import regex as re
import torch
from transformers import AutoTokenizer
from vllm.exceptions import VLLMValidationError
from vllm.renderers import TokenizeParams
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
from vllm.renderers.chat_utils.chat_message import (
_PROMPT_EMBEDS_MISSING_DATA_ERROR,
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR,
MM_PARSER_MAP,
)
from vllm.renderers.chat_utils.content_parser import (
_ENABLE_PROMPT_EMBEDS_ERROR,
MODALITY_PLACEHOLDERS_MAP,
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
)
from vllm.renderers.hf import (
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,
_build_mixed_prompt_embeds,
_build_prompt_embeds_positions,
_build_prompt_embeds_updates,
_ensure_prompt_embeds_placeholder_token,
_expand_prompt_embeds_placeholders,
)
# Cover distinct tokenizer families:
# GPT2TokenizerFast (BPE, OpenAI-style)
# Qwen2TokenizerFast (SentencePiece BPE variant)
# BertTokenizerFast (WordPiece)
TOKENIZER_IDS: Final[list[str]] = [
"openai-community/gpt2",
"Qwen/Qwen2.5-1.5B-Instruct",
"bert-base-uncased",
]
@pytest.fixture(params=TOKENIZER_IDS, ids=TOKENIZER_IDS)
def tokenizer(request):
"""A fresh tokenizer instance per tokenizer family."""
return AutoTokenizer.from_pretrained(request.param)
# Minimal chat template that works with any tokenizer. Iterates
# `message.content` as either a string or a list of dicts (openai format).
_SIMPLE_CHAT_TEMPLATE: Final[str] = (
"{% for m in messages %}"
"{% if m['content'] is string %}{{m['content']}}"
"{% else %}{% for p in m['content'] %}{{p['text']}}{% endfor %}"
"{% endif %}\n{% endfor %}"
)
async def _maybe_await(fn, *args, **kwargs):
"""Call *fn* and `await` the result if it's a coroutine."""
result = fn(*args, **kwargs)
if inspect.iscoroutine(result):
result = await result
return result
# Parametrize over sync / async parse paths so every end-to-end test
# exercises both.
_PARSE_FUNCTIONS = [parse_chat_messages, parse_chat_messages_async]
@pytest.fixture(params=_PARSE_FUNCTIONS, ids=["sync", "async"])
def parse_fn(request):
"""Either the sync or async `parse_chat_messages` callable."""
return request.param
def _encode_tensor(t: torch.Tensor) -> str:
buf = io.BytesIO()
torch.save(t, buf)
return base64.b64encode(buf.getvalue()).decode("utf-8")
_MOCK_HIDDEN_SIZE: Final[int] = 8
_MOCK_DTYPE: Final[torch.dtype] = torch.float32
def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicMock:
mc = mock.MagicMock()
mc.enable_prompt_embeds = enable_prompt_embeds
mc.multimodal_config = None
mc.allowed_local_media_path = None
mc.allowed_media_domains = None
# Test text-only code path in `MultiModalItemTracker.resolve_items`.
mc.is_multimodal_model = False
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
# and dtype, so the mock must return concrete values.
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
mc.hf_text_config = object()
mc.dtype = _MOCK_DTYPE
return mc
def test_prompt_embeds_keys_registered():
assert "prompt_embeds" in MODALITY_PLACEHOLDERS_MAP
assert MODALITY_PLACEHOLDERS_MAP["prompt_embeds"] == "<##PROMPT_EMBEDS##>"
assert "prompt_embeds" in MM_PARSER_MAP
def test_ensure_placeholder_token_is_single_token_and_idempotent(tokenizer):
"""Ensure the placeholder token is a single token and that multiple calls to
"ensure" are idempotent, across all tokenizer families."""
tid1 = _ensure_prompt_embeds_placeholder_token(tokenizer)
tid2 = _ensure_prompt_embeds_placeholder_token(tokenizer)
assert tid1 == tid2
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
assert ids == [tid1]
# Repeating it in a string N times must produce exactly that many tokens.
N = 5
ids_rep = tokenizer.encode(
PROMPT_EMBEDS_PLACEHOLDER_TOKEN * N, add_special_tokens=False
)
assert ids_rep == [tid1] * N
def test_parse_chat_messages_openai_format():
NUM_TOKENS = 3
t = torch.randn(NUM_TOKENS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
b64 = _encode_tensor(t)
mc = _make_mock_model_config()
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello "},
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": " world"},
],
}
]
conv, mm_data, _ = parse_chat_messages(
messages,
mc,
content_format="openai",
)
# The middle content part is rewritten to a single placeholder-token
# sentinel.
texts = [p["text"] for p in conv[0]["content"]]
assert texts == [
"Hello ",
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
" world",
]
assert mm_data is not None and "prompt_embeds" in mm_data
assert torch.equal(mm_data["prompt_embeds"][0], t)
# Each layout entry is one content part:
# ("text", "A") -> {"type": "text", "text": "A"}
# ("embed", N) -> {"type": "prompt_embeds", "data": <base64 of (N, H) tensor>}
@pytest.mark.parametrize(
"layout",
[
# Case: Single embed only.
[("embed", 2)],
# Case: Embed at the start of the message.
[("embed", 3), ("text", "B")],
# Case: Embed at the end of the message.
[("text", "A"), ("embed", 1)],
# Case: Embed sandwiched between text spans.
[("text", "A"), ("embed", 2), ("text", "B")],
# Case: Multiple embeds with text in between.
[("text", "A"), ("embed", 2), ("text", "B"), ("embed", 3)],
# Case: Adjacent embeds with no separating text.
[("embed", 1), ("embed", 2)],
# Case: Multiple text spans before a trailing embed.
[("text", "A"), ("text", "B"), ("embed", 1)],
# Case: Long-ish run mixing both kinds.
[
("text", "head"),
("embed", 4),
("text", "mid"),
("embed", 1),
("embed", 2),
("text", "tail"),
],
],
ids=[
"single-embed",
"embed-then-text",
"text-then-embed",
"text-embed-text",
"text-embed-text-embed",
"adjacent-embeds",
"text-text-embed",
"long-mixed-run",
],
)
@pytest.mark.parametrize(
"interleave_mm_strings",
# `None`: text-only path where `multimodal_config` is absent.
# `False`: non-interleave multimodal path (the common default).
# `True`: sentinel-substitution interleave path.
# All three must preserve the request ordering of prompt_embeds
# relative to surrounding text because prompt_embeds are spliced at the
# token offset during rendering.
[None, False, True],
ids=["text-only", "interleave-off", "interleave-on"],
)
def test_parse_chat_messages_string_format_preserves_position(
layout, interleave_mm_strings
):
mc = _make_mock_model_config()
if interleave_mm_strings is not None:
mm_cfg = mock.MagicMock()
mm_cfg.interleave_mm_strings = interleave_mm_strings
mc.multimodal_config = mm_cfg
content: list[dict] = []
expected_parts: list[str] = []
expected_embeds: list[torch.Tensor] = []
for kind, value in layout:
if kind == "text":
content.append({"type": "text", "text": value})
expected_parts.append(value)
else: # prompt embeds
num_tokens = value
t = torch.randn(num_tokens, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
expected_embeds.append(t)
content.append({"type": "prompt_embeds", "data": _encode_tensor(t)})
# Parser emits ONE sentinel per part.
expected_parts.append(PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
messages = [{"role": "user", "content": content}]
conv, mm_data, _ = parse_chat_messages(
messages,
mc,
content_format="string",
)
assert conv[0]["content"] == "\n".join(expected_parts)
assert mm_data is not None and "prompt_embeds" in mm_data
assert len(mm_data["prompt_embeds"]) == len(expected_embeds)
for got, want in zip(mm_data["prompt_embeds"], expected_embeds, strict=True):
assert torch.equal(got, want)
def test_parse_chat_messages_requires_flag():
t = torch.randn(2, 4)
b64 = _encode_tensor(t)
mc = _make_mock_model_config(enable_prompt_embeds=False)
messages = [
{
"role": "user",
"content": [{"type": "prompt_embeds", "data": b64}],
}
]
with pytest.raises(VLLMValidationError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
parse_chat_messages(
messages,
mc,
content_format="openai",
)
def test_parse_chat_messages_rejects_missing_data():
# `data` is marked `Required` on `ChatCompletionContentPartPromptEmbedsParam`;
# malformed requests without `data` must surface a clear validation error
# rather than being silently dropped.
mc = _make_mock_model_config()
messages = [
{
"role": "user",
"content": [{"type": "prompt_embeds"}], # no `data`
}
]
with pytest.raises(VLLMValidationError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
parse_chat_messages(
messages,
mc,
content_format="openai",
)
# Reserved placeholder guard: when `enable_prompt_embeds=True` the tokenizer is
# mutated to make `<prompt_embeds>` a single unsplittable token. Any user text
# containing that literal sequence would tokenize to the same sentinel ID and
# be mistaken for a splice point, so we reject it at parse time.
_PLACEHOLDER_ERROR_PATTERN: Final[str] = re.sub(
r"\\{[^}]*\\}", ".*", re.escape(_RESERVED_PLACEHOLDER_IN_TEXT_ERROR)
)
@pytest.mark.parametrize(
"content",
[
# Case: Top-level string content (wrapped as a single text part).
f"hello {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} world",
# Case: List with a typed text part containing the placeholder.
[{"type": "text", "text": f"leading {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"}],
# Case: List with a plain-string part (no wrapping dict).
[f"raw string {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"],
],
ids=["top-level-string", "typed-text-part", "plain-string-part"],
)
def test_parse_chat_messages_rejects_placeholder_in_user_text(content):
mc = _make_mock_model_config() # enable_prompt_embeds=True by default
messages = [{"role": "user", "content": content}]
with pytest.raises(VLLMValidationError, match=_PLACEHOLDER_ERROR_PATTERN):
parse_chat_messages(messages, mc, content_format="openai")
def test_parse_chat_messages_allows_placeholder_in_text_when_feature_disabled():
# When `enable_prompt_embeds=False` the tokenizer is never mutated, so the
# literal `<prompt_embeds>` is just ordinary text and must pass through.
mc = _make_mock_model_config(enable_prompt_embeds=False)
messages = [
{
"role": "user",
"content": f"benign mention of {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} here",
}
]
conv, mm_data, _ = parse_chat_messages(messages, mc, content_format="openai")
assert mm_data is None or "prompt_embeds" not in mm_data
# Text reaches the rendered conversation unchanged.
texts = [p["text"] for p in conv[0]["content"]]
assert PROMPT_EMBEDS_PLACEHOLDER_TOKEN in "".join(texts)
# Token-stream spec: ints are regular token IDs, tuples `(N,)` expand to
# a placeholder span of length N (creates corresponding `(N, H)` tensor).
# `expected` lists the `(start_idx, length)` pairs that
# `_build_prompt_embeds_positions` should return.
@pytest.mark.parametrize(
"stream, expected",
[
# Case: Single run in the middle.
([10, 20, (3,), 30], [(2, 3)]),
# Case: Single run at the start.
([(2,), 10, 20], [(0, 2)]),
# Case: Single run at the end.
([10, 20, (4,)], [(2, 4)]),
# Case: Two runs with tokens between.
([1, (2,), 2, 3, (3,), 4], [(1, 2), (5, 3)]),
# Case: Adjacent runs (no separating tokens).
([(1,), (2,)], [(0, 1), (1, 2)]),
# Case: Three runs.
([5, (2,), 6, (1,), 7, (3,), 8], [(1, 2), (4, 1), (6, 3)]),
],
ids=[
"single-middle",
"single-start",
"single-end",
"two-runs-separated",
"two-runs-adjacent",
"three-runs",
],
)
def test_build_positions(tokenizer, stream, expected):
H = 4
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
tensors: list[torch.Tensor] = []
token_ids: list[int] = []
for item in stream:
if isinstance(item, tuple):
length = item[0]
tensors.append(torch.randn(length, H))
token_ids.extend([tid] * length)
else:
token_ids.append(item)
mm_updates = _build_prompt_embeds_updates(tensors, tid)
positions = _build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
assert positions == expected
def test_build_positions_length_mismatch(tokenizer):
N1, H1 = 2, 4
N2, H2 = 3, 4
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
# 2 tensors expected but only a single placeholder run in the token
# stream (simulating dropping the second one).
tensors = [torch.randn(N1, H1), torch.randn(N2, H2)]
token_ids = [1, tid, tid, 2, 3]
mm_updates = _build_prompt_embeds_updates(tensors, tid)
# The error constant is a `str.format` template, escape it and turn
# the `{field}` placeholders into `.*` so it matches any substitution.
pattern = re.sub(
r"\\{[^}]*\\}", ".*", re.escape(_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR)
)
with pytest.raises(ValueError, match=pattern):
_build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
# ints = regular token IDs (any value)
# (N,) = embed span of length N
@pytest.mark.parametrize(
"stream",
[
[10, 20, (3,), 30],
[(2,), 10, 20],
[10, 20, (4,)],
[1, (2,), 2, 3, (3,), 4],
[(1,), (2,)],
[5, (2,), 6, (1,), 7, (3,), 8],
],
ids=[
"single-middle",
"single-start",
"single-end",
"two-spans-separated",
"two-spans-adjacent",
"three-spans",
],
)
def test_build_mixed_prompt_embeds(stream):
H = 8
_PLACEHOLDER = 0 # sentinel for embed positions in token_ids
tensors: list[torch.Tensor] = []
token_ids: list[int] = []
positions: list[tuple[int, int]] = []
cursor = 0
for item in stream:
if isinstance(item, tuple):
length = item[0]
tensors.append(torch.randn(length, H))
positions.append((cursor, length))
token_ids.extend([_PLACEHOLDER] * length)
cursor += length
else:
token_ids.append(item)
cursor += 1
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
assert embeds.shape == (len(token_ids), H)
assert len(mask) == len(token_ids)
# Mask: False exactly at embed positions, True everywhere else.
expected_mask = torch.ones(len(token_ids), dtype=torch.bool)
for start, length in positions:
expected_mask[start : start + length] = False
assert mask == expected_mask.tolist()
# Embed rows match input tensors at the right positions.
for tensor, (start, length) in zip(tensors, positions):
assert torch.equal(embeds[start : start + length], tensor)
# Non-embed positions remain zero-filled.
assert torch.all(embeds[expected_mask] == 0)
# End-to-end tests: each runs both sync and async parse paths via the
# `parse_fn` fixture.
@pytest.mark.asyncio
@pytest.mark.parametrize("role", ["user", "system"])
async def test_end_to_end_expand_and_build(tokenizer, parse_fn, role):
"""Full renderer pipeline: parse -> chat template -> expand -> locate
-> build mixed prompt, across tokenizers, roles, and sync/async."""
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
LEN_A, LEN_B = 3, 2
t_a = torch.randn(LEN_A, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
t_b = torch.randn(LEN_B, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
NUM_TENSORS = 2
mc = _make_mock_model_config()
messages = [
{
"role": role,
"content": [
{"type": "text", "text": "Hello "},
{"type": "prompt_embeds", "data": _encode_tensor(t_a)},
{"type": "text", "text": " world "},
{"type": "prompt_embeds", "data": _encode_tensor(t_b)},
{"type": "text", "text": "!"},
],
}
]
conv, mm_data, _ = await _maybe_await(
parse_fn, messages, mc, content_format="openai"
)
tensors = list(mm_data["prompt_embeds"])
assert len(tensors) == NUM_TENSORS
# Tokenize: each prompt_embeds part becomes 1 placeholder token.
# `return_dict=False` to get a flat `list[int]` on transformers v5
# (where the default flipped to True and yields a `BatchEncoding` dict).
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
assert sum(t == tid for t in token_ids) == NUM_TENSORS
# Expand, locate, and build.
mm_updates = _build_prompt_embeds_updates(tensors, tid)
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
assert len(expanded) == len(token_ids) + LEN_A + LEN_B - NUM_TENSORS
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
assert positions[0][1] == LEN_A
assert positions[1][1] == LEN_B
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
assert mask.count(False) == LEN_A + LEN_B
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_A], t_a)
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_B], t_b)
@pytest.mark.asyncio
async def test_end_to_end_multi_message_conversation(tokenizer, parse_fn):
"""Full pipeline with prompt_embeds spread across system + user messages,
verifying ordering and positioning in the final token stream."""
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
LEN_SYS, LEN_USR = 4, 3
t_sys = torch.randn(LEN_SYS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
t_usr = torch.randn(LEN_USR, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
NUM_TENSORS = 2 # t_sys and t_usr.
mc = _make_mock_model_config()
messages = [
{
"role": "system",
"content": [
{"type": "text", "text": "You are helpful."},
{"type": "prompt_embeds", "data": _encode_tensor(t_sys)},
],
},
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": _encode_tensor(t_usr)},
{"type": "text", "text": "Summarize."},
],
},
]
conv, mm_data, _ = await _maybe_await(
parse_fn, messages, mc, content_format="openai"
)
tensors = list(mm_data["prompt_embeds"])
assert len(tensors) == NUM_TENSORS
# Tokenize, expand, locate, and build.
# `return_dict=False` to get a flat `list[int]` on transformers v5
# (where the default flipped to True and yields a `BatchEncoding` dict).
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
mm_updates = _build_prompt_embeds_updates(tensors, tid)
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
assert positions[0][1] == LEN_SYS
assert positions[1][1] == LEN_USR
# System embed must appear before user embed in the token stream.
assert positions[0][0] < positions[1][0]
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
assert mask.count(False) == LEN_SYS + LEN_USR
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_SYS], t_sys)
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_USR], t_usr)
def test_truncation_keeps_the_mixed_mask_aligned_with_the_prompt():
"""`prompt_is_token_ids` must be truncated with `prompt_token_ids`.
The mask is positional: the engine pairs it element-wise with the prompt
tokens to decide which positions read a real token id and which read a row
of `prompt_embeds`. Truncating the prompt without truncating the mask
leaves the two describing different positions, so embed placeholders stop
being recognised as such.
"""
hidden_size = 8
placeholder_token_id = 0
num_head, embed_len, num_tail = 10, 5, 5
token_ids = (
list(range(1, num_head + 1))
+ [placeholder_token_id] * embed_len
+ list(range(1, num_tail + 1))
)
tensors = [torch.randn(embed_len, hidden_size)]
positions = [(num_head, embed_len)]
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
prompt = {
"prompt_token_ids": token_ids,
"prompt_embeds": embeds,
"prompt_is_token_ids": mask,
}
keep = 8
tok_params = TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=keep,
truncation_side="left",
)
result = tok_params.apply_post_tokenization(None, prompt)
# Keeping the last 8 of 20 positions starts inside the embed span, so the
# first 3 surviving positions are embed rows and the rest are real tokens.
# Comparing contents rather than lengths is what pins the alignment: a
# mask reduced from the wrong side has the right length and still
# describes the wrong positions.
assert result["prompt_is_token_ids"] == [False] * 3 + [True] * num_tail
assert result["prompt_token_ids"] == token_ids[-keep:]
assert torch.equal(result["prompt_embeds"], embeds[-keep:])
File diff suppressed because it is too large Load Diff
-796
View File
@@ -1,796 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
import pybase64
import pytest
import torch
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import SingletonPrompt
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import HfRenderer
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
MODEL_NAME = "openai-community/gpt2"
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
runner_type = "generate"
model: str = MODEL_NAME
tokenizer: str = MODEL_NAME
trust_remote_code: bool = False
tokenizer_revision = None
tokenizer_mode = "auto"
hf_config = MockHFConfig()
hf_text_config = hf_config
encoder_config: dict[str, Any] | None = None
enable_prompt_embeds: bool = True
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
supports_multimodal_inputs: bool = False
renderer_num_workers: int = 1
hidden_size: int = 768
dtype: torch.dtype = torch.float32
def get_hidden_size(self) -> int:
return self.hidden_size
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
@dataclass
class DummyTokenizer:
truncation_side: str = "left"
max_chars_per_token: int = 1
# Deliberately outside the range of ids `encode` returns, so a test can
# tell a pad token apart from a real one.
pad_token_id: int = 99999
def __post_init__(self) -> None:
self._captured_encode_kwargs: dict = {}
self._captured_text_len: int = 0
def decode(self, tokens: list[int]):
return str(tokens)
def encode(self, text: str, **kwargs):
self._captured_encode_kwargs = kwargs
self._captured_text_len = len(text)
in_length = len(text)
truncation = kwargs.get("truncation")
max_length = kwargs.get("max_length")
if truncation and max_length is not None:
return list(range(min(in_length, max_length)))
return list(range(in_length))
def __call__(self, text: str, **kwargs):
# BaseRenderer._tokenize_prompt calls the tokenizer via __call__ (to
# unify the output type), so mirror a real tokenizer's BatchEncoding.
return {"input_ids": self.encode(text, **kwargs)}
def _build_renderer(
model_config: MockModelConfig,
*,
truncation_side: str = "left",
max_chars_per_token: int = 1,
):
renderer = HfRenderer(
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
tokenizer=(
None
if model_config.skip_tokenizer_init
else DummyTokenizer(
truncation_side=truncation_side,
max_chars_per_token=max_chars_per_token,
)
),
)
return renderer
def _preprocess_prompt(
model_config: ModelConfig,
prompt_or_prompts: SingletonPrompt | bytes | Sequence[SingletonPrompt | bytes],
):
return [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompt_to_seq(prompt_or_prompts)
]
class TestValidatePrompt:
def test_empty_input(self):
renderer = _build_renderer(MockModelConfig())
with pytest.raises(ValueError, match="at least one prompt"):
renderer.render_prompts(_preprocess_prompt(renderer.model_config, []))
def test_invalid_type(self):
renderer = _build_renderer(MockModelConfig())
with pytest.raises(TypeError, match="should be a list of integers"):
renderer.render_prompts(
_preprocess_prompt(renderer.model_config, [[1, 2], ["foo", "bar"]])
)
class TestRenderPrompt:
def test_tokens_input(self):
renderer = _build_renderer(MockModelConfig())
tokens = [101, 7592, 2088]
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, tokens)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 1
assert results[0]["prompt_token_ids"] == tokens
def test_token_list_input(self):
renderer = _build_renderer(MockModelConfig())
token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, token_lists)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 3
assert results[0]["prompt_token_ids"] == [101, 7592, 2088]
assert results[1]["prompt_token_ids"] == [102, 1234, 5678, 9012]
assert results[2]["prompt_token_ids"] == [103, 4567]
def test_text_input(self):
renderer = _build_renderer(MockModelConfig())
text_input = "x" * 10
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, text_input)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 10
def test_text_list_input(self):
renderer = _build_renderer(MockModelConfig())
text_list_input = ["x" * 10, "x" * 12, "x" * 14]
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, text_list_input)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 3
for text_input, result in zip(text_list_input, results):
assert len(result["prompt_token_ids"]) == len(text_input)
def test_zero_truncation(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 200)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=0),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 0
def test_pos_truncation(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 200)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=50),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 50
def test_neg_truncation(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 200)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=-1),
)
assert len(results) == 1
# max_output_tokens defaults to 0, so there is no output budget to
# reserve and -1 truncates to the full max_total_tokens.
assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens
def test_truncation_left(self):
renderer = _build_renderer(MockModelConfig(), truncation_side="left")
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, long_tokens)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),
)
assert len(results) == 1
# Should keep the last 5 tokens: [105, 106, 107, 108, 109]
assert results[0]["prompt_token_ids"] == [105, 106, 107, 108, 109]
def test_truncation_right(self):
renderer = _build_renderer(MockModelConfig(), truncation_side="right")
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, long_tokens)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),
)
assert len(results) == 1
# Should keep the first 5 tokens: [100, 101, 102, 103, 104]
assert results[0]["prompt_token_ids"] == [100, 101, 102, 103, 104]
def test_text_max_length_exceeded_obvious(self):
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
# Exceeds max_total_tokens and max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
long_tokens = "x" * 150
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, long_tokens)
)
with pytest.raises(
VLLMValidationError,
match="maximum context length is",
):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
# Should not even attempt tokenization
assert renderer.tokenizer._captured_encode_kwargs == {}
def test_text_max_length_exceeded_nonobvious(self):
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=2)
# Exceeds max_total_tokens but not max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
long_tokens = "x" * 150
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, long_tokens)
)
with pytest.raises(
VLLMValidationError,
match="maximum context length is",
):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
# Should only tokenize the first max_total_tokens + 1 tokens
assert renderer.tokenizer._captured_encode_kwargs["truncation"] is True
assert renderer.tokenizer._captured_encode_kwargs["max_length"] == 101
def test_token_max_length_exceeded(self):
renderer = _build_renderer(MockModelConfig())
long_tokens = list(range(150)) # Exceeds max_total_tokens=100
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, long_tokens)
)
with pytest.raises(
VLLMValidationError,
match="maximum context length is",
):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=None),
)
def test_no_tokenizer_for_text(self):
renderer = _build_renderer(MockModelConfig(skip_tokenizer_init=True))
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "Hello world")
)
with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
def test_tokens_input_with_needs_detokenization(self):
renderer = _build_renderer(MockModelConfig())
tokens = [1, 2, 3, 4]
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, tokens)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
needs_detokenization=True,
),
)
assert len(results) == 1
assert results[0]["prompt_token_ids"] == tokens
assert results[0]["prompt"] == "[1, 2, 3, 4]"
@pytest.mark.parametrize(
("truncation_side", "expected"),
[("left", [106, 107, 108, 109]), ("right", [100, 101, 102, 103])],
)
def test_detokenization_matches_truncated_tokens(self, truncation_side, expected):
"""Echoed prompt text must match the truncated tokens sent to the engine."""
renderer = _build_renderer(MockModelConfig())
params = TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=4,
truncation_side=truncation_side,
needs_detokenization=True,
)
def render():
return renderer.render_prompts(
_preprocess_prompt(renderer.model_config, list(range(100, 110)))
)
results = renderer.tokenize_prompts(render(), params)
async_results = asyncio.run(renderer.tokenize_prompts_async(render(), params))
for result in (*results, *async_results):
assert result["prompt_token_ids"] == expected
assert result["prompt"] == str(expected)
def test_explicit_side_tokenizer_unbounded(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 500)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=4,
truncation_side="left",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 4
kwargs = renderer.tokenizer._captured_encode_kwargs
assert kwargs["truncation"] is False
def test_explicit_side_left_text(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=5,
truncation_side="left",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 5
assert results[0]["prompt_token_ids"] == list(range(45, 50))
def test_explicit_side_right_text(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=5,
truncation_side="right",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 5
assert results[0]["prompt_token_ids"] == list(range(5))
def test_padding_with_left_truncation_keeps_the_prompt(self):
"""Padding must not be truncated away.
`padding` and `truncate_prompt_tokens`/`truncation_side` are all
settable on one pooling request. Padding to the full input length
before truncating from the left leaves a prompt made entirely of pad
tokens, and the request still succeeds -- so the model embeds nothing
but padding.
"""
renderer = _build_renderer(MockModelConfig())
pad_id = renderer.tokenizer.pad_token_id
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
pad_prompt_tokens=-1,
truncate_prompt_tokens=5,
truncation_side="left",
),
)
assert len(results) == 1
token_ids = results[0]["prompt_token_ids"]
# The sentinel: on a padding-first pipeline every surviving id is a
# pad token, so the prompt reaches the model with no content at all.
assert set(token_ids) != {pad_id}
# Transformers semantics: truncate to 5, then pad out to the full
# input length.
assert token_ids[:5] == list(range(45, 50))
assert token_ids[5:] == [pad_id] * 95
def test_padding_without_truncation_is_unchanged(self):
renderer = _build_renderer(MockModelConfig())
pad_id = renderer.tokenizer.pad_token_id
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, pad_prompt_tokens=-1),
)
assert len(results) == 1
assert results[0]["prompt_token_ids"] == list(range(50)) + [pad_id] * 50
def test_explicit_side_text_pretokenization_guard(self):
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 500)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=4,
truncation_side="left",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 4
assert renderer.tokenizer._captured_text_len <= 100
class TestMaxTokensNotReservation:
"""Regression tests for GitHub issue #42474.
``max_tokens`` must be treated as an upper bound on generation length, not
as a hard reservation that is subtracted from the available input space.
"""
def test_large_max_tokens_does_not_eat_prompt_budget(self):
"""Prompt fits in context even when max_tokens is large."""
renderer = _build_renderer(MockModelConfig())
# 50-token prompt with a 100-token context window and max_tokens=60.
# Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error.
# After the fix: max_input_tokens = 100 → 50 ≤ 100 → OK.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=60),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 50
def test_max_tokens_zero_still_rejects_oversized_prompt(self):
"""Genuine prompt overflow is still caught regardless of max_tokens."""
renderer = _build_renderer(MockModelConfig())
# 150-token prompt with 100-token context → should still fail.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 150)
)
with pytest.raises(VLLMValidationError, match="maximum context length"):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=0),
)
def test_negative_truncation_uses_max_total(self):
"""truncate_prompt_tokens=-1 reserves one output token."""
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 200)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
max_output_tokens=40,
truncate_prompt_tokens=-1,
),
)
# -1 maps to max_total_tokens (100) rather than the old
# max_total - max_output (60), minus the one token held back so
# truncation cannot leave a zero-token output budget.
assert len(results[0]["prompt_token_ids"]) == 99
def test_negative_pad_uses_max_total(self):
"""pad_prompt_tokens=-1 pads to max_total_tokens, not max_input."""
renderer = _build_renderer(MockModelConfig())
pad_id = renderer.tokenizer.pad_token_id
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 30)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
max_output_tokens=40,
pad_prompt_tokens=-1,
),
)
# -1 should map to max_total_tokens (100), not max_total - max_output (60).
assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70
def test_max_tokens_exceeds_context_is_still_clamped_by_api(self):
"""Renderer allows the prompt; API layer clamps max_tokens later."""
renderer = _build_renderer(MockModelConfig())
# Simulates the reported scenario: max_tokens > context - prompt_length.
# Renderer should accept the 40-token prompt; get_max_tokens() clamps
# sampling max_tokens to 60 at the entrypoint layer.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 40)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=80),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 40
def test_with_kwargs_preserves_output_budget(self):
"""with_kwargs without an explicit max_length keeps the output budget.
Under the old semantics `with_kwargs()` recomputed
`max_output_tokens = max_total - max_input`, which (after this fix)
would have silently reset a real output budget to zero.
"""
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
chained = tok_params.with_kwargs(truncation_side="left")
assert chained.max_output_tokens == 60
assert chained.max_input_tokens == 100
def test_with_kwargs_explicit_max_length_sets_budget(self):
"""An explicit max_length still re-derives the output budget."""
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
chained = tok_params.with_kwargs(max_length=80)
assert chained.max_output_tokens == 20
assert chained.max_input_tokens == 100
class TestRenderEmbedPrompt:
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
"""Helper to create base64-encoded tensor bytes."""
buffer = io.BytesIO()
torch.save(tensor, buffer)
buffer.seek(0)
return pybase64.b64encode(buffer.read())
def test_single_prompt_embed(self):
renderer = _build_renderer(MockModelConfig())
# Create a test tensor
tensor_input = torch.randn(10, 768, dtype=torch.float32)
embed_bytes = self._create_test_embed_bytes(tensor_input)
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, embed_bytes)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 1
assert torch.equal(results[0]["prompt_embeds"], tensor_input)
def test_multiple_prompt_embeds(self):
hidden_size = 512
renderer = _build_renderer(MockModelConfig(hidden_size=hidden_size))
# Create multiple test tensors
tensor_inputs = [
torch.randn(8, hidden_size, dtype=torch.float32),
torch.randn(12, hidden_size, dtype=torch.float32),
]
prompts = renderer.render_prompts(
_preprocess_prompt(
renderer.model_config,
[self._create_test_embed_bytes(t) for t in tensor_inputs],
)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 2
for i, result in enumerate(results):
assert torch.allclose(result["prompt_embeds"], tensor_inputs[i])
def test_prompt_embed_truncation(self):
renderer = _build_renderer(MockModelConfig())
# Create tensor with more tokens than truncation limit
tensor_input = torch.randn(20, 768, dtype=torch.float32)
prompts = renderer.render_prompts(
_preprocess_prompt(
renderer.model_config, self._create_test_embed_bytes(tensor_input)
)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=10,
),
)
assert len(results) == 1
# Should keep last 10 tokens
expected = tensor_input[-10:]
assert torch.equal(results[0]["prompt_embeds"], expected)
def test_prompt_embed_different_dtypes(self):
hidden_size = 256
# Test different supported dtypes
dtypes = [torch.float32, torch.float16, torch.bfloat16]
for dtype in dtypes:
renderer = _build_renderer(
MockModelConfig(hidden_size=hidden_size, dtype=dtype)
)
tensor_input = torch.randn(5, hidden_size, dtype=dtype)
prompts = renderer.render_prompts(
_preprocess_prompt(
renderer.model_config, self._create_test_embed_bytes(tensor_input)
)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 1
assert results[0]["prompt_embeds"].dtype == dtype
def test_prompt_embed_squeeze_batch_dim(self):
renderer = _build_renderer(MockModelConfig())
# Test tensor with batch dimension gets squeezed
tensor_input = torch.randn(1, 10, 768, dtype=torch.float32)
prompts = renderer.render_prompts(
_preprocess_prompt(
renderer.model_config, self._create_test_embed_bytes(tensor_input)
)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 1
# Should be squeezed to 2D
assert results[0]["prompt_embeds"].shape == (10, 768)
def test_both_prompts_and_embeds(self):
hidden_size = 256
renderer = _build_renderer(MockModelConfig(hidden_size=hidden_size))
text_input = "Hello world"
tensor_input = torch.randn(5, hidden_size, dtype=torch.float32)
prompts = renderer.render_prompts(
_preprocess_prompt(
renderer.model_config,
[text_input, self._create_test_embed_bytes(tensor_input)],
)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100),
)
assert len(results) == 2
# First should be tokens prompt
assert "prompt_token_ids" in results[0]
assert len(results[0]["prompt_token_ids"]) == len(text_input)
# Second should be embed prompt
assert torch.equal(results[1]["prompt_embeds"], tensor_input)
File diff suppressed because it is too large Load Diff
-484
View File
@@ -1,484 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Golden tests for the native Inkling renderer encoding.
The message fixtures mirror the Rust renderer's fixture tests
(``rust/src/chat/src/renderer/inkling/tests.rs``) so the two frontends stay
in token-level parity. Image blocks emit the bare ``<|content_image|>`` marker;
``InklingMultiModalProcessor`` inserts the per-patch placeholder run.
"""
import pytest
from vllm.renderers.inkling import (
InklingRenderer,
_HfBackedTmlTokenizer,
_resolve_reasoning_effort,
)
from vllm.renderers.inkling_encoding import (
SPECIAL_TOKEN_SPELLINGS,
render_inkling_messages,
)
from vllm.renderers.params import ChatParams
@pytest.fixture()
def should_do_global_cleanup_after_test() -> bool:
# These tests touch no distributed or device state; the global
# cleanup fixture is unnecessary (and trips a torch MPS allocator
# assert on macOS dev machines).
return False
# One id per special token, mirroring the real Inkling vocab layout; plain
# text encodes one token per character so decoded output is exact.
_SPECIAL_VOCAB = {
"<|message_user|>": 200000,
"<|message_model|>": 200001,
"<|message_system|>": 200002,
"<|message_tool|>": 200003,
"<|content_text|>": 200004,
"<|content_image|>": 200005,
"<|content_model_end_sampling|>": 200006,
"<|content_thinking|>": 200008,
"<|end_message|>": 200010,
"<|content_audio_input|>": 200020,
# The HF vocab spells CONTENT_XML as an unused slot.
"<|unused_200024|>": 200024,
"<|audio_end|>": 200043,
"<|content_invoke_tool_json|>": 200049,
}
_ID_TO_SPECIAL = {v: k for k, v in _SPECIAL_VOCAB.items()}
class FakeHfTokenizer:
def get_vocab(self):
return dict(_SPECIAL_VOCAB)
def encode(self, text, add_special_tokens=False):
assert not add_special_tokens
return [ord(ch) for ch in text]
def decode(token_ids):
return "".join(
_ID_TO_SPECIAL.get(tid, chr(tid) if tid < 200000 else f"<{tid}>")
for tid in token_ids
)
@pytest.fixture
def inkling_tokenizer():
return _HfBackedTmlTokenizer(FakeHfTokenizer())
def render_text(inkling_tokenizer, messages, **kwargs):
return decode(render_inkling_messages(messages, inkling_tokenizer, **kwargs))
class TestRustFixtureParity:
def test_tool_round_trip(self, inkling_tokenizer):
messages = [
{
"role": "assistant",
"reasoning_content": "think",
"content": "answer",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": '{"city":"SF"}',
},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
assert render_text(
inkling_tokenizer, messages, add_generation_prompt=False
) == (
"<|message_model|><|content_thinking|>think<|end_message|>"
"<|message_model|><|content_text|>answer<|end_message|>"
"<|message_model|>get_weather<|content_invoke_tool_json|>"
'{"name":"get_weather","args":{"city":"SF"}}<|end_message|>'
"<|content_model_end_sampling|>"
"<|message_tool|>get_weather<|content_text|>sunny<|end_message|>"
)
def test_tool_declare(self, inkling_tokenizer):
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"required": ["city"],
"properties": {"city": {"type": "string"}},
},
},
}
]
messages = [
{
"role": "developer",
"content": "rules",
"tools": [
{
"type": "function",
"function": {
"name": "local_tool",
"parameters": {"z": 1, "a": {"b": 2}},
},
}
],
},
{"role": "user", "content": "hi"},
]
assert render_text(inkling_tokenizer, messages, tools=tools) == (
"<|message_system|>tool_declare<|unused_200024|>"
'[{"description":"Get weather information","name":"get_weather",'
'"parameters":{"properties":{"city":{"type":"string"}},'
'"required":["city"],"type":"object"},"type":"function"},'
'{"description":"","name":"local_tool",'
'"parameters":{"a":{"b":2},"z":1},"type":"function"}]'
"<|end_message|>"
"<|message_system|><|content_text|>rules<|end_message|>"
"<|message_user|><|content_text|>hi<|end_message|>"
"<|message_model|>"
)
def test_text_image(self, inkling_tokenizer):
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "look"},
{"type": "image_url", "image_url": "data:image/png;base64,"},
],
}
]
# Multimodal preprocessing expands this bare marker after rendering.
assert render_text(inkling_tokenizer, messages) == (
"<|message_user|><|content_text|>look<|end_message|>"
"<|message_user|><|content_image|><|end_message|>"
"<|message_model|>"
)
class TestRenderingSemantics:
@pytest.mark.parametrize(
("value", "expected"),
[
("none", 0.0),
("minimal", 0.1),
("low", 0.2),
("medium", 0.7),
("high", 0.9),
("xhigh", 0.99),
("max", 0.99),
(None, 0.9),
(0.8, 0.8),
(True, None),
("invalid", None),
],
)
def test_resolve_reasoning_effort(self, value, expected):
assert _resolve_reasoning_effort(value) == expected
def test_generation_prompt_default(self, inkling_tokenizer):
text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}])
assert text.endswith("<|end_message|><|message_model|>")
def test_developer_folds_into_system(self, inkling_tokenizer):
assert (
render_text(
inkling_tokenizer,
[{"role": "developer", "content": "be nice"}],
add_generation_prompt=False,
)
== "<|message_system|><|content_text|>be nice<|end_message|>"
)
def test_empty_string_content_skipped(self, inkling_tokenizer):
assert (
render_text(
inkling_tokenizer,
[{"role": "user", "content": ""}],
add_generation_prompt=False,
)
== ""
)
def test_empty_reasoning_skipped(self, inkling_tokenizer):
assert (
render_text(
inkling_tokenizer,
[{"role": "assistant", "reasoning_content": "", "content": "hi"}],
add_generation_prompt=False,
)
== "<|message_model|><|content_text|>hi<|end_message|>"
"<|content_model_end_sampling|>"
)
def test_reasoning_field(self, inkling_tokenizer):
messages = [
{
"role": "assistant",
"reasoning": "think",
"content": "answer",
}
]
assert render_text(
inkling_tokenizer, messages, add_generation_prompt=False
) == (
"<|message_model|><|content_thinking|>think<|end_message|>"
"<|message_model|><|content_text|>answer<|end_message|>"
"<|content_model_end_sampling|>"
)
def test_audio_part(self, inkling_tokenizer):
messages = [
{
"role": "user",
"content": [{"type": "input_audio", "input_audio": {}}],
}
]
assert render_text(
inkling_tokenizer, messages, add_generation_prompt=False
) == ("<|message_user|><|content_audio_input|><|audio_end|><|end_message|>")
def test_tool_response_name_from_message(self, inkling_tokenizer):
assert (
render_text(
inkling_tokenizer,
[{"role": "tool", "name": "my_tool", "content": "ok"}],
add_generation_prompt=False,
)
== "<|message_tool|>my_tool<|content_text|>ok<|end_message|>"
)
def test_tool_call_args_object_form(self, inkling_tokenizer):
messages = [
{
"role": "assistant",
"tool_calls": [
{
"id": "x",
"function": {
"name": "f",
# dict-form arguments, unsorted keys
"arguments": {"z": 1, "a": 2},
},
}
],
}
]
assert render_text(
inkling_tokenizer, messages, add_generation_prompt=False
) == (
"<|message_model|>f<|content_invoke_tool_json|>"
'{"name":"f","args":{"a":2,"z":1}}<|end_message|>'
"<|content_model_end_sampling|>"
)
def test_tool_call_empty_args(self, inkling_tokenizer):
messages = [
{
"role": "assistant",
"tool_calls": [{"id": "x", "function": {"name": "f", "arguments": ""}}],
}
]
assert render_text(
inkling_tokenizer, messages, add_generation_prompt=False
) == (
"<|message_model|>f<|content_invoke_tool_json|>"
'{"name":"f","args":{}}<|end_message|>'
"<|content_model_end_sampling|>"
)
def test_tool_call_non_object_args_rejected(self, inkling_tokenizer):
messages = [
{
"role": "assistant",
"tool_calls": [
{"id": "x", "function": {"name": "f", "arguments": "[1]"}}
],
}
]
with pytest.raises(TypeError, match="decode to an object"):
render_inkling_messages(
messages, inkling_tokenizer, add_generation_prompt=False
)
def test_unsupported_role_rejected(self, inkling_tokenizer):
with pytest.raises(ValueError, match="unsupported Inkling message role"):
render_inkling_messages(
[{"role": "narrator", "content": "hi"}], inkling_tokenizer
)
class TestReasoningEffort:
def test_frontend_defaults_to_high(self, inkling_tokenizer):
renderer = InklingRenderer.__new__(InklingRenderer)
renderer._inkling_tokenizer = inkling_tokenizer
text = decode(
renderer._render(
[
{"role": "system", "content": "rules"},
{"role": "user", "content": "hi"},
],
ChatParams(),
)
)
assert text.startswith(
"<|message_system|><|content_text|>rules<|end_message|>"
"<|message_system|><|content_text|>Thinking effort level: 0.9"
"<|end_message|>"
)
@pytest.mark.parametrize("value", ["none", 0, 0.0, -0.0])
def test_zero_effort_has_one_canonical_spelling(self, inkling_tokenizer, value):
renderer = InklingRenderer.__new__(InklingRenderer)
renderer._inkling_tokenizer = inkling_tokenizer
text = decode(
renderer._render(
[{"role": "user", "content": "hi"}],
ChatParams(chat_template_kwargs={"reasoning_effort": value}),
)
)
assert text.startswith(
"<|message_system|><|content_text|>Thinking effort level: 0.0"
"<|end_message|>"
)
def test_emits_one_effort_after_initial_prefix(self, inkling_tokenizer):
effort_block = (
"<|message_system|><|content_text|>Thinking effort level: 0.7"
"<|end_message|>"
)
text = render_text(
inkling_tokenizer,
[
{"role": "system", "content": "rules"},
{"role": "developer", "content": "policy"},
{"role": "user", "content": "user1"},
{"role": "assistant", "content": "assistant1"},
{"role": "user", "content": "user2"},
],
tools=[{"type": "function", "function": {"name": "f"}}],
reasoning_effort=0.7,
)
assert text.count(effort_block) == 1
assert (
text.index("tool_declare")
< text.index("rules")
< text.index("policy")
< text.index(effort_block)
< text.index("user1")
< text.index("assistant1")
< text.index("user2")
)
def test_renders_after_tool_declare(self, inkling_tokenizer):
tools = [{"type": "function", "function": {"name": "f"}}]
text = render_text(
inkling_tokenizer,
[{"role": "user", "content": "hi"}],
tools=tools,
reasoning_effort=0.8,
)
declare_end = text.index("<|end_message|>") + len("<|end_message|>")
assert text[declare_end:].startswith(
"<|message_system|><|content_text|>Thinking effort level: 0.8"
"<|end_message|>"
)
@pytest.mark.parametrize(
("value", "expected"),
[(0.2, "0.2"), (0.7, "0.7"), (0.9, "0.9"), (0.99, "0.99")],
)
def test_formats_at_most_two_decimals(self, inkling_tokenizer, value, expected):
text = render_text(
inkling_tokenizer,
[{"role": "user", "content": "hi"}],
reasoning_effort=value,
)
assert f"Thinking effort level: {expected}<|end_message|>" in text
def test_absent_emits_no_block(self, inkling_tokenizer):
text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}])
assert "Thinking effort" not in text
@pytest.mark.parametrize("value", [0.9900001, 1, 1.5, -0.1])
def test_out_of_range_rejected(self, inkling_tokenizer, value):
with pytest.raises(ValueError, match="must be in"):
render_inkling_messages(
[{"role": "user", "content": "hi"}],
inkling_tokenizer,
reasoning_effort=value,
)
class TestSpecialTokenResolution:
def test_missing_special_token_raises(self):
class IncompleteTokenizer(FakeHfTokenizer):
def get_vocab(self):
vocab = dict(_SPECIAL_VOCAB)
del vocab["<|content_invoke_tool_json|>"]
return vocab
with pytest.raises(ValueError, match="missing special tokens"):
_HfBackedTmlTokenizer(IncompleteTokenizer())
def test_semantic_spelling_preferred(self):
class SemanticSpellingTokenizer(FakeHfTokenizer):
def get_vocab(self):
vocab = dict(_SPECIAL_VOCAB)
del vocab["<|unused_200024|>"]
vocab["<|content_xml|>"] = 200024
return vocab
inkling_tokenizer = _HfBackedTmlTokenizer(SemanticSpellingTokenizer())
tools = [{"type": "function", "function": {"name": "f"}}]
ids = render_inkling_messages(
[{"role": "user", "content": "hi"}], inkling_tokenizer, tools=tools
)
assert 200024 in ids
def test_all_spellings_covered(self):
# Every semantic token must resolve from the reference vocab.
for token, spellings in SPECIAL_TOKEN_SPELLINGS.items():
assert any(s in _SPECIAL_VOCAB for s in spellings), token
def test_render_inkling_messages_direct_protocol():
"""The encoding core only needs the structural tokenizer protocol."""
class ProtocolTokenizer:
def encode_text(self, text):
return [ord(c) for c in text]
def encode_special(self, token):
return {
"<|message_user|>": 200000,
"<|message_model|>": 200001,
"<|content_text|>": 200004,
"<|content_model_end_sampling|>": 200006,
"<|end_message|>": 200010,
}[token]
ids = render_inkling_messages(
[{"role": "user", "content": "hi"}],
ProtocolTokenizer(),
add_generation_prompt=True,
)
assert ids == [200000, 200004, ord("h"), ord("i"), 200010, 200001]
-354
View File
@@ -1,354 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any
import pytest
from vllm.exceptions import VLLMValidationError
from vllm.renderers import ChatParams
from vllm.renderers.kimi_k3 import KimiK3Renderer, _merge_k3_media_io_kwargs
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.registry import TokenizerRegistry
class StubTokenizer:
"""Stands in for the model's TikTokenTokenizer.
Records the kwargs it is called with and returns fixed token ids, so tests
can assert how the renderer drives ``apply_chat_template`` without the real
(git-LFS) tiktoken vocabulary.
"""
def __init__(self, token_ids: list[int]) -> None:
self.token_ids = token_ids
self.calls: list[dict[str, Any]] = []
self.conversations: list[list[dict[str, Any]]] = []
def apply_chat_template(self, conversation, **kwargs) -> list[int]:
self.conversations.append(conversation)
self.calls.append(kwargs)
return list(self.token_ids)
@dataclass
class MockHFConfig:
model_type: str = "kimi_k3"
@dataclass
class MockModelConfig:
runner_type: str = "generate"
is_multimodal_model: bool = False
supports_multimodal_inputs: bool = False
multimodal_config: Any = None
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
allowed_local_media_path: str = ""
allowed_media_domains: Any = None
enable_prompt_embeds: bool = False
renderer_num_workers: int = 1
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
def _make_renderer(tokenizer: StubTokenizer) -> KimiK3Renderer:
config = MockVllmConfig(MockModelConfig(), MockParallelConfig())
return KimiK3Renderer(config, tokenizer)
def test_kimi_k3_registered():
assert RENDERER_REGISTRY.load_renderer_cls("kimi_k3").__name__ == "KimiK3Renderer"
assert (
TokenizerRegistry.load_tokenizer_cls("kimi_k3").__name__ == "CachedHfTokenizer"
)
def test_k3_media_io_defaults_preserve_original_mode():
# Default: K3 keeps the original image mode (no background flattening).
assert _merge_k3_media_io_kwargs(None) == {"image": {"image_mode": None}}
# Server-/request-level values take precedence over the K3 default.
assert _merge_k3_media_io_kwargs({"image": {"image_mode": "RGB"}}) == {
"image": {"image_mode": "RGB"}
}
# Unrelated image kwargs are merged with the default.
assert _merge_k3_media_io_kwargs(
{"image": {"rgba_background_color": (0, 0, 0)}}
) == {"image": {"image_mode": None, "rgba_background_color": (0, 0, 0)}}
def test_apply_chat_template_forces_tokenize_and_pins_return_dict():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
tools = [{"type": "function", "function": {"name": "search"}}]
params = ChatParams(
chat_template_kwargs={"tools": tools, "tokenize": False, "thinking": True}
)
token_ids = renderer._apply_chat_template(
[{"role": "user", "content": "hi"}], params
)
assert token_ids == [7, 8, 9]
kwargs = tokenizer.calls[-1]
# tokenize is forced on even though the request asked for False, so K3 keeps
# the special-vs-ordinary token distinction instead of re-tokenizing a string.
assert kwargs["tokenize"] is True
# return_dict is pinned False so we always get a flat list of ids.
assert kwargs["return_dict"] is False
assert kwargs["tools"] == tools
assert kwargs["thinking"] is True
def test_apply_chat_template_translates_standard_thinking_kwargs():
# Standard enable_thinking/reasoning_effort kwargs must be translated
# to K3's native thinking/thinking_effort.
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={"enable_thinking": False, "reasoning_effort": "none"}
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["thinking"] is False
assert "thinking_effort" not in kwargs
assert "enable_thinking" not in kwargs
assert "reasoning_effort" not in kwargs
@pytest.mark.parametrize("reasoning_effort", ["low", "high", "max"])
def test_apply_chat_template_translates_supported_reasoning_effort(
reasoning_effort: str,
):
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["thinking_effort"] == reasoning_effort
assert "reasoning_effort" not in kwargs
@pytest.mark.parametrize("reasoning_effort", ["minimal", "medium", "xhigh"])
def test_apply_chat_template_rejects_unsupported_reasoning_effort(
reasoning_effort: str,
):
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert exc_info.value.parameter == "thinking_effort"
assert exc_info.value.value == reasoning_effort
assert tokenizer.calls == []
def test_apply_chat_template_validates_canonical_native_thinking_effort():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={
"thinking_effort": "low",
"reasoning_effort": "medium",
}
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert tokenizer.calls[-1]["thinking_effort"] == "low"
@pytest.mark.parametrize("thinking_effort", ["none", "minimal", "medium", "xhigh"])
def test_apply_chat_template_rejects_unsupported_native_thinking_effort(
thinking_effort: str,
):
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(chat_template_kwargs={"thinking_effort": thinking_effort})
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert exc_info.value.parameter == "thinking_effort"
assert exc_info.value.value == thinking_effort
assert tokenizer.calls == []
def test_apply_chat_template_native_k3_kwargs_take_precedence():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={
"thinking": True,
"enable_thinking": False,
"thinking_effort": "low",
"reasoning_effort": "high",
}
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["thinking"] is True
assert kwargs["thinking_effort"] == "low"
def test_apply_chat_template_adds_k3_api_metadata():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
response_format = {"type": "json_object"}
params = ChatParams(
tool_choice="required",
response_format=response_format,
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
kwargs = tokenizer.calls[-1]
assert kwargs["tool_choice"] == "required"
assert kwargs["response_format"] == response_format
def test_apply_chat_template_auto_tool_choice_keeps_template_kwarg():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
params = ChatParams(
chat_template_kwargs={"tool_choice": "required"},
tool_choice="auto",
)
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
assert tokenizer.calls[-1]["tool_choice"] == "required"
def test_apply_chat_template_omits_tool_choice_without_tools():
tokenizer = StubTokenizer([7, 8, 9])
renderer = _make_renderer(tokenizer)
renderer._apply_chat_template(
[{"role": "user", "content": "hi"}], ChatParams(tool_choice=None)
)
assert "tool_choice" not in tokenizer.calls[-1]
def test_render_messages_returns_token_prompt():
renderer = _make_renderer(StubTokenizer([1, 2, 3]))
conversation, prompt = renderer.render_messages(
[{"role": "user", "content": "hi"}], ChatParams()
)
assert prompt == {"prompt_token_ids": [1, 2, 3]}
assert "multi_modal_data" not in prompt
assert conversation[0]["role"] == "user"
def test_render_messages_derives_private_xtml_tool_attrs():
tokenizer = StubTokenizer([1, 2, 3])
renderer = _make_renderer(tokenizer)
conversation, _ = renderer.render_messages(
[
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "lookup:0",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
},
{
"id": "lookup:1",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
},
],
},
{
"role": "tool",
"tool_call_id": "lookup:1",
"tool": "client-supplied-name",
"index": 99,
"content": "second",
},
{
"role": "tool",
"tool_call_id": "lookup:0",
"content": "first",
},
],
ChatParams(),
)
assert [message["content"] for message in conversation[1:]] == [
"first",
"second",
]
assert conversation[1]["tool"] == "lookup"
assert conversation[1]["index"] == 1
assert conversation[2]["tool"] == "lookup"
assert conversation[2]["index"] == 2
assert tokenizer.conversations[-1] == conversation
def test_render_messages_ignores_client_supplied_xtml_tool_attrs():
tokenizer = StubTokenizer([1, 2, 3])
renderer = _make_renderer(tokenizer)
conversation, _ = renderer.render_messages(
[
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "lookup:0",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_call_id": "unknown",
"tool": "lookup",
"index": 1,
"content": "result",
},
],
ChatParams(),
)
assert "tool" not in conversation[1]
assert "index" not in conversation[1]
@pytest.mark.asyncio
async def test_render_messages_async_returns_token_prompt():
renderer = _make_renderer(StubTokenizer([4, 5]))
conversation, prompt = await renderer.render_messages_async(
[{"role": "user", "content": "hi"}], ChatParams()
)
assert prompt == {"prompt_token_ids": [4, 5]}
assert conversation[0]["role"] == "user"
-142
View File
@@ -1,142 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import time
from dataclasses import dataclass
from typing import Any
from unittest.mock import Mock
import pytest
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
from vllm.renderers import ChatParams
from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template
from vllm.tokenizers.mistral import MistralTokenizer
MODEL_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
runner_type = "generate"
model: str = MODEL_NAME
tokenizer: str = MODEL_NAME
trust_remote_code: bool = False
max_model_len: int = 100
tokenizer_revision = None
tokenizer_mode = "mistral"
hf_config = MockHFConfig()
encoder_config: dict[str, Any] | None = None
enable_prompt_embeds: bool = True
skip_tokenizer_init: bool = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
supports_multimodal_inputs: bool = False
renderer_num_workers: int = 1
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
@pytest.mark.asyncio
async def test_async_mistral_tokenizer_does_not_block_event_loop():
expected_tokens = [1, 2, 3]
# Mock the blocking version to sleep
def mocked_apply_chat_template(*_args, **_kwargs):
time.sleep(2)
return expected_tokens
mock_model_config = MockModelConfig(skip_tokenizer_init=True)
mock_tokenizer = Mock(spec=MistralTokenizer)
mock_tokenizer.apply_chat_template = mocked_apply_chat_template
mock_renderer = MistralRenderer(
MockVllmConfig(mock_model_config, parallel_config=MockParallelConfig()),
tokenizer=mock_tokenizer,
)
task = mock_renderer.render_messages_async([], ChatParams())
# Ensure the event loop is not blocked
blocked_count = 0
for _i in range(20): # Check over ~2 seconds
start = time.perf_counter()
await asyncio.sleep(0)
elapsed = time.perf_counter() - start
# an overly generous elapsed time for slow machines
if elapsed >= 0.5:
blocked_count += 1
await asyncio.sleep(0.1)
# Ensure task completes
_, prompt = await task
assert prompt["prompt_token_ids"] == expected_tokens, (
"Mocked blocking tokenizer was not called"
)
assert blocked_count == 0, "Event loop blocked during tokenization"
def test_apply_mistral_chat_template_thinking_chunk():
messages = [
{
"role": "system",
"content": [
{"type": "text", "text": "You are a helpful assistant."},
{
"type": "thinking",
"closed": True,
"thinking": "Only return the answer when you are confident.",
},
],
},
{"role": "user", "content": "What is 2+2?"},
{
"role": "assistant",
"content": [
{"type": "text", "text": "Let me think about it."},
{"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
{
"type": "text",
"text": "The answer is 4.",
},
],
},
{"role": "user", "content": "Thanks, what is 3+3?"},
]
mistral_tokenizer = MistralTokenizer.from_pretrained(
"mistralai/Magistral-Small-2509"
)
tokens_ids = safe_apply_chat_template(
mistral_tokenizer, messages, chat_template=None, tools=None
)
string_tokens = mistral_tokenizer.mistral.decode(
tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
)
expected_tokens = (
r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
r"[INST]What is 2+2?[/INST]"
r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
r"[INST]Thanks, what is 3+3?[/INST]"
)
assert string_tokens == expected_tokens
@@ -1,166 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Truncation must bound the prompt *after* multimodal placeholders expand.
``TokenizeParams.apply_post_tokenization`` truncates the unexpanded prompt, so
one placeholder token can expand into many and push the result back over
``truncate_prompt_tokens`` (vllm-project/vllm#57038).
"""
import numpy as np
import pytest
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal.inputs import PlaceholderRange
from vllm.renderers import renderer_from_config
from vllm.renderers.base import BaseRenderer
from vllm.renderers.hf import _ensure_prompt_embeds_placeholder_token
from vllm.renderers.params import TokenizeParams
def _renderer():
"""A BaseRenderer stub: only the tokenizer attribute is needed here."""
class _StubRenderer(BaseRenderer):
def __init__(self):
self.tokenizer = None
def render_messages(self, *args, **kwargs):
raise NotImplementedError
return _StubRenderer()
def _mm_input(num_tokens: int, offset: int, length: int, **extra):
return {
"type": "multimodal",
"prompt_token_ids": list(range(num_tokens)),
"mm_kwargs": {},
"mm_hashes": {},
"mm_placeholders": {"audio": [PlaceholderRange(offset=offset, length=length)]},
**extra,
}
def _params(limit: int | None, side: str = "right"):
return TokenizeParams(
max_total_tokens=4096,
truncate_prompt_tokens=limit,
truncation_side=side,
)
def test_expanded_prompt_is_bounded():
"""The reported case: audio fits, trailing text is cut to the limit."""
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
assert engine_input["prompt_token_ids"] == list(range(96))
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
offset=1, length=52
)
def test_prompt_within_limit_is_untouched():
engine_input = _mm_input(num_tokens=64, offset=1, length=52)
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
assert engine_input["prompt_token_ids"] == list(range(64))
def test_split_multimodal_span_is_rejected():
"""Cutting mid-span would desync features from token ids, so reject."""
engine_input = _mm_input(num_tokens=147, offset=80, length=50)
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
def test_dropped_multimodal_span_is_rejected():
"""A span entirely past the cut would be discarded silently."""
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
def test_left_truncation_shifts_placeholder_offsets():
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
_renderer()._truncate_expanded_prompt(engine_input, _params(96, side="left"))
assert engine_input["prompt_token_ids"] == list(range(51, 147))
# The span started at 100 and 51 leading tokens were dropped.
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
offset=49, length=20
)
def test_text_only_input_is_untouched():
"""Only multimodal prompts grow after tokenization, so nothing else runs."""
engine_input = {
"type": "token",
"prompt_token_ids": list(range(147)),
"prompt_token_offsets": [(i, i + 1) for i in range(147)],
}
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
assert engine_input["prompt_token_ids"] == list(range(147))
assert len(engine_input["prompt_token_offsets"]) == 147
def test_no_truncation_requested_is_noop():
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
_renderer()._truncate_expanded_prompt(engine_input, _params(None))
assert engine_input["prompt_token_ids"] == list(range(147))
def test_mixed_prompt_embeds_truncation_covers_embeds_spans():
"""Bound in _process_singleton, where prompt_embeds spans are already attached."""
model_config = ModelConfig(
"Qwen/Qwen3-ASR-0.6B-hf",
revision="7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c",
dtype="float32",
max_model_len=4096,
mm_processor_cache_gb=0,
enable_prompt_embeds=True,
)
renderer = renderer_from_config(VllmConfig(model_config=model_config))
tok = renderer.tokenizer
pe_id = _ensure_prompt_embeds_placeholder_token(tok)
hidden = model_config.get_hidden_size()
audio = np.zeros(4 * 16000, dtype=np.float32)
pe_len = 20
tensors = [torch.zeros(pe_len, hidden, dtype=torch.float32)]
def _build_prompt():
text = "<|audio_start|><|audio_pad|><|audio_end|>"
ids = tok.encode(text) + [pe_id] + tok.encode(" tail" * 40)
return {
"prompt_token_ids": ids,
"multi_modal_data": {"audio": (audio, 16000)},
"_prompt_embeds": (tensors, pe_id),
}
# A limit past every span (audio ends at 52, prompt_embeds at 74):
# truncation should succeed and bound the prompt.
out = renderer._process_singleton(
_build_prompt(),
tok_params=_params(79),
)
assert len(out["prompt_token_ids"]) == 79
# A limit inside the prompt_embeds span (54..74): must be rejected.
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
renderer._process_singleton(
_build_prompt(),
tok_params=_params(64),
)