[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,626 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Offline unit tests for `prompt_embeds` chat-completion content parts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import io
|
||||
from typing import Final
|
||||
from unittest import mock
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
|
||||
from vllm.renderers.chat_utils.chat_message import (
|
||||
_PROMPT_EMBEDS_MISSING_DATA_ERROR,
|
||||
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR,
|
||||
MM_PARSER_MAP,
|
||||
)
|
||||
from vllm.renderers.chat_utils.content_parser import (
|
||||
_ENABLE_PROMPT_EMBEDS_ERROR,
|
||||
MODALITY_PLACEHOLDERS_MAP,
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
)
|
||||
from vllm.renderers.hf import (
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,
|
||||
_build_mixed_prompt_embeds,
|
||||
_build_prompt_embeds_positions,
|
||||
_build_prompt_embeds_updates,
|
||||
_ensure_prompt_embeds_placeholder_token,
|
||||
_expand_prompt_embeds_placeholders,
|
||||
)
|
||||
|
||||
# Cover distinct tokenizer families:
|
||||
# GPT2TokenizerFast (BPE, OpenAI-style)
|
||||
# Qwen2TokenizerFast (SentencePiece BPE variant)
|
||||
# BertTokenizerFast (WordPiece)
|
||||
TOKENIZER_IDS: Final[list[str]] = [
|
||||
"openai-community/gpt2",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"bert-base-uncased",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(params=TOKENIZER_IDS, ids=TOKENIZER_IDS)
|
||||
def tokenizer(request):
|
||||
"""A fresh tokenizer instance per tokenizer family."""
|
||||
return AutoTokenizer.from_pretrained(request.param)
|
||||
|
||||
|
||||
# Minimal chat template that works with any tokenizer. Iterates
|
||||
# `message.content` as either a string or a list of dicts (openai format).
|
||||
_SIMPLE_CHAT_TEMPLATE: Final[str] = (
|
||||
"{% for m in messages %}"
|
||||
"{% if m['content'] is string %}{{m['content']}}"
|
||||
"{% else %}{% for p in m['content'] %}{{p['text']}}{% endfor %}"
|
||||
"{% endif %}\n{% endfor %}"
|
||||
)
|
||||
|
||||
|
||||
async def _maybe_await(fn, *args, **kwargs):
|
||||
"""Call *fn* and `await` the result if it's a coroutine."""
|
||||
result = fn(*args, **kwargs)
|
||||
if inspect.iscoroutine(result):
|
||||
result = await result
|
||||
return result
|
||||
|
||||
|
||||
# Parametrize over sync / async parse paths so every end-to-end test
|
||||
# exercises both.
|
||||
_PARSE_FUNCTIONS = [parse_chat_messages, parse_chat_messages_async]
|
||||
|
||||
|
||||
@pytest.fixture(params=_PARSE_FUNCTIONS, ids=["sync", "async"])
|
||||
def parse_fn(request):
|
||||
"""Either the sync or async `parse_chat_messages` callable."""
|
||||
return request.param
|
||||
|
||||
|
||||
def _encode_tensor(t: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(t, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
_MOCK_HIDDEN_SIZE: Final[int] = 8
|
||||
_MOCK_DTYPE: Final[torch.dtype] = torch.float32
|
||||
|
||||
|
||||
def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicMock:
|
||||
mc = mock.MagicMock()
|
||||
mc.enable_prompt_embeds = enable_prompt_embeds
|
||||
mc.multimodal_config = None
|
||||
mc.allowed_local_media_path = None
|
||||
mc.allowed_media_domains = None
|
||||
# Test text-only code path in `MultiModalItemTracker.resolve_items`.
|
||||
mc.is_multimodal_model = False
|
||||
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
|
||||
# and dtype, so the mock must return concrete values.
|
||||
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
|
||||
mc.hf_text_config = object()
|
||||
mc.dtype = _MOCK_DTYPE
|
||||
return mc
|
||||
|
||||
|
||||
def test_prompt_embeds_keys_registered():
|
||||
assert "prompt_embeds" in MODALITY_PLACEHOLDERS_MAP
|
||||
assert MODALITY_PLACEHOLDERS_MAP["prompt_embeds"] == "<##PROMPT_EMBEDS##>"
|
||||
assert "prompt_embeds" in MM_PARSER_MAP
|
||||
|
||||
|
||||
def test_ensure_placeholder_token_is_single_token_and_idempotent(tokenizer):
|
||||
"""Ensure the placeholder token is a single token and that multiple calls to
|
||||
"ensure" are idempotent, across all tokenizer families."""
|
||||
tid1 = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
tid2 = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
assert tid1 == tid2
|
||||
|
||||
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
|
||||
assert ids == [tid1]
|
||||
|
||||
# Repeating it in a string N times must produce exactly that many tokens.
|
||||
N = 5
|
||||
ids_rep = tokenizer.encode(
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN * N, add_special_tokens=False
|
||||
)
|
||||
assert ids_rep == [tid1] * N
|
||||
|
||||
|
||||
def test_parse_chat_messages_openai_format():
|
||||
NUM_TOKENS = 3
|
||||
t = torch.randn(NUM_TOKENS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
b64 = _encode_tensor(t)
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello "},
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": " world"},
|
||||
],
|
||||
}
|
||||
]
|
||||
conv, mm_data, _ = parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
# The middle content part is rewritten to a single placeholder-token
|
||||
# sentinel.
|
||||
texts = [p["text"] for p in conv[0]["content"]]
|
||||
assert texts == [
|
||||
"Hello ",
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
" world",
|
||||
]
|
||||
assert mm_data is not None and "prompt_embeds" in mm_data
|
||||
assert torch.equal(mm_data["prompt_embeds"][0], t)
|
||||
|
||||
|
||||
# Each layout entry is one content part:
|
||||
# ("text", "A") -> {"type": "text", "text": "A"}
|
||||
# ("embed", N) -> {"type": "prompt_embeds", "data": <base64 of (N, H) tensor>}
|
||||
@pytest.mark.parametrize(
|
||||
"layout",
|
||||
[
|
||||
# Case: Single embed only.
|
||||
[("embed", 2)],
|
||||
# Case: Embed at the start of the message.
|
||||
[("embed", 3), ("text", "B")],
|
||||
# Case: Embed at the end of the message.
|
||||
[("text", "A"), ("embed", 1)],
|
||||
# Case: Embed sandwiched between text spans.
|
||||
[("text", "A"), ("embed", 2), ("text", "B")],
|
||||
# Case: Multiple embeds with text in between.
|
||||
[("text", "A"), ("embed", 2), ("text", "B"), ("embed", 3)],
|
||||
# Case: Adjacent embeds with no separating text.
|
||||
[("embed", 1), ("embed", 2)],
|
||||
# Case: Multiple text spans before a trailing embed.
|
||||
[("text", "A"), ("text", "B"), ("embed", 1)],
|
||||
# Case: Long-ish run mixing both kinds.
|
||||
[
|
||||
("text", "head"),
|
||||
("embed", 4),
|
||||
("text", "mid"),
|
||||
("embed", 1),
|
||||
("embed", 2),
|
||||
("text", "tail"),
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"single-embed",
|
||||
"embed-then-text",
|
||||
"text-then-embed",
|
||||
"text-embed-text",
|
||||
"text-embed-text-embed",
|
||||
"adjacent-embeds",
|
||||
"text-text-embed",
|
||||
"long-mixed-run",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"interleave_mm_strings",
|
||||
# `None`: text-only path where `multimodal_config` is absent.
|
||||
# `False`: non-interleave multimodal path (the common default).
|
||||
# `True`: sentinel-substitution interleave path.
|
||||
# All three must preserve the request ordering of prompt_embeds
|
||||
# relative to surrounding text because prompt_embeds are spliced at the
|
||||
# token offset during rendering.
|
||||
[None, False, True],
|
||||
ids=["text-only", "interleave-off", "interleave-on"],
|
||||
)
|
||||
def test_parse_chat_messages_string_format_preserves_position(
|
||||
layout, interleave_mm_strings
|
||||
):
|
||||
mc = _make_mock_model_config()
|
||||
if interleave_mm_strings is not None:
|
||||
mm_cfg = mock.MagicMock()
|
||||
mm_cfg.interleave_mm_strings = interleave_mm_strings
|
||||
mc.multimodal_config = mm_cfg
|
||||
|
||||
content: list[dict] = []
|
||||
expected_parts: list[str] = []
|
||||
expected_embeds: list[torch.Tensor] = []
|
||||
for kind, value in layout:
|
||||
if kind == "text":
|
||||
content.append({"type": "text", "text": value})
|
||||
expected_parts.append(value)
|
||||
else: # prompt embeds
|
||||
num_tokens = value
|
||||
t = torch.randn(num_tokens, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
expected_embeds.append(t)
|
||||
content.append({"type": "prompt_embeds", "data": _encode_tensor(t)})
|
||||
# Parser emits ONE sentinel per part.
|
||||
expected_parts.append(PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
|
||||
|
||||
messages = [{"role": "user", "content": content}]
|
||||
conv, mm_data, _ = parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conv[0]["content"] == "\n".join(expected_parts)
|
||||
assert mm_data is not None and "prompt_embeds" in mm_data
|
||||
assert len(mm_data["prompt_embeds"]) == len(expected_embeds)
|
||||
for got, want in zip(mm_data["prompt_embeds"], expected_embeds, strict=True):
|
||||
assert torch.equal(got, want)
|
||||
|
||||
|
||||
def test_parse_chat_messages_requires_flag():
|
||||
t = torch.randn(2, 4)
|
||||
b64 = _encode_tensor(t)
|
||||
mc = _make_mock_model_config(enable_prompt_embeds=False)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": b64}],
|
||||
}
|
||||
]
|
||||
with pytest.raises(VLLMValidationError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_rejects_missing_data():
|
||||
# `data` is marked `Required` on `ChatCompletionContentPartPromptEmbedsParam`;
|
||||
# malformed requests without `data` must surface a clear validation error
|
||||
# rather than being silently dropped.
|
||||
mc = _make_mock_model_config()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}], # no `data`
|
||||
}
|
||||
]
|
||||
with pytest.raises(VLLMValidationError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
|
||||
# Reserved placeholder guard: when `enable_prompt_embeds=True` the tokenizer is
|
||||
# mutated to make `<prompt_embeds>` a single unsplittable token. Any user text
|
||||
# containing that literal sequence would tokenize to the same sentinel ID and
|
||||
# be mistaken for a splice point, so we reject it at parse time.
|
||||
_PLACEHOLDER_ERROR_PATTERN: Final[str] = re.sub(
|
||||
r"\\{[^}]*\\}", ".*", re.escape(_RESERVED_PLACEHOLDER_IN_TEXT_ERROR)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
# Case: Top-level string content (wrapped as a single text part).
|
||||
f"hello {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} world",
|
||||
# Case: List with a typed text part containing the placeholder.
|
||||
[{"type": "text", "text": f"leading {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"}],
|
||||
# Case: List with a plain-string part (no wrapping dict).
|
||||
[f"raw string {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"],
|
||||
],
|
||||
ids=["top-level-string", "typed-text-part", "plain-string-part"],
|
||||
)
|
||||
def test_parse_chat_messages_rejects_placeholder_in_user_text(content):
|
||||
mc = _make_mock_model_config() # enable_prompt_embeds=True by default
|
||||
messages = [{"role": "user", "content": content}]
|
||||
with pytest.raises(VLLMValidationError, match=_PLACEHOLDER_ERROR_PATTERN):
|
||||
parse_chat_messages(messages, mc, content_format="openai")
|
||||
|
||||
|
||||
def test_parse_chat_messages_allows_placeholder_in_text_when_feature_disabled():
|
||||
# When `enable_prompt_embeds=False` the tokenizer is never mutated, so the
|
||||
# literal `<prompt_embeds>` is just ordinary text and must pass through.
|
||||
mc = _make_mock_model_config(enable_prompt_embeds=False)
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"benign mention of {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} here",
|
||||
}
|
||||
]
|
||||
conv, mm_data, _ = parse_chat_messages(messages, mc, content_format="openai")
|
||||
assert mm_data is None or "prompt_embeds" not in mm_data
|
||||
# Text reaches the rendered conversation unchanged.
|
||||
texts = [p["text"] for p in conv[0]["content"]]
|
||||
assert PROMPT_EMBEDS_PLACEHOLDER_TOKEN in "".join(texts)
|
||||
|
||||
|
||||
# Token-stream spec: ints are regular token IDs, tuples `(N,)` expand to
|
||||
# a placeholder span of length N (creates corresponding `(N, H)` tensor).
|
||||
# `expected` lists the `(start_idx, length)` pairs that
|
||||
# `_build_prompt_embeds_positions` should return.
|
||||
@pytest.mark.parametrize(
|
||||
"stream, expected",
|
||||
[
|
||||
# Case: Single run in the middle.
|
||||
([10, 20, (3,), 30], [(2, 3)]),
|
||||
# Case: Single run at the start.
|
||||
([(2,), 10, 20], [(0, 2)]),
|
||||
# Case: Single run at the end.
|
||||
([10, 20, (4,)], [(2, 4)]),
|
||||
# Case: Two runs with tokens between.
|
||||
([1, (2,), 2, 3, (3,), 4], [(1, 2), (5, 3)]),
|
||||
# Case: Adjacent runs (no separating tokens).
|
||||
([(1,), (2,)], [(0, 1), (1, 2)]),
|
||||
# Case: Three runs.
|
||||
([5, (2,), 6, (1,), 7, (3,), 8], [(1, 2), (4, 1), (6, 3)]),
|
||||
],
|
||||
ids=[
|
||||
"single-middle",
|
||||
"single-start",
|
||||
"single-end",
|
||||
"two-runs-separated",
|
||||
"two-runs-adjacent",
|
||||
"three-runs",
|
||||
],
|
||||
)
|
||||
def test_build_positions(tokenizer, stream, expected):
|
||||
H = 4
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
tensors: list[torch.Tensor] = []
|
||||
token_ids: list[int] = []
|
||||
for item in stream:
|
||||
if isinstance(item, tuple):
|
||||
length = item[0]
|
||||
tensors.append(torch.randn(length, H))
|
||||
token_ids.extend([tid] * length)
|
||||
else:
|
||||
token_ids.append(item)
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
positions = _build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
|
||||
assert positions == expected
|
||||
|
||||
|
||||
def test_build_positions_length_mismatch(tokenizer):
|
||||
N1, H1 = 2, 4
|
||||
N2, H2 = 3, 4
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
# 2 tensors expected but only a single placeholder run in the token
|
||||
# stream (simulating dropping the second one).
|
||||
tensors = [torch.randn(N1, H1), torch.randn(N2, H2)]
|
||||
token_ids = [1, tid, tid, 2, 3]
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
# The error constant is a `str.format` template, escape it and turn
|
||||
# the `{field}` placeholders into `.*` so it matches any substitution.
|
||||
pattern = re.sub(
|
||||
r"\\{[^}]*\\}", ".*", re.escape(_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR)
|
||||
)
|
||||
with pytest.raises(ValueError, match=pattern):
|
||||
_build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
|
||||
|
||||
|
||||
# ints = regular token IDs (any value)
|
||||
# (N,) = embed span of length N
|
||||
@pytest.mark.parametrize(
|
||||
"stream",
|
||||
[
|
||||
[10, 20, (3,), 30],
|
||||
[(2,), 10, 20],
|
||||
[10, 20, (4,)],
|
||||
[1, (2,), 2, 3, (3,), 4],
|
||||
[(1,), (2,)],
|
||||
[5, (2,), 6, (1,), 7, (3,), 8],
|
||||
],
|
||||
ids=[
|
||||
"single-middle",
|
||||
"single-start",
|
||||
"single-end",
|
||||
"two-spans-separated",
|
||||
"two-spans-adjacent",
|
||||
"three-spans",
|
||||
],
|
||||
)
|
||||
def test_build_mixed_prompt_embeds(stream):
|
||||
H = 8
|
||||
_PLACEHOLDER = 0 # sentinel for embed positions in token_ids
|
||||
|
||||
tensors: list[torch.Tensor] = []
|
||||
token_ids: list[int] = []
|
||||
positions: list[tuple[int, int]] = []
|
||||
cursor = 0
|
||||
for item in stream:
|
||||
if isinstance(item, tuple):
|
||||
length = item[0]
|
||||
tensors.append(torch.randn(length, H))
|
||||
positions.append((cursor, length))
|
||||
token_ids.extend([_PLACEHOLDER] * length)
|
||||
cursor += length
|
||||
else:
|
||||
token_ids.append(item)
|
||||
cursor += 1
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
|
||||
|
||||
assert embeds.shape == (len(token_ids), H)
|
||||
assert len(mask) == len(token_ids)
|
||||
|
||||
# Mask: False exactly at embed positions, True everywhere else.
|
||||
expected_mask = torch.ones(len(token_ids), dtype=torch.bool)
|
||||
for start, length in positions:
|
||||
expected_mask[start : start + length] = False
|
||||
assert mask == expected_mask.tolist()
|
||||
|
||||
# Embed rows match input tensors at the right positions.
|
||||
for tensor, (start, length) in zip(tensors, positions):
|
||||
assert torch.equal(embeds[start : start + length], tensor)
|
||||
|
||||
# Non-embed positions remain zero-filled.
|
||||
assert torch.all(embeds[expected_mask] == 0)
|
||||
|
||||
|
||||
# End-to-end tests: each runs both sync and async parse paths via the
|
||||
# `parse_fn` fixture.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("role", ["user", "system"])
|
||||
async def test_end_to_end_expand_and_build(tokenizer, parse_fn, role):
|
||||
"""Full renderer pipeline: parse -> chat template -> expand -> locate
|
||||
-> build mixed prompt, across tokenizers, roles, and sync/async."""
|
||||
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
|
||||
LEN_A, LEN_B = 3, 2
|
||||
t_a = torch.randn(LEN_A, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
t_b = torch.randn(LEN_B, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
NUM_TENSORS = 2
|
||||
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": role,
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello "},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_a)},
|
||||
{"type": "text", "text": " world "},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_b)},
|
||||
{"type": "text", "text": "!"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conv, mm_data, _ = await _maybe_await(
|
||||
parse_fn, messages, mc, content_format="openai"
|
||||
)
|
||||
tensors = list(mm_data["prompt_embeds"])
|
||||
assert len(tensors) == NUM_TENSORS
|
||||
|
||||
# Tokenize: each prompt_embeds part becomes 1 placeholder token.
|
||||
# `return_dict=False` to get a flat `list[int]` on transformers v5
|
||||
# (where the default flipped to True and yields a `BatchEncoding` dict).
|
||||
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
|
||||
assert sum(t == tid for t in token_ids) == NUM_TENSORS
|
||||
|
||||
# Expand, locate, and build.
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
assert len(expanded) == len(token_ids) + LEN_A + LEN_B - NUM_TENSORS
|
||||
|
||||
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
|
||||
assert positions[0][1] == LEN_A
|
||||
assert positions[1][1] == LEN_B
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
|
||||
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
|
||||
assert mask.count(False) == LEN_A + LEN_B
|
||||
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_A], t_a)
|
||||
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_B], t_b)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_to_end_multi_message_conversation(tokenizer, parse_fn):
|
||||
"""Full pipeline with prompt_embeds spread across system + user messages,
|
||||
verifying ordering and positioning in the final token stream."""
|
||||
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
|
||||
LEN_SYS, LEN_USR = 4, 3
|
||||
t_sys = torch.randn(LEN_SYS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
t_usr = torch.randn(LEN_USR, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
NUM_TENSORS = 2 # t_sys and t_usr.
|
||||
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_sys)},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_usr)},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
conv, mm_data, _ = await _maybe_await(
|
||||
parse_fn, messages, mc, content_format="openai"
|
||||
)
|
||||
tensors = list(mm_data["prompt_embeds"])
|
||||
assert len(tensors) == NUM_TENSORS
|
||||
|
||||
# Tokenize, expand, locate, and build.
|
||||
# `return_dict=False` to get a flat `list[int]` on transformers v5
|
||||
# (where the default flipped to True and yields a `BatchEncoding` dict).
|
||||
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
|
||||
|
||||
assert positions[0][1] == LEN_SYS
|
||||
assert positions[1][1] == LEN_USR
|
||||
# System embed must appear before user embed in the token stream.
|
||||
assert positions[0][0] < positions[1][0]
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
|
||||
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
|
||||
assert mask.count(False) == LEN_SYS + LEN_USR
|
||||
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_SYS], t_sys)
|
||||
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_USR], t_usr)
|
||||
|
||||
|
||||
def test_truncation_keeps_the_mixed_mask_aligned_with_the_prompt():
|
||||
"""`prompt_is_token_ids` must be truncated with `prompt_token_ids`.
|
||||
|
||||
The mask is positional: the engine pairs it element-wise with the prompt
|
||||
tokens to decide which positions read a real token id and which read a row
|
||||
of `prompt_embeds`. Truncating the prompt without truncating the mask
|
||||
leaves the two describing different positions, so embed placeholders stop
|
||||
being recognised as such.
|
||||
"""
|
||||
hidden_size = 8
|
||||
placeholder_token_id = 0
|
||||
num_head, embed_len, num_tail = 10, 5, 5
|
||||
|
||||
token_ids = (
|
||||
list(range(1, num_head + 1))
|
||||
+ [placeholder_token_id] * embed_len
|
||||
+ list(range(1, num_tail + 1))
|
||||
)
|
||||
tensors = [torch.randn(embed_len, hidden_size)]
|
||||
positions = [(num_head, embed_len)]
|
||||
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
|
||||
|
||||
prompt = {
|
||||
"prompt_token_ids": token_ids,
|
||||
"prompt_embeds": embeds,
|
||||
"prompt_is_token_ids": mask,
|
||||
}
|
||||
keep = 8
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=keep,
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
result = tok_params.apply_post_tokenization(None, prompt)
|
||||
|
||||
# Keeping the last 8 of 20 positions starts inside the embed span, so the
|
||||
# first 3 surviving positions are embed rows and the rest are real tokens.
|
||||
# Comparing contents rather than lengths is what pins the alignment: a
|
||||
# mask reduced from the wrong side has the right length and still
|
||||
# describes the wrong positions.
|
||||
assert result["prompt_is_token_ids"] == [False] * 3 + [True] * num_tail
|
||||
assert result["prompt_token_ids"] == token_ids[-keep:]
|
||||
assert torch.equal(result["prompt_embeds"], embeds[-keep:])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,796 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import pybase64
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import SingletonPrompt
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
runner_type = "generate"
|
||||
model: str = MODEL_NAME
|
||||
tokenizer: str = MODEL_NAME
|
||||
trust_remote_code: bool = False
|
||||
tokenizer_revision = None
|
||||
tokenizer_mode = "auto"
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = hf_config
|
||||
encoder_config: dict[str, Any] | None = None
|
||||
enable_prompt_embeds: bool = True
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
hidden_size: int = 768
|
||||
dtype: torch.dtype = torch.float32
|
||||
|
||||
def get_hidden_size(self) -> int:
|
||||
return self.hidden_size
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyTokenizer:
|
||||
truncation_side: str = "left"
|
||||
max_chars_per_token: int = 1
|
||||
# Deliberately outside the range of ids `encode` returns, so a test can
|
||||
# tell a pad token apart from a real one.
|
||||
pad_token_id: int = 99999
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._captured_encode_kwargs: dict = {}
|
||||
self._captured_text_len: int = 0
|
||||
|
||||
def decode(self, tokens: list[int]):
|
||||
return str(tokens)
|
||||
|
||||
def encode(self, text: str, **kwargs):
|
||||
self._captured_encode_kwargs = kwargs
|
||||
self._captured_text_len = len(text)
|
||||
|
||||
in_length = len(text)
|
||||
truncation = kwargs.get("truncation")
|
||||
max_length = kwargs.get("max_length")
|
||||
if truncation and max_length is not None:
|
||||
return list(range(min(in_length, max_length)))
|
||||
|
||||
return list(range(in_length))
|
||||
|
||||
def __call__(self, text: str, **kwargs):
|
||||
# BaseRenderer._tokenize_prompt calls the tokenizer via __call__ (to
|
||||
# unify the output type), so mirror a real tokenizer's BatchEncoding.
|
||||
return {"input_ids": self.encode(text, **kwargs)}
|
||||
|
||||
|
||||
def _build_renderer(
|
||||
model_config: MockModelConfig,
|
||||
*,
|
||||
truncation_side: str = "left",
|
||||
max_chars_per_token: int = 1,
|
||||
):
|
||||
renderer = HfRenderer(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
tokenizer=(
|
||||
None
|
||||
if model_config.skip_tokenizer_init
|
||||
else DummyTokenizer(
|
||||
truncation_side=truncation_side,
|
||||
max_chars_per_token=max_chars_per_token,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
return renderer
|
||||
|
||||
|
||||
def _preprocess_prompt(
|
||||
model_config: ModelConfig,
|
||||
prompt_or_prompts: SingletonPrompt | bytes | Sequence[SingletonPrompt | bytes],
|
||||
):
|
||||
return [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompt_to_seq(prompt_or_prompts)
|
||||
]
|
||||
|
||||
|
||||
class TestValidatePrompt:
|
||||
def test_empty_input(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
with pytest.raises(ValueError, match="at least one prompt"):
|
||||
renderer.render_prompts(_preprocess_prompt(renderer.model_config, []))
|
||||
|
||||
def test_invalid_type(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
with pytest.raises(TypeError, match="should be a list of integers"):
|
||||
renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, [[1, 2], ["foo", "bar"]])
|
||||
)
|
||||
|
||||
|
||||
class TestRenderPrompt:
|
||||
def test_tokens_input(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
tokens = [101, 7592, 2088]
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, tokens)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["prompt_token_ids"] == tokens
|
||||
|
||||
def test_token_list_input(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
token_lists = [[101, 7592, 2088], [102, 1234, 5678, 9012], [103, 4567]]
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, token_lists)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0]["prompt_token_ids"] == [101, 7592, 2088]
|
||||
assert results[1]["prompt_token_ids"] == [102, 1234, 5678, 9012]
|
||||
assert results[2]["prompt_token_ids"] == [103, 4567]
|
||||
|
||||
def test_text_input(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
text_input = "x" * 10
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, text_input)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 10
|
||||
|
||||
def test_text_list_input(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
text_list_input = ["x" * 10, "x" * 12, "x" * 14]
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, text_list_input)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 3
|
||||
for text_input, result in zip(text_list_input, results):
|
||||
assert len(result["prompt_token_ids"]) == len(text_input)
|
||||
|
||||
def test_zero_truncation(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 200)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=0),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 0
|
||||
|
||||
def test_pos_truncation(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 200)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=50),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 50
|
||||
|
||||
def test_neg_truncation(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 200)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=-1),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
# max_output_tokens defaults to 0, so there is no output budget to
|
||||
# reserve and -1 truncates to the full max_total_tokens.
|
||||
assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens
|
||||
|
||||
def test_truncation_left(self):
|
||||
renderer = _build_renderer(MockModelConfig(), truncation_side="left")
|
||||
|
||||
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, long_tokens)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
# Should keep the last 5 tokens: [105, 106, 107, 108, 109]
|
||||
assert results[0]["prompt_token_ids"] == [105, 106, 107, 108, 109]
|
||||
|
||||
def test_truncation_right(self):
|
||||
renderer = _build_renderer(MockModelConfig(), truncation_side="right")
|
||||
|
||||
long_tokens = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109] # 10 tokens
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, long_tokens)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=5),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
# Should keep the first 5 tokens: [100, 101, 102, 103, 104]
|
||||
assert results[0]["prompt_token_ids"] == [100, 101, 102, 103, 104]
|
||||
|
||||
def test_text_max_length_exceeded_obvious(self):
|
||||
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
|
||||
|
||||
# Exceeds max_total_tokens and max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
|
||||
long_tokens = "x" * 150
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, long_tokens)
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="maximum context length is",
|
||||
):
|
||||
renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
# Should not even attempt tokenization
|
||||
assert renderer.tokenizer._captured_encode_kwargs == {}
|
||||
|
||||
def test_text_max_length_exceeded_nonobvious(self):
|
||||
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=2)
|
||||
|
||||
# Exceeds max_total_tokens but not max_total_tokens * VLLM_MAX_CHARS_PER_TOKEN
|
||||
long_tokens = "x" * 150
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, long_tokens)
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="maximum context length is",
|
||||
):
|
||||
renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
# Should only tokenize the first max_total_tokens + 1 tokens
|
||||
assert renderer.tokenizer._captured_encode_kwargs["truncation"] is True
|
||||
assert renderer.tokenizer._captured_encode_kwargs["max_length"] == 101
|
||||
|
||||
def test_token_max_length_exceeded(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
long_tokens = list(range(150)) # Exceeds max_total_tokens=100
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, long_tokens)
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="maximum context length is",
|
||||
):
|
||||
renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, truncate_prompt_tokens=None),
|
||||
)
|
||||
|
||||
def test_no_tokenizer_for_text(self):
|
||||
renderer = _build_renderer(MockModelConfig(skip_tokenizer_init=True))
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "Hello world")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):
|
||||
renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
def test_tokens_input_with_needs_detokenization(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
tokens = [1, 2, 3, 4]
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, tokens)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
needs_detokenization=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["prompt_token_ids"] == tokens
|
||||
assert results[0]["prompt"] == "[1, 2, 3, 4]"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("truncation_side", "expected"),
|
||||
[("left", [106, 107, 108, 109]), ("right", [100, 101, 102, 103])],
|
||||
)
|
||||
def test_detokenization_matches_truncated_tokens(self, truncation_side, expected):
|
||||
"""Echoed prompt text must match the truncated tokens sent to the engine."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
params = TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side=truncation_side,
|
||||
needs_detokenization=True,
|
||||
)
|
||||
|
||||
def render():
|
||||
return renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, list(range(100, 110)))
|
||||
)
|
||||
|
||||
results = renderer.tokenize_prompts(render(), params)
|
||||
async_results = asyncio.run(renderer.tokenize_prompts_async(render(), params))
|
||||
|
||||
for result in (*results, *async_results):
|
||||
assert result["prompt_token_ids"] == expected
|
||||
assert result["prompt"] == str(expected)
|
||||
|
||||
def test_explicit_side_tokenizer_unbounded(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 500)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 4
|
||||
|
||||
kwargs = renderer.tokenizer._captured_encode_kwargs
|
||||
assert kwargs["truncation"] is False
|
||||
|
||||
def test_explicit_side_left_text(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=5,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 5
|
||||
assert results[0]["prompt_token_ids"] == list(range(45, 50))
|
||||
|
||||
def test_explicit_side_right_text(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=5,
|
||||
truncation_side="right",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 5
|
||||
assert results[0]["prompt_token_ids"] == list(range(5))
|
||||
|
||||
def test_padding_with_left_truncation_keeps_the_prompt(self):
|
||||
"""Padding must not be truncated away.
|
||||
|
||||
`padding` and `truncate_prompt_tokens`/`truncation_side` are all
|
||||
settable on one pooling request. Padding to the full input length
|
||||
before truncating from the left leaves a prompt made entirely of pad
|
||||
tokens, and the request still succeeds -- so the model embeds nothing
|
||||
but padding.
|
||||
"""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
pad_id = renderer.tokenizer.pad_token_id
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
pad_prompt_tokens=-1,
|
||||
truncate_prompt_tokens=5,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
token_ids = results[0]["prompt_token_ids"]
|
||||
|
||||
# The sentinel: on a padding-first pipeline every surviving id is a
|
||||
# pad token, so the prompt reaches the model with no content at all.
|
||||
assert set(token_ids) != {pad_id}
|
||||
|
||||
# Transformers semantics: truncate to 5, then pad out to the full
|
||||
# input length.
|
||||
assert token_ids[:5] == list(range(45, 50))
|
||||
assert token_ids[5:] == [pad_id] * 95
|
||||
|
||||
def test_padding_without_truncation_is_unchanged(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
pad_id = renderer.tokenizer.pad_token_id
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, pad_prompt_tokens=-1),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["prompt_token_ids"] == list(range(50)) + [pad_id] * 50
|
||||
|
||||
def test_explicit_side_text_pretokenization_guard(self):
|
||||
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 500)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 4
|
||||
|
||||
assert renderer.tokenizer._captured_text_len <= 100
|
||||
|
||||
|
||||
class TestMaxTokensNotReservation:
|
||||
"""Regression tests for GitHub issue #42474.
|
||||
|
||||
``max_tokens`` must be treated as an upper bound on generation length, not
|
||||
as a hard reservation that is subtracted from the available input space.
|
||||
"""
|
||||
|
||||
def test_large_max_tokens_does_not_eat_prompt_budget(self):
|
||||
"""Prompt fits in context even when max_tokens is large."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# 50-token prompt with a 100-token context window and max_tokens=60.
|
||||
# Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error.
|
||||
# After the fix: max_input_tokens = 100 → 50 ≤ 100 → OK.
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, max_output_tokens=60),
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 50
|
||||
|
||||
def test_max_tokens_zero_still_rejects_oversized_prompt(self):
|
||||
"""Genuine prompt overflow is still caught regardless of max_tokens."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# 150-token prompt with 100-token context → should still fail.
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 150)
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="maximum context length"):
|
||||
renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, max_output_tokens=0),
|
||||
)
|
||||
|
||||
def test_negative_truncation_uses_max_total(self):
|
||||
"""truncate_prompt_tokens=-1 reserves one output token."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 200)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
max_output_tokens=40,
|
||||
truncate_prompt_tokens=-1,
|
||||
),
|
||||
)
|
||||
# -1 maps to max_total_tokens (100) rather than the old
|
||||
# max_total - max_output (60), minus the one token held back so
|
||||
# truncation cannot leave a zero-token output budget.
|
||||
assert len(results[0]["prompt_token_ids"]) == 99
|
||||
|
||||
def test_negative_pad_uses_max_total(self):
|
||||
"""pad_prompt_tokens=-1 pads to max_total_tokens, not max_input."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
pad_id = renderer.tokenizer.pad_token_id
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 30)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
max_output_tokens=40,
|
||||
pad_prompt_tokens=-1,
|
||||
),
|
||||
)
|
||||
# -1 should map to max_total_tokens (100), not max_total - max_output (60).
|
||||
assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70
|
||||
|
||||
def test_max_tokens_exceeds_context_is_still_clamped_by_api(self):
|
||||
"""Renderer allows the prompt; API layer clamps max_tokens later."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# Simulates the reported scenario: max_tokens > context - prompt_length.
|
||||
# Renderer should accept the 40-token prompt; get_max_tokens() clamps
|
||||
# sampling max_tokens to 60 at the entrypoint layer.
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 40)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, max_output_tokens=80),
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 40
|
||||
|
||||
def test_with_kwargs_preserves_output_budget(self):
|
||||
"""with_kwargs without an explicit max_length keeps the output budget.
|
||||
|
||||
Under the old semantics `with_kwargs()` recomputed
|
||||
`max_output_tokens = max_total - max_input`, which (after this fix)
|
||||
would have silently reset a real output budget to zero.
|
||||
"""
|
||||
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
|
||||
chained = tok_params.with_kwargs(truncation_side="left")
|
||||
|
||||
assert chained.max_output_tokens == 60
|
||||
assert chained.max_input_tokens == 100
|
||||
|
||||
def test_with_kwargs_explicit_max_length_sets_budget(self):
|
||||
"""An explicit max_length still re-derives the output budget."""
|
||||
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
|
||||
chained = tok_params.with_kwargs(max_length=80)
|
||||
|
||||
assert chained.max_output_tokens == 20
|
||||
assert chained.max_input_tokens == 100
|
||||
|
||||
|
||||
class TestRenderEmbedPrompt:
|
||||
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
|
||||
"""Helper to create base64-encoded tensor bytes."""
|
||||
buffer = io.BytesIO()
|
||||
torch.save(tensor, buffer)
|
||||
buffer.seek(0)
|
||||
return pybase64.b64encode(buffer.read())
|
||||
|
||||
def test_single_prompt_embed(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# Create a test tensor
|
||||
tensor_input = torch.randn(10, 768, dtype=torch.float32)
|
||||
embed_bytes = self._create_test_embed_bytes(tensor_input)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, embed_bytes)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert torch.equal(results[0]["prompt_embeds"], tensor_input)
|
||||
|
||||
def test_multiple_prompt_embeds(self):
|
||||
hidden_size = 512
|
||||
renderer = _build_renderer(MockModelConfig(hidden_size=hidden_size))
|
||||
|
||||
# Create multiple test tensors
|
||||
tensor_inputs = [
|
||||
torch.randn(8, hidden_size, dtype=torch.float32),
|
||||
torch.randn(12, hidden_size, dtype=torch.float32),
|
||||
]
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
renderer.model_config,
|
||||
[self._create_test_embed_bytes(t) for t in tensor_inputs],
|
||||
)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 2
|
||||
for i, result in enumerate(results):
|
||||
assert torch.allclose(result["prompt_embeds"], tensor_inputs[i])
|
||||
|
||||
def test_prompt_embed_truncation(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# Create tensor with more tokens than truncation limit
|
||||
tensor_input = torch.randn(20, 768, dtype=torch.float32)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
renderer.model_config, self._create_test_embed_bytes(tensor_input)
|
||||
)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=10,
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
# Should keep last 10 tokens
|
||||
expected = tensor_input[-10:]
|
||||
assert torch.equal(results[0]["prompt_embeds"], expected)
|
||||
|
||||
def test_prompt_embed_different_dtypes(self):
|
||||
hidden_size = 256
|
||||
# Test different supported dtypes
|
||||
dtypes = [torch.float32, torch.float16, torch.bfloat16]
|
||||
|
||||
for dtype in dtypes:
|
||||
renderer = _build_renderer(
|
||||
MockModelConfig(hidden_size=hidden_size, dtype=dtype)
|
||||
)
|
||||
tensor_input = torch.randn(5, hidden_size, dtype=dtype)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
renderer.model_config, self._create_test_embed_bytes(tensor_input)
|
||||
)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["prompt_embeds"].dtype == dtype
|
||||
|
||||
def test_prompt_embed_squeeze_batch_dim(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# Test tensor with batch dimension gets squeezed
|
||||
tensor_input = torch.randn(1, 10, 768, dtype=torch.float32)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
renderer.model_config, self._create_test_embed_bytes(tensor_input)
|
||||
)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
# Should be squeezed to 2D
|
||||
assert results[0]["prompt_embeds"].shape == (10, 768)
|
||||
|
||||
def test_both_prompts_and_embeds(self):
|
||||
hidden_size = 256
|
||||
renderer = _build_renderer(MockModelConfig(hidden_size=hidden_size))
|
||||
|
||||
text_input = "Hello world"
|
||||
tensor_input = torch.randn(5, hidden_size, dtype=torch.float32)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(
|
||||
renderer.model_config,
|
||||
[text_input, self._create_test_embed_bytes(tensor_input)],
|
||||
)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100),
|
||||
)
|
||||
|
||||
assert len(results) == 2
|
||||
# First should be tokens prompt
|
||||
assert "prompt_token_ids" in results[0]
|
||||
assert len(results[0]["prompt_token_ids"]) == len(text_input)
|
||||
# Second should be embed prompt
|
||||
assert torch.equal(results[1]["prompt_embeds"], tensor_input)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,484 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Golden tests for the native Inkling renderer encoding.
|
||||
|
||||
The message fixtures mirror the Rust renderer's fixture tests
|
||||
(``rust/src/chat/src/renderer/inkling/tests.rs``) so the two frontends stay
|
||||
in token-level parity. Image blocks emit the bare ``<|content_image|>`` marker;
|
||||
``InklingMultiModalProcessor`` inserts the per-patch placeholder run.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.renderers.inkling import (
|
||||
InklingRenderer,
|
||||
_HfBackedTmlTokenizer,
|
||||
_resolve_reasoning_effort,
|
||||
)
|
||||
from vllm.renderers.inkling_encoding import (
|
||||
SPECIAL_TOKEN_SPELLINGS,
|
||||
render_inkling_messages,
|
||||
)
|
||||
from vllm.renderers.params import ChatParams
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def should_do_global_cleanup_after_test() -> bool:
|
||||
# These tests touch no distributed or device state; the global
|
||||
# cleanup fixture is unnecessary (and trips a torch MPS allocator
|
||||
# assert on macOS dev machines).
|
||||
return False
|
||||
|
||||
|
||||
# One id per special token, mirroring the real Inkling vocab layout; plain
|
||||
# text encodes one token per character so decoded output is exact.
|
||||
_SPECIAL_VOCAB = {
|
||||
"<|message_user|>": 200000,
|
||||
"<|message_model|>": 200001,
|
||||
"<|message_system|>": 200002,
|
||||
"<|message_tool|>": 200003,
|
||||
"<|content_text|>": 200004,
|
||||
"<|content_image|>": 200005,
|
||||
"<|content_model_end_sampling|>": 200006,
|
||||
"<|content_thinking|>": 200008,
|
||||
"<|end_message|>": 200010,
|
||||
"<|content_audio_input|>": 200020,
|
||||
# The HF vocab spells CONTENT_XML as an unused slot.
|
||||
"<|unused_200024|>": 200024,
|
||||
"<|audio_end|>": 200043,
|
||||
"<|content_invoke_tool_json|>": 200049,
|
||||
}
|
||||
|
||||
_ID_TO_SPECIAL = {v: k for k, v in _SPECIAL_VOCAB.items()}
|
||||
|
||||
|
||||
class FakeHfTokenizer:
|
||||
def get_vocab(self):
|
||||
return dict(_SPECIAL_VOCAB)
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
assert not add_special_tokens
|
||||
return [ord(ch) for ch in text]
|
||||
|
||||
|
||||
def decode(token_ids):
|
||||
return "".join(
|
||||
_ID_TO_SPECIAL.get(tid, chr(tid) if tid < 200000 else f"<{tid}>")
|
||||
for tid in token_ids
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def inkling_tokenizer():
|
||||
return _HfBackedTmlTokenizer(FakeHfTokenizer())
|
||||
|
||||
|
||||
def render_text(inkling_tokenizer, messages, **kwargs):
|
||||
return decode(render_inkling_messages(messages, inkling_tokenizer, **kwargs))
|
||||
|
||||
|
||||
class TestRustFixtureParity:
|
||||
def test_tool_round_trip(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "think",
|
||||
"content": "answer",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"SF"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
assert render_text(
|
||||
inkling_tokenizer, messages, add_generation_prompt=False
|
||||
) == (
|
||||
"<|message_model|><|content_thinking|>think<|end_message|>"
|
||||
"<|message_model|><|content_text|>answer<|end_message|>"
|
||||
"<|message_model|>get_weather<|content_invoke_tool_json|>"
|
||||
'{"name":"get_weather","args":{"city":"SF"}}<|end_message|>'
|
||||
"<|content_model_end_sampling|>"
|
||||
"<|message_tool|>get_weather<|content_text|>sunny<|end_message|>"
|
||||
)
|
||||
|
||||
def test_tool_declare(self, inkling_tokenizer):
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"required": ["city"],
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
messages = [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "rules",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "local_tool",
|
||||
"parameters": {"z": 1, "a": {"b": 2}},
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "hi"},
|
||||
]
|
||||
assert render_text(inkling_tokenizer, messages, tools=tools) == (
|
||||
"<|message_system|>tool_declare<|unused_200024|>"
|
||||
'[{"description":"Get weather information","name":"get_weather",'
|
||||
'"parameters":{"properties":{"city":{"type":"string"}},'
|
||||
'"required":["city"],"type":"object"},"type":"function"},'
|
||||
'{"description":"","name":"local_tool",'
|
||||
'"parameters":{"a":{"b":2},"z":1},"type":"function"}]'
|
||||
"<|end_message|>"
|
||||
"<|message_system|><|content_text|>rules<|end_message|>"
|
||||
"<|message_user|><|content_text|>hi<|end_message|>"
|
||||
"<|message_model|>"
|
||||
)
|
||||
|
||||
def test_text_image(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": "data:image/png;base64,"},
|
||||
],
|
||||
}
|
||||
]
|
||||
# Multimodal preprocessing expands this bare marker after rendering.
|
||||
assert render_text(inkling_tokenizer, messages) == (
|
||||
"<|message_user|><|content_text|>look<|end_message|>"
|
||||
"<|message_user|><|content_image|><|end_message|>"
|
||||
"<|message_model|>"
|
||||
)
|
||||
|
||||
|
||||
class TestRenderingSemantics:
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("none", 0.0),
|
||||
("minimal", 0.1),
|
||||
("low", 0.2),
|
||||
("medium", 0.7),
|
||||
("high", 0.9),
|
||||
("xhigh", 0.99),
|
||||
("max", 0.99),
|
||||
(None, 0.9),
|
||||
(0.8, 0.8),
|
||||
(True, None),
|
||||
("invalid", None),
|
||||
],
|
||||
)
|
||||
def test_resolve_reasoning_effort(self, value, expected):
|
||||
assert _resolve_reasoning_effort(value) == expected
|
||||
|
||||
def test_generation_prompt_default(self, inkling_tokenizer):
|
||||
text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}])
|
||||
assert text.endswith("<|end_message|><|message_model|>")
|
||||
|
||||
def test_developer_folds_into_system(self, inkling_tokenizer):
|
||||
assert (
|
||||
render_text(
|
||||
inkling_tokenizer,
|
||||
[{"role": "developer", "content": "be nice"}],
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
== "<|message_system|><|content_text|>be nice<|end_message|>"
|
||||
)
|
||||
|
||||
def test_empty_string_content_skipped(self, inkling_tokenizer):
|
||||
assert (
|
||||
render_text(
|
||||
inkling_tokenizer,
|
||||
[{"role": "user", "content": ""}],
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
== ""
|
||||
)
|
||||
|
||||
def test_empty_reasoning_skipped(self, inkling_tokenizer):
|
||||
assert (
|
||||
render_text(
|
||||
inkling_tokenizer,
|
||||
[{"role": "assistant", "reasoning_content": "", "content": "hi"}],
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
== "<|message_model|><|content_text|>hi<|end_message|>"
|
||||
"<|content_model_end_sampling|>"
|
||||
)
|
||||
|
||||
def test_reasoning_field(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "think",
|
||||
"content": "answer",
|
||||
}
|
||||
]
|
||||
assert render_text(
|
||||
inkling_tokenizer, messages, add_generation_prompt=False
|
||||
) == (
|
||||
"<|message_model|><|content_thinking|>think<|end_message|>"
|
||||
"<|message_model|><|content_text|>answer<|end_message|>"
|
||||
"<|content_model_end_sampling|>"
|
||||
)
|
||||
|
||||
def test_audio_part(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_audio", "input_audio": {}}],
|
||||
}
|
||||
]
|
||||
assert render_text(
|
||||
inkling_tokenizer, messages, add_generation_prompt=False
|
||||
) == ("<|message_user|><|content_audio_input|><|audio_end|><|end_message|>")
|
||||
|
||||
def test_tool_response_name_from_message(self, inkling_tokenizer):
|
||||
assert (
|
||||
render_text(
|
||||
inkling_tokenizer,
|
||||
[{"role": "tool", "name": "my_tool", "content": "ok"}],
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
== "<|message_tool|>my_tool<|content_text|>ok<|end_message|>"
|
||||
)
|
||||
|
||||
def test_tool_call_args_object_form(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "x",
|
||||
"function": {
|
||||
"name": "f",
|
||||
# dict-form arguments, unsorted keys
|
||||
"arguments": {"z": 1, "a": 2},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
assert render_text(
|
||||
inkling_tokenizer, messages, add_generation_prompt=False
|
||||
) == (
|
||||
"<|message_model|>f<|content_invoke_tool_json|>"
|
||||
'{"name":"f","args":{"a":2,"z":1}}<|end_message|>'
|
||||
"<|content_model_end_sampling|>"
|
||||
)
|
||||
|
||||
def test_tool_call_empty_args(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [{"id": "x", "function": {"name": "f", "arguments": ""}}],
|
||||
}
|
||||
]
|
||||
assert render_text(
|
||||
inkling_tokenizer, messages, add_generation_prompt=False
|
||||
) == (
|
||||
"<|message_model|>f<|content_invoke_tool_json|>"
|
||||
'{"name":"f","args":{}}<|end_message|>'
|
||||
"<|content_model_end_sampling|>"
|
||||
)
|
||||
|
||||
def test_tool_call_non_object_args_rejected(self, inkling_tokenizer):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{"id": "x", "function": {"name": "f", "arguments": "[1]"}}
|
||||
],
|
||||
}
|
||||
]
|
||||
with pytest.raises(TypeError, match="decode to an object"):
|
||||
render_inkling_messages(
|
||||
messages, inkling_tokenizer, add_generation_prompt=False
|
||||
)
|
||||
|
||||
def test_unsupported_role_rejected(self, inkling_tokenizer):
|
||||
with pytest.raises(ValueError, match="unsupported Inkling message role"):
|
||||
render_inkling_messages(
|
||||
[{"role": "narrator", "content": "hi"}], inkling_tokenizer
|
||||
)
|
||||
|
||||
|
||||
class TestReasoningEffort:
|
||||
def test_frontend_defaults_to_high(self, inkling_tokenizer):
|
||||
renderer = InklingRenderer.__new__(InklingRenderer)
|
||||
renderer._inkling_tokenizer = inkling_tokenizer
|
||||
|
||||
text = decode(
|
||||
renderer._render(
|
||||
[
|
||||
{"role": "system", "content": "rules"},
|
||||
{"role": "user", "content": "hi"},
|
||||
],
|
||||
ChatParams(),
|
||||
)
|
||||
)
|
||||
|
||||
assert text.startswith(
|
||||
"<|message_system|><|content_text|>rules<|end_message|>"
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.9"
|
||||
"<|end_message|>"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("value", ["none", 0, 0.0, -0.0])
|
||||
def test_zero_effort_has_one_canonical_spelling(self, inkling_tokenizer, value):
|
||||
renderer = InklingRenderer.__new__(InklingRenderer)
|
||||
renderer._inkling_tokenizer = inkling_tokenizer
|
||||
|
||||
text = decode(
|
||||
renderer._render(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
ChatParams(chat_template_kwargs={"reasoning_effort": value}),
|
||||
)
|
||||
)
|
||||
|
||||
assert text.startswith(
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.0"
|
||||
"<|end_message|>"
|
||||
)
|
||||
|
||||
def test_emits_one_effort_after_initial_prefix(self, inkling_tokenizer):
|
||||
effort_block = (
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.7"
|
||||
"<|end_message|>"
|
||||
)
|
||||
text = render_text(
|
||||
inkling_tokenizer,
|
||||
[
|
||||
{"role": "system", "content": "rules"},
|
||||
{"role": "developer", "content": "policy"},
|
||||
{"role": "user", "content": "user1"},
|
||||
{"role": "assistant", "content": "assistant1"},
|
||||
{"role": "user", "content": "user2"},
|
||||
],
|
||||
tools=[{"type": "function", "function": {"name": "f"}}],
|
||||
reasoning_effort=0.7,
|
||||
)
|
||||
|
||||
assert text.count(effort_block) == 1
|
||||
assert (
|
||||
text.index("tool_declare")
|
||||
< text.index("rules")
|
||||
< text.index("policy")
|
||||
< text.index(effort_block)
|
||||
< text.index("user1")
|
||||
< text.index("assistant1")
|
||||
< text.index("user2")
|
||||
)
|
||||
|
||||
def test_renders_after_tool_declare(self, inkling_tokenizer):
|
||||
tools = [{"type": "function", "function": {"name": "f"}}]
|
||||
text = render_text(
|
||||
inkling_tokenizer,
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tools=tools,
|
||||
reasoning_effort=0.8,
|
||||
)
|
||||
declare_end = text.index("<|end_message|>") + len("<|end_message|>")
|
||||
assert text[declare_end:].startswith(
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.8"
|
||||
"<|end_message|>"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[(0.2, "0.2"), (0.7, "0.7"), (0.9, "0.9"), (0.99, "0.99")],
|
||||
)
|
||||
def test_formats_at_most_two_decimals(self, inkling_tokenizer, value, expected):
|
||||
text = render_text(
|
||||
inkling_tokenizer,
|
||||
[{"role": "user", "content": "hi"}],
|
||||
reasoning_effort=value,
|
||||
)
|
||||
assert f"Thinking effort level: {expected}<|end_message|>" in text
|
||||
|
||||
def test_absent_emits_no_block(self, inkling_tokenizer):
|
||||
text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}])
|
||||
assert "Thinking effort" not in text
|
||||
|
||||
@pytest.mark.parametrize("value", [0.9900001, 1, 1.5, -0.1])
|
||||
def test_out_of_range_rejected(self, inkling_tokenizer, value):
|
||||
with pytest.raises(ValueError, match="must be in"):
|
||||
render_inkling_messages(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
inkling_tokenizer,
|
||||
reasoning_effort=value,
|
||||
)
|
||||
|
||||
|
||||
class TestSpecialTokenResolution:
|
||||
def test_missing_special_token_raises(self):
|
||||
class IncompleteTokenizer(FakeHfTokenizer):
|
||||
def get_vocab(self):
|
||||
vocab = dict(_SPECIAL_VOCAB)
|
||||
del vocab["<|content_invoke_tool_json|>"]
|
||||
return vocab
|
||||
|
||||
with pytest.raises(ValueError, match="missing special tokens"):
|
||||
_HfBackedTmlTokenizer(IncompleteTokenizer())
|
||||
|
||||
def test_semantic_spelling_preferred(self):
|
||||
class SemanticSpellingTokenizer(FakeHfTokenizer):
|
||||
def get_vocab(self):
|
||||
vocab = dict(_SPECIAL_VOCAB)
|
||||
del vocab["<|unused_200024|>"]
|
||||
vocab["<|content_xml|>"] = 200024
|
||||
return vocab
|
||||
|
||||
inkling_tokenizer = _HfBackedTmlTokenizer(SemanticSpellingTokenizer())
|
||||
tools = [{"type": "function", "function": {"name": "f"}}]
|
||||
ids = render_inkling_messages(
|
||||
[{"role": "user", "content": "hi"}], inkling_tokenizer, tools=tools
|
||||
)
|
||||
assert 200024 in ids
|
||||
|
||||
def test_all_spellings_covered(self):
|
||||
# Every semantic token must resolve from the reference vocab.
|
||||
for token, spellings in SPECIAL_TOKEN_SPELLINGS.items():
|
||||
assert any(s in _SPECIAL_VOCAB for s in spellings), token
|
||||
|
||||
|
||||
def test_render_inkling_messages_direct_protocol():
|
||||
"""The encoding core only needs the structural tokenizer protocol."""
|
||||
|
||||
class ProtocolTokenizer:
|
||||
def encode_text(self, text):
|
||||
return [ord(c) for c in text]
|
||||
|
||||
def encode_special(self, token):
|
||||
return {
|
||||
"<|message_user|>": 200000,
|
||||
"<|message_model|>": 200001,
|
||||
"<|content_text|>": 200004,
|
||||
"<|content_model_end_sampling|>": 200006,
|
||||
"<|end_message|>": 200010,
|
||||
}[token]
|
||||
|
||||
ids = render_inkling_messages(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
ProtocolTokenizer(),
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
assert ids == [200000, 200004, ord("h"), ord("i"), 200010, 200001]
|
||||
@@ -1,354 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams
|
||||
from vllm.renderers.kimi_k3 import KimiK3Renderer, _merge_k3_media_io_kwargs
|
||||
from vllm.renderers.registry import RENDERER_REGISTRY
|
||||
from vllm.tokenizers.registry import TokenizerRegistry
|
||||
|
||||
|
||||
class StubTokenizer:
|
||||
"""Stands in for the model's TikTokenTokenizer.
|
||||
|
||||
Records the kwargs it is called with and returns fixed token ids, so tests
|
||||
can assert how the renderer drives ``apply_chat_template`` without the real
|
||||
(git-LFS) tiktoken vocabulary.
|
||||
"""
|
||||
|
||||
def __init__(self, token_ids: list[int]) -> None:
|
||||
self.token_ids = token_ids
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self.conversations: list[list[dict[str, Any]]] = []
|
||||
|
||||
def apply_chat_template(self, conversation, **kwargs) -> list[int]:
|
||||
self.conversations.append(conversation)
|
||||
self.calls.append(kwargs)
|
||||
return list(self.token_ids)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "kimi_k3"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
runner_type: str = "generate"
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
multimodal_config: Any = None
|
||||
hf_config: MockHFConfig = field(default_factory=MockHFConfig)
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: Any = None
|
||||
enable_prompt_embeds: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
def _make_renderer(tokenizer: StubTokenizer) -> KimiK3Renderer:
|
||||
config = MockVllmConfig(MockModelConfig(), MockParallelConfig())
|
||||
return KimiK3Renderer(config, tokenizer)
|
||||
|
||||
|
||||
def test_kimi_k3_registered():
|
||||
assert RENDERER_REGISTRY.load_renderer_cls("kimi_k3").__name__ == "KimiK3Renderer"
|
||||
assert (
|
||||
TokenizerRegistry.load_tokenizer_cls("kimi_k3").__name__ == "CachedHfTokenizer"
|
||||
)
|
||||
|
||||
|
||||
def test_k3_media_io_defaults_preserve_original_mode():
|
||||
# Default: K3 keeps the original image mode (no background flattening).
|
||||
assert _merge_k3_media_io_kwargs(None) == {"image": {"image_mode": None}}
|
||||
|
||||
# Server-/request-level values take precedence over the K3 default.
|
||||
assert _merge_k3_media_io_kwargs({"image": {"image_mode": "RGB"}}) == {
|
||||
"image": {"image_mode": "RGB"}
|
||||
}
|
||||
|
||||
# Unrelated image kwargs are merged with the default.
|
||||
assert _merge_k3_media_io_kwargs(
|
||||
{"image": {"rgba_background_color": (0, 0, 0)}}
|
||||
) == {"image": {"image_mode": None, "rgba_background_color": (0, 0, 0)}}
|
||||
|
||||
|
||||
def test_apply_chat_template_forces_tokenize_and_pins_return_dict():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
tools = [{"type": "function", "function": {"name": "search"}}]
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={"tools": tools, "tokenize": False, "thinking": True}
|
||||
)
|
||||
|
||||
token_ids = renderer._apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}], params
|
||||
)
|
||||
|
||||
assert token_ids == [7, 8, 9]
|
||||
kwargs = tokenizer.calls[-1]
|
||||
# tokenize is forced on even though the request asked for False, so K3 keeps
|
||||
# the special-vs-ordinary token distinction instead of re-tokenizing a string.
|
||||
assert kwargs["tokenize"] is True
|
||||
# return_dict is pinned False so we always get a flat list of ids.
|
||||
assert kwargs["return_dict"] is False
|
||||
assert kwargs["tools"] == tools
|
||||
assert kwargs["thinking"] is True
|
||||
|
||||
|
||||
def test_apply_chat_template_translates_standard_thinking_kwargs():
|
||||
# Standard enable_thinking/reasoning_effort kwargs must be translated
|
||||
# to K3's native thinking/thinking_effort.
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={"enable_thinking": False, "reasoning_effort": "none"}
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["thinking"] is False
|
||||
assert "thinking_effort" not in kwargs
|
||||
assert "enable_thinking" not in kwargs
|
||||
assert "reasoning_effort" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", ["low", "high", "max"])
|
||||
def test_apply_chat_template_translates_supported_reasoning_effort(
|
||||
reasoning_effort: str,
|
||||
):
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["thinking_effort"] == reasoning_effort
|
||||
assert "reasoning_effort" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", ["minimal", "medium", "xhigh"])
|
||||
def test_apply_chat_template_rejects_unsupported_reasoning_effort(
|
||||
reasoning_effort: str,
|
||||
):
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert exc_info.value.parameter == "thinking_effort"
|
||||
assert exc_info.value.value == reasoning_effort
|
||||
assert tokenizer.calls == []
|
||||
|
||||
|
||||
def test_apply_chat_template_validates_canonical_native_thinking_effort():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={
|
||||
"thinking_effort": "low",
|
||||
"reasoning_effort": "medium",
|
||||
}
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert tokenizer.calls[-1]["thinking_effort"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking_effort", ["none", "minimal", "medium", "xhigh"])
|
||||
def test_apply_chat_template_rejects_unsupported_native_thinking_effort(
|
||||
thinking_effort: str,
|
||||
):
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(chat_template_kwargs={"thinking_effort": thinking_effort})
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert exc_info.value.parameter == "thinking_effort"
|
||||
assert exc_info.value.value == thinking_effort
|
||||
assert tokenizer.calls == []
|
||||
|
||||
|
||||
def test_apply_chat_template_native_k3_kwargs_take_precedence():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={
|
||||
"thinking": True,
|
||||
"enable_thinking": False,
|
||||
"thinking_effort": "low",
|
||||
"reasoning_effort": "high",
|
||||
}
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["thinking"] is True
|
||||
assert kwargs["thinking_effort"] == "low"
|
||||
|
||||
|
||||
def test_apply_chat_template_adds_k3_api_metadata():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
response_format = {"type": "json_object"}
|
||||
params = ChatParams(
|
||||
tool_choice="required",
|
||||
response_format=response_format,
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
kwargs = tokenizer.calls[-1]
|
||||
assert kwargs["tool_choice"] == "required"
|
||||
assert kwargs["response_format"] == response_format
|
||||
|
||||
|
||||
def test_apply_chat_template_auto_tool_choice_keeps_template_kwarg():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
params = ChatParams(
|
||||
chat_template_kwargs={"tool_choice": "required"},
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
|
||||
|
||||
assert tokenizer.calls[-1]["tool_choice"] == "required"
|
||||
|
||||
|
||||
def test_apply_chat_template_omits_tool_choice_without_tools():
|
||||
tokenizer = StubTokenizer([7, 8, 9])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
|
||||
renderer._apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}], ChatParams(tool_choice=None)
|
||||
)
|
||||
|
||||
assert "tool_choice" not in tokenizer.calls[-1]
|
||||
|
||||
|
||||
def test_render_messages_returns_token_prompt():
|
||||
renderer = _make_renderer(StubTokenizer([1, 2, 3]))
|
||||
|
||||
conversation, prompt = renderer.render_messages(
|
||||
[{"role": "user", "content": "hi"}], ChatParams()
|
||||
)
|
||||
|
||||
assert prompt == {"prompt_token_ids": [1, 2, 3]}
|
||||
assert "multi_modal_data" not in prompt
|
||||
assert conversation[0]["role"] == "user"
|
||||
|
||||
|
||||
def test_render_messages_derives_private_xtml_tool_attrs():
|
||||
tokenizer = StubTokenizer([1, 2, 3])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
|
||||
conversation, _ = renderer.render_messages(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "lookup:0",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "lookup:1",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "lookup:1",
|
||||
"tool": "client-supplied-name",
|
||||
"index": 99,
|
||||
"content": "second",
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "lookup:0",
|
||||
"content": "first",
|
||||
},
|
||||
],
|
||||
ChatParams(),
|
||||
)
|
||||
|
||||
assert [message["content"] for message in conversation[1:]] == [
|
||||
"first",
|
||||
"second",
|
||||
]
|
||||
assert conversation[1]["tool"] == "lookup"
|
||||
assert conversation[1]["index"] == 1
|
||||
assert conversation[2]["tool"] == "lookup"
|
||||
assert conversation[2]["index"] == 2
|
||||
assert tokenizer.conversations[-1] == conversation
|
||||
|
||||
|
||||
def test_render_messages_ignores_client_supplied_xtml_tool_attrs():
|
||||
tokenizer = StubTokenizer([1, 2, 3])
|
||||
renderer = _make_renderer(tokenizer)
|
||||
|
||||
conversation, _ = renderer.render_messages(
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "lookup:0",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "unknown",
|
||||
"tool": "lookup",
|
||||
"index": 1,
|
||||
"content": "result",
|
||||
},
|
||||
],
|
||||
ChatParams(),
|
||||
)
|
||||
|
||||
assert "tool" not in conversation[1]
|
||||
assert "index" not in conversation[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_messages_async_returns_token_prompt():
|
||||
renderer = _make_renderer(StubTokenizer([4, 5]))
|
||||
|
||||
conversation, prompt = await renderer.render_messages_async(
|
||||
[{"role": "user", "content": "hi"}], ChatParams()
|
||||
)
|
||||
|
||||
assert prompt == {"prompt_token_ids": [4, 5]}
|
||||
assert conversation[0]["role"] == "user"
|
||||
@@ -1,142 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
|
||||
|
||||
from vllm.renderers import ChatParams
|
||||
from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
MODEL_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
runner_type = "generate"
|
||||
model: str = MODEL_NAME
|
||||
tokenizer: str = MODEL_NAME
|
||||
trust_remote_code: bool = False
|
||||
max_model_len: int = 100
|
||||
tokenizer_revision = None
|
||||
tokenizer_mode = "mistral"
|
||||
hf_config = MockHFConfig()
|
||||
encoder_config: dict[str, Any] | None = None
|
||||
enable_prompt_embeds: bool = True
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
supports_multimodal_inputs: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_mistral_tokenizer_does_not_block_event_loop():
|
||||
expected_tokens = [1, 2, 3]
|
||||
|
||||
# Mock the blocking version to sleep
|
||||
def mocked_apply_chat_template(*_args, **_kwargs):
|
||||
time.sleep(2)
|
||||
return expected_tokens
|
||||
|
||||
mock_model_config = MockModelConfig(skip_tokenizer_init=True)
|
||||
mock_tokenizer = Mock(spec=MistralTokenizer)
|
||||
mock_tokenizer.apply_chat_template = mocked_apply_chat_template
|
||||
mock_renderer = MistralRenderer(
|
||||
MockVllmConfig(mock_model_config, parallel_config=MockParallelConfig()),
|
||||
tokenizer=mock_tokenizer,
|
||||
)
|
||||
|
||||
task = mock_renderer.render_messages_async([], ChatParams())
|
||||
|
||||
# Ensure the event loop is not blocked
|
||||
blocked_count = 0
|
||||
for _i in range(20): # Check over ~2 seconds
|
||||
start = time.perf_counter()
|
||||
await asyncio.sleep(0)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
# an overly generous elapsed time for slow machines
|
||||
if elapsed >= 0.5:
|
||||
blocked_count += 1
|
||||
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Ensure task completes
|
||||
_, prompt = await task
|
||||
assert prompt["prompt_token_ids"] == expected_tokens, (
|
||||
"Mocked blocking tokenizer was not called"
|
||||
)
|
||||
assert blocked_count == 0, "Event loop blocked during tokenization"
|
||||
|
||||
|
||||
def test_apply_mistral_chat_template_thinking_chunk():
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{
|
||||
"type": "thinking",
|
||||
"closed": True,
|
||||
"thinking": "Only return the answer when you are confident.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me think about it."},
|
||||
{"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "The answer is 4.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Thanks, what is 3+3?"},
|
||||
]
|
||||
mistral_tokenizer = MistralTokenizer.from_pretrained(
|
||||
"mistralai/Magistral-Small-2509"
|
||||
)
|
||||
|
||||
tokens_ids = safe_apply_chat_template(
|
||||
mistral_tokenizer, messages, chat_template=None, tools=None
|
||||
)
|
||||
|
||||
string_tokens = mistral_tokenizer.mistral.decode(
|
||||
tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
|
||||
)
|
||||
|
||||
expected_tokens = (
|
||||
r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
|
||||
r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
|
||||
r"[INST]What is 2+2?[/INST]"
|
||||
r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
|
||||
r"[INST]Thanks, what is 3+3?[/INST]"
|
||||
)
|
||||
|
||||
assert string_tokens == expected_tokens
|
||||
@@ -1,166 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Truncation must bound the prompt *after* multimodal placeholders expand.
|
||||
|
||||
``TokenizeParams.apply_post_tokenization`` truncates the unexpanded prompt, so
|
||||
one placeholder token can expand into many and push the result back over
|
||||
``truncate_prompt_tokens`` (vllm-project/vllm#57038).
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.hf import _ensure_prompt_embeds_placeholder_token
|
||||
from vllm.renderers.params import TokenizeParams
|
||||
|
||||
|
||||
def _renderer():
|
||||
"""A BaseRenderer stub: only the tokenizer attribute is needed here."""
|
||||
|
||||
class _StubRenderer(BaseRenderer):
|
||||
def __init__(self):
|
||||
self.tokenizer = None
|
||||
|
||||
def render_messages(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
return _StubRenderer()
|
||||
|
||||
|
||||
def _mm_input(num_tokens: int, offset: int, length: int, **extra):
|
||||
return {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": list(range(num_tokens)),
|
||||
"mm_kwargs": {},
|
||||
"mm_hashes": {},
|
||||
"mm_placeholders": {"audio": [PlaceholderRange(offset=offset, length=length)]},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _params(limit: int | None, side: str = "right"):
|
||||
return TokenizeParams(
|
||||
max_total_tokens=4096,
|
||||
truncate_prompt_tokens=limit,
|
||||
truncation_side=side,
|
||||
)
|
||||
|
||||
|
||||
def test_expanded_prompt_is_bounded():
|
||||
"""The reported case: audio fits, trailing text is cut to the limit."""
|
||||
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(96))
|
||||
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
|
||||
offset=1, length=52
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_within_limit_is_untouched():
|
||||
engine_input = _mm_input(num_tokens=64, offset=1, length=52)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(64))
|
||||
|
||||
|
||||
def test_split_multimodal_span_is_rejected():
|
||||
"""Cutting mid-span would desync features from token ids, so reject."""
|
||||
engine_input = _mm_input(num_tokens=147, offset=80, length=50)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
|
||||
def test_dropped_multimodal_span_is_rejected():
|
||||
"""A span entirely past the cut would be discarded silently."""
|
||||
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
|
||||
def test_left_truncation_shifts_placeholder_offsets():
|
||||
engine_input = _mm_input(num_tokens=147, offset=100, length=20)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96, side="left"))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(51, 147))
|
||||
# The span started at 100 and 51 leading tokens were dropped.
|
||||
assert engine_input["mm_placeholders"]["audio"][0] == PlaceholderRange(
|
||||
offset=49, length=20
|
||||
)
|
||||
|
||||
|
||||
def test_text_only_input_is_untouched():
|
||||
"""Only multimodal prompts grow after tokenization, so nothing else runs."""
|
||||
engine_input = {
|
||||
"type": "token",
|
||||
"prompt_token_ids": list(range(147)),
|
||||
"prompt_token_offsets": [(i, i + 1) for i in range(147)],
|
||||
}
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(96))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(147))
|
||||
assert len(engine_input["prompt_token_offsets"]) == 147
|
||||
|
||||
|
||||
def test_no_truncation_requested_is_noop():
|
||||
engine_input = _mm_input(num_tokens=147, offset=1, length=52)
|
||||
|
||||
_renderer()._truncate_expanded_prompt(engine_input, _params(None))
|
||||
|
||||
assert engine_input["prompt_token_ids"] == list(range(147))
|
||||
|
||||
|
||||
def test_mixed_prompt_embeds_truncation_covers_embeds_spans():
|
||||
"""Bound in _process_singleton, where prompt_embeds spans are already attached."""
|
||||
model_config = ModelConfig(
|
||||
"Qwen/Qwen3-ASR-0.6B-hf",
|
||||
revision="7f1569a48a89f3e3f4dc3a5c9d28bddd903bc76c",
|
||||
dtype="float32",
|
||||
max_model_len=4096,
|
||||
mm_processor_cache_gb=0,
|
||||
enable_prompt_embeds=True,
|
||||
)
|
||||
renderer = renderer_from_config(VllmConfig(model_config=model_config))
|
||||
tok = renderer.tokenizer
|
||||
pe_id = _ensure_prompt_embeds_placeholder_token(tok)
|
||||
hidden = model_config.get_hidden_size()
|
||||
audio = np.zeros(4 * 16000, dtype=np.float32)
|
||||
|
||||
pe_len = 20
|
||||
tensors = [torch.zeros(pe_len, hidden, dtype=torch.float32)]
|
||||
|
||||
def _build_prompt():
|
||||
text = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
ids = tok.encode(text) + [pe_id] + tok.encode(" tail" * 40)
|
||||
return {
|
||||
"prompt_token_ids": ids,
|
||||
"multi_modal_data": {"audio": (audio, 16000)},
|
||||
"_prompt_embeds": (tensors, pe_id),
|
||||
}
|
||||
|
||||
# A limit past every span (audio ends at 52, prompt_embeds at 74):
|
||||
# truncation should succeed and bound the prompt.
|
||||
out = renderer._process_singleton(
|
||||
_build_prompt(),
|
||||
tok_params=_params(79),
|
||||
)
|
||||
assert len(out["prompt_token_ids"]) == 79
|
||||
|
||||
# A limit inside the prompt_embeds span (54..74): must be rejected.
|
||||
with pytest.raises(VLLMValidationError, match="truncate_prompt_tokens"):
|
||||
renderer._process_singleton(
|
||||
_build_prompt(),
|
||||
tok_params=_params(64),
|
||||
)
|
||||
Reference in New Issue
Block a user