[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,558 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import AsyncGenerator
from unittest.mock import MagicMock
import httpx
import pytest
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
BASE_MODEL_PATHS,
CHAT_TEMPLATE,
MockHFConfig,
MockModelConfig,
_build_renderer,
)
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
MODEL_NAME as MOCK_MODEL_NAME,
)
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.exceptions import VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.reasoning import ReasoningParserManager
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
# any model with a chat template defined in tokenizer_config should work here
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@pytest.fixture(scope="module")
def default_server_args():
return [
# use half precision for speed and memory savings in CI environment
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
]
@pytest.fixture(scope="module")
def server(default_server_args):
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions(
server: RemoteOpenAIServer, model_name: str
) -> None:
conversations = [
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
[{"role": "user", "content": "Reply with exactly the word: beta"}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
choices = data["choices"]
assert len(choices) == 2
indices = {choice["index"] for choice in choices}
assert indices == {0, 1}
# Each conversation should produce a non-empty text response.
for choice in choices:
assert choice["message"]["content"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions_with_json_schema(
server: RemoteOpenAIServer, model_name: str
) -> None:
schema = {
"type": "object",
"properties": {
"answer": {"type": "string", "enum": ["yes", "no"]},
},
"required": ["answer"],
}
conversations = [
[{"role": "user", "content": "Is the sky blue? Answer in JSON."}],
[{"role": "user", "content": "Is fire cold? Answer in JSON."}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
"response_format": {
"type": "json_schema",
"json_schema": {"name": "answer", "schema": schema, "strict": True},
},
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
choices = data["choices"]
assert len(choices) == 2
for choice in choices:
parsed = json.loads(choice["message"]["content"])
assert "answer" in parsed
assert parsed["answer"] in ("yes", "no")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions_logprobs_not_token_id_placeholders(
server: RemoteOpenAIServer, model_name: str
) -> None:
# Regression test: requesting `return_token_ids` alongside logprobs must not
# corrupt the logprob `token` fields into "token_id:{id}" placeholders. That
# placeholder rendering is controlled by `return_tokens_as_token_ids`, which
# this request leaves unset.
conversations = [
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
"logprobs": True,
"top_logprobs": 1,
"return_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
content = data["choices"][0]["logprobs"]["content"]
assert content
for entry in content:
assert not entry["token"].startswith("token_id:")
for top in entry["top_logprobs"]:
assert not top["token"].startswith("token_id:")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions_return_tokens_as_token_ids(
server: RemoteOpenAIServer, model_name: str
) -> None:
# Complementary check: when `return_tokens_as_token_ids` is explicitly set,
# the logprob tokens *should* be rendered as "token_id:{id}" placeholders,
# proving the new field is actually wired through.
conversations = [
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
"logprobs": True,
"top_logprobs": 1,
"return_tokens_as_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
content = data["choices"][0]["logprobs"]["content"]
assert content
assert all(entry["token"].startswith("token_id:") for entry in content)
@pytest.mark.asyncio
async def test_batched_chat_completions_logprob_token_ids(
server: RemoteOpenAIServer,
) -> None:
conversations = [[{"role": "user", "content": "Hello"}]]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": MODEL_NAME,
"messages": conversations,
"max_tokens": 1,
"temperature": 0,
"logprobs": True,
"top_logprobs": 5,
"logprob_token_ids": [100, 1000, 5000],
"return_tokens_as_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
content = response.json()["choices"][0]["logprobs"]["content"]
assert content
sampled_token = content[0]["token"]
assert {entry["token"] for entry in content[0]["top_logprobs"]} == {
"token_id:100",
"token_id:1000",
"token_id:5000",
sampled_token,
}
def _make_request_output(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> RequestOutput:
return RequestOutput(
request_id=f"req-{prompt_idx}",
prompt=None,
prompt_token_ids=prompt_token_ids or [1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=[4, 5],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
)
async def _generator(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text, prompt_token_ids)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
"""Echo must only echo the assistant turn, never the user's prompt.
With `add_generation_prompt` (the default) the response role is `assistant`
while the last conversation message is the user's, so there is nothing to
echo; without the role check the user prompt leaks into the answer.
"""
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
conversations = [[{"role": "user", "content": "USER PROMPT"}]]
request = BatchChatCompletionRequest(
model="test-model",
messages=conversations,
echo=True,
)
response = await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "ASSISTANT ANSWER")],
request_id="req-echo",
model_name="test-model",
all_conversations=conversations,
tokenizer=None,
request_metadata=RequestResponseMetadata(request_id="req-echo"),
)
assert response.choices[0].message.content == "ASSISTANT ANSWER"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
"""A trailing assistant turn is a real prefix and must still be echoed."""
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
conversations = [
[
{"role": "user", "content": "USER PROMPT"},
{"role": "assistant", "content": "PREFIX "},
]
]
request = BatchChatCompletionRequest(
model="test-model",
messages=conversations,
echo=True,
add_generation_prompt=False,
continue_final_message=True,
)
response = await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "ASSISTANT ANSWER")],
request_id="req-echo-2",
model_name="test-model",
all_conversations=conversations,
tokenizer=None,
request_metadata=RequestResponseMetadata(request_id="req-echo-2"),
)
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
"""Harmony must constrain only the final channel of every conversation."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.model_config = MockModelConfig()
mock_engine.errored = False
mock_engine.input_processor = MagicMock()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.renderer = _build_renderer(mock_engine.model_config)
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
online_renderer = OnlineRenderer(
model_config=mock_engine.model_config,
renderer=mock_engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="openai_gptoss",
)
serving = OpenAIServingChatBatch(
mock_engine,
models,
response_role="assistant",
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
request_logger=None,
)
request = BatchChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=[
[{"role": "user", "content": "France?"}],
[{"role": "user", "content": "Japan?"}],
],
response_format={
"type": "json_schema",
"json_schema": {"name": "city", "schema": {"type": "object"}},
},
)
await serving.create_batch_chat_completion(request)
calls = mock_engine.generate.call_args_list
assert len(calls) == 2
for call in calls:
assert call.args[1].structured_outputs.structural_tag is not None
@pytest.mark.skip_global_cleanup
def test_batch_rejects_kv_transfer_prompt_token_ids():
"""One pre-tokenized prompt cannot stand in for every conversation."""
with pytest.raises(
VLLMValidationError, match=r"parameter=kv_transfer_params\.prompt_token_ids"
):
BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_parser_receives_each_prompt_prefix() -> None:
prefixes = []
class RecordingParser:
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = prompt_token_ids
def parse(self, model_output, **kwargs):
prefixes.append(self.prompt_token_ids)
return None, model_output, None
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request = BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
)
await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
request_id="req-prefix",
model_name="test-model",
all_conversations=request.messages,
tokenizer=object(),
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
parser_cls=RecordingParser,
)
assert prefixes == [[1], [2]]
REASONING_PARSER = "kimi_k3"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_reasoning_wiring_matches_single_chat() -> None:
"""`/v1/chat/completions/batch` must hand the engine the same reasoning
wiring as `/v1/chat/completions`.
Both `reasoning_ended` and `reasoning_parser_kwargs` are carried on the
request into `EngineCoreRequest`, where
`StructuredOutputManager._get_reasoner()` rebuilds the request-local
reasoning parser from the forwarded `chat_template_kwargs` -- "so the
structured-output gate observes the same template kwargs used by the
frontend". A divergence makes a template-kwarg-driven parser fall back to
its default (`thinking` on) for a request that disabled thinking, so
`is_reasoning_end()` keeps reporting False, the structured-output gate
never opens, and the grammar is silently never applied.
Both frontends run over one and the same model and renderer, so the only
thing that can differ between the two calls is the serving code itself.
"""
mock_model_config = MockModelConfig()
shared_renderer = _build_renderer(mock_model_config)
def _build_serving(cls):
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.model_config = mock_model_config
mock_engine.errored = False
mock_engine.input_processor = MagicMock()
mock_engine.renderer = shared_renderer
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
online_renderer = OnlineRenderer(
model_config=mock_model_config,
renderer=mock_engine.renderer,
request_logger=None,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
reasoning_parser=REASONING_PARSER,
)
return mock_engine, cls(
mock_engine,
models,
response_role="assistant",
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
reasoning_parser=REASONING_PARSER,
)
conversation = [{"role": "user", "content": "What is the capital of France?"}]
chat_template_kwargs = {"enable_thinking": False}
single_engine, single_serving = _build_serving(OpenAIServingChat)
await single_serving.create_chat_completion(
ChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=conversation,
chat_template_kwargs=chat_template_kwargs,
)
)
single_call = single_engine.generate.call_args_list[-1]
batch_engine, batch_serving = _build_serving(OpenAIServingChatBatch)
await batch_serving.create_batch_chat_completion(
BatchChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=[conversation],
chat_template_kwargs=chat_template_kwargs,
)
)
batch_call = batch_engine.generate.call_args_list[-1]
# Guard against a vacuous comparison: the reference path must be wired up.
assert single_call.kwargs["reasoning_ended"] is not None
assert single_call.kwargs["reasoning_parser_kwargs"] is not None
assert batch_call.kwargs["reasoning_ended"] == single_call.kwargs["reasoning_ended"]
assert (
batch_call.kwargs["reasoning_parser_kwargs"]
== single_call.kwargs["reasoning_parser_kwargs"]
)
# ...and the forwarded kwargs must actually turn thinking off for the
# engine-side parser, i.e. the structured-output gate opens right away.
engine_chat_template_kwargs = batch_call.kwargs["reasoning_parser_kwargs"][
"chat_template_kwargs"
]
reasoner = ReasoningParserManager.get_reasoning_parser(REASONING_PARSER)(
tokenizer=batch_engine.renderer.tokenizer,
chat_template_kwargs=engine_chat_template_kwargs,
)
assert reasoner.is_reasoning_end([]) is True
@@ -1,298 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for `prompt_embeds` content parts in the Chat Completions API."""
import asyncio
import io
import openai
import pybase64 as base64
import pytest
import pytest_asyncio
import torch
from openai import BadRequestError
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm.platforms import current_platform
MODEL_NAME = "facebook/opt-125m"
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
# Matches `--dtype` in `server_args` to avoid an implicit cast in
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
# model's dtype automatically, we match here just to skip the conversion).
SERVER_DTYPE: torch.dtype = torch.bfloat16
@pytest.fixture(scope="module")
def server_args() -> list[str]:
return [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
"--chat-template",
str(CHAT_TEMPLATE),
# Prompt Embeds server args
"--enable-prompt-embeds",
]
@pytest.fixture(scope="module")
def server(server_args, request):
if current_platform.is_rocm():
# Materialize HF embeddings before the server reserves ROCm VRAM.
request.getfixturevalue("prompt_embeds_b64")
request.getfixturevalue("aligned_content_and_embeds_b64")
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
def _encode_embeds(embeds: torch.Tensor) -> str:
buf = io.BytesIO()
torch.save(embeds, buf)
return base64.b64encode(buf.getvalue()).decode("utf-8")
@pytest.fixture(scope="module")
def prompt_embeds_b64(hf_runner) -> list[str]:
"""Pre-compute embeddings for two short prompts and return as base64."""
prompts = ["Hello, my name is", "What is an LLM?"]
with hf_runner(MODEL_NAME) as hf_model:
embeddings = hf_model.get_prompt_embeddings(prompts)
# Cast to the server's dtype so `safe_load_prompt_embeds` doesn't need to
# convert on its own, the function accepts any floating-point dtype and
# will cast to the model's dtype, but matching up front skips the work.
return [_encode_embeds(e.to(SERVER_DTYPE)) for e in embeddings]
@pytest.mark.asyncio
async def test_single_prompt_embeds_part(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""A user message with one prompt_embeds part + text."""
b64 = prompt_embeds_b64[0]
chat = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": "Continue:"},
],
}
],
)
assert chat.choices[0].message.content is not None
assert len(chat.choices[0].message.content) > 0
@pytest.mark.asyncio
async def test_multiple_prompt_embeds_parts(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""Multiple prompt_embeds parts in a single message."""
b64_a, b64_b = prompt_embeds_b64
chat = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64_a},
{"type": "text", "text": " and "},
{"type": "prompt_embeds", "data": b64_b},
],
}
],
)
assert chat.choices[0].message.content is not None
assert len(chat.choices[0].message.content) > 0
@pytest.mark.asyncio
async def test_multi_message_conversation(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""prompt_embeds in both system and user messages."""
b64_sys, b64_usr = prompt_embeds_b64
chat = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "system",
"content": [
{"type": "text", "text": "You are helpful."},
{"type": "prompt_embeds", "data": b64_sys},
],
},
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64_usr},
{"type": "text", "text": "Summarize."},
],
},
],
)
assert chat.choices[0].message.content is not None
assert len(chat.choices[0].message.content) > 0
@pytest.mark.asyncio
async def test_streaming(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""Streaming chat completion with prompt_embeds."""
b64 = prompt_embeds_b64[0]
# Non-streaming baseline.
baseline = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": "Continue:"},
],
}
],
)
expected = baseline.choices[0].message.content
# Streaming.
stream = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
stream=True,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": "Continue:"},
],
}
],
)
chunks: list[str] = []
async for chunk in stream:
delta = chunk.choices[0].delta.content
if delta:
chunks.append(delta)
assert "".join(chunks) == expected
@pytest.fixture(scope="module")
def aligned_content_and_embeds_b64(hf_runner) -> tuple[str, str]:
"""Return `(content, base64_embeds)` where the embeddings are the model's
embedding of `content` tokenized WITHOUT special tokens.
"""
content = "Hello, my name is"
with hf_runner(MODEL_NAME) as hf_model:
ids = hf_model.tokenizer(
content, add_special_tokens=False, return_tensors="pt"
).input_ids
ids = hf_model.wrap_device({"input_ids": ids})["input_ids"]
embed_layer = hf_model.model.get_input_embeddings()
embeds = embed_layer(ids).squeeze(0).to(SERVER_DTYPE).cpu()
return content, _encode_embeds(embeds)
@pytest.mark.asyncio
async def test_text_content_and_prompt_embeds_match(
client: openai.AsyncOpenAI,
aligned_content_and_embeds_b64: tuple[str, str],
):
"""Equal content in text and `prompt_embeds` should yield identical
Chat Completions output under greedy decoding.
"""
content, encoded_embeds = aligned_content_and_embeds_b64
text_resp, embeds_resp = await asyncio.gather(
client.chat.completions.create(
model=MODEL_NAME,
max_tokens=10,
temperature=0.0,
messages=[{"role": "user", "content": content}],
),
client.chat.completions.create(
model=MODEL_NAME,
max_tokens=10,
temperature=0.0,
messages=[
{
"role": "user",
"content": [{"type": "prompt_embeds", "data": encoded_embeds}],
}
],
),
)
text_out = text_resp.choices[0].message.content
embeds_out = embeds_resp.choices[0].message.content
assert text_out is not None and len(text_out) > 0
assert embeds_out is not None and len(embeds_out) > 0
assert text_out == embeds_out
@pytest.mark.asyncio
async def test_missing_data_field(
client: openai.AsyncOpenAI,
):
"""A prompt_embeds part without `data` should return a clear error."""
with pytest.raises(BadRequestError):
await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
messages=[
{
"role": "user",
"content": [{"type": "prompt_embeds"}],
}
],
)
@pytest.mark.asyncio
async def test_invalid_base64(
client: openai.AsyncOpenAI,
):
"""Invalid base64 in the `data` field should return a clear error."""
with pytest.raises(BadRequestError):
await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": "not_valid_base64!!"},
],
}
],
)
File diff suppressed because it is too large Load Diff
@@ -1,341 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for ``thinking_token_budget`` with reasoning models.
Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
"""
import asyncio
import json
from typing import Literal
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
from vllm.platforms import current_platform
from vllm.tokenizers import get_tokenizer
MODEL_NAME = "Qwen/Qwen3-0.6B"
QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
THINK_BUDGET = 5
REASONING_START_STR = "<think>"
REASONING_END_STR = "</think>"
def _count_reasoning_decode_token_ids_between_markers(
full_token_ids: list[int],
reasoning_start_ids: list[int],
reasoning_end_ids: list[int],
) -> int | None:
"""Count decode tokens in the thinking span (after last start, before first end)."""
if not reasoning_start_ids or not reasoning_end_ids:
raise ValueError("reasoning marker token id lists must be non-empty")
def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
n = len(needle)
if n > len(haystack):
return -1
for i in range(len(haystack) - n, -1, -1):
if haystack[i : i + n] == needle:
return i
return -1
last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
if last_start < 0:
return None
pos_after_start = last_start + len(reasoning_start_ids)
end_n = len(reasoning_end_ids)
for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
if full_token_ids[j : j + end_n] == reasoning_end_ids:
return j - pos_after_start
return len(full_token_ids) - pos_after_start
@pytest.fixture(scope="module")
def server():
args = [
"--reasoning-parser",
"qwen3",
"--reasoning-config",
'{"reasoning_start_str": "<think>", "reasoning_end_str": "</think>"}',
"--max-model-len",
"2048",
"--enforce-eager",
"--gpu-memory-utilization",
"0.4",
"--no-async-scheduling",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def server_with_auto_reasoning_config():
args = [
"--reasoning-parser",
"qwen3",
"--max-model-len",
"2048",
"--enforce-eager",
"--gpu-memory-utilization",
"0.4",
"--no-async-scheduling",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def server_qwen35_fp8_mtp_tp2():
"""Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
if current_platform.device_count() < 2:
pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
if not current_platform.supports_fp8():
pytest.skip("FP8 is not supported on this platform")
spec_cfg = {
"method": "mtp",
"num_speculative_tokens": 2,
"max_model_len": 32768,
}
args = [
"--tensor-parallel-size",
"2",
"--max-model-len",
"32768",
"--speculative-config",
json.dumps(spec_cfg),
"--reasoning-parser",
"qwen3",
"--reasoning-config",
json.dumps(
{
"reasoning_start_str": REASONING_START_STR,
"reasoning_end_str": REASONING_END_STR,
}
),
]
env_dict: dict[str, str] = {}
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
# on 0,1 do not exhaust memory on the same devices as this worker.
if current_platform.device_count() >= 4:
env_dict["CUDA_VISIBLE_DEVICES"] = "2,3"
with RemoteOpenAIServer(
QWEN35_FP8_MTP_MODEL,
args,
max_wait_seconds=3000,
env_dict=env_dict,
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(request, server, server_with_auto_reasoning_config):
server_map = {
"default": server,
"auto_config": server_with_auto_reasoning_config,
}
target_server = server_map[request.param]
async with target_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
"""Test that mixed requests (some with thinking_token_budget, some without)
complete successfully without errors."""
response_with_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
extra_body={"thinking_token_budget": THINK_BUDGET},
)
response_without_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
)
msg_with = response_with_budget.choices[0].message
msg_without = response_without_budget.choices[0].message
assert msg_with.content or getattr(msg_with, "reasoning", None)
assert msg_without.content or getattr(msg_without, "reasoning", None)
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
"""Test that thinking_token_budget limits the number of reasoning tokens.
Counts reasoning decode tokens by id, which is robust to how tokens are
grouped into streamed chunks (a single chunk can carry several tokens under
async scheduling / stream_interval > 1). Counting chunks under-counts.
"""
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
prompt_token_ids: list[int] = []
decode_token_ids: list[int] = []
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
stream=True,
extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True},
)
async for chunk in stream:
if not chunk.choices:
continue
if getattr(chunk, "prompt_token_ids", None):
prompt_token_ids = list(chunk.prompt_token_ids)
delta_ids = getattr(chunk.choices[0], "token_ids", None)
if delta_ids:
decode_token_ids.extend(delta_ids)
reasoning_token_count = _count_reasoning_decode_token_ids_between_markers(
prompt_token_ids + decode_token_ids, start_ids, end_ids
)
assert reasoning_token_count is not None, "missing reasoning start marker in ids"
assert reasoning_token_count == THINK_BUDGET, (
f"reasoning tokens ({reasoning_token_count}) != "
f"thinking_token_budget ({THINK_BUDGET})"
)
@pytest.mark.asyncio
@multi_gpu_only(num_gpus=2)
@requires_fp8
async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
server_qwen35_fp8_mtp_tp2,
):
"""Concurrent chat requests: some with ``thinking_token_budget``, some without.
Exercises the scheduler / input processor under a mixed batch on the same
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
"""
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
("budget", 1),
("budget", 12),
("plain", None),
("budget", 20),
("budget", 14),
("plain", None),
("plain", None),
("budget", 12),
("plain", None),
]
tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
async def budgeted_call(expected_budget: int):
return await client.chat.completions.create(
model=QWEN35_FP8_MTP_MODEL,
messages=MESSAGES,
max_tokens=256,
stream=False,
extra_body={
"thinking_token_budget": expected_budget,
"return_token_ids": True,
},
)
async def plain_call():
return await client.chat.completions.create(
model=QWEN35_FP8_MTP_MODEL,
messages=MESSAGES,
max_tokens=256,
stream=False,
)
coros = []
for row in _batch_spec:
if row[0] == "budget":
b = row[1]
assert isinstance(b, int)
coros.append(budgeted_call(b))
else:
coros.append(plain_call())
results = await asyncio.gather(*coros)
for i, (response, (kind, expected_budget)) in enumerate(
zip(results, _batch_spec, strict=True)
):
msg = response.choices[0].message
assert msg.content or getattr(msg, "reasoning", None), (
f"index {i} ({kind}): empty message"
)
if kind == "budget":
assert expected_budget is not None
assert response.prompt_token_ids is not None
assert response.choices[0].token_ids is not None
full_ids = list(response.prompt_token_ids) + list(
response.choices[0].token_ids
)
n_reason = _count_reasoning_decode_token_ids_between_markers(
full_ids, start_ids, end_ids
)
assert n_reason is not None, f"index {i}: missing reasoning start in ids"
assert n_reason == expected_budget, (
f"index {i}: reasoning decode token ids ({n_reason}) != "
f"thinking_token_budget ({expected_budget})"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_streaming_with_thinking_disabled_stays_in_content(
client: openai.AsyncOpenAI,
):
request_kwargs = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": "Which is larger, 4 or 12?"
" Output exactly one token: 4 or 12.",
}
],
"max_tokens": 16,
"temperature": 0.0,
"extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
}
response = await client.chat.completions.create(**request_kwargs)
message = response.choices[0].message
assert message.content is not None and message.content.strip() != ""
assert getattr(message, "reasoning", None) in (None, "")
stream = await client.chat.completions.create(
**request_kwargs,
stream=True,
)
content_chunks = []
reasoning_chunks = []
async for chunk in stream:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if getattr(delta, "content", None):
content_chunks.append(delta.content)
if getattr(delta, "reasoning", None):
reasoning_chunks.append(delta.reasoning)
assert "".join(content_chunks).strip() != ""
assert reasoning_chunks == []