[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,558 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import AsyncGenerator
from unittest.mock import MagicMock
import httpx
import pytest
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
BASE_MODEL_PATHS,
CHAT_TEMPLATE,
MockHFConfig,
MockModelConfig,
_build_renderer,
)
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
MODEL_NAME as MOCK_MODEL_NAME,
)
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.exceptions import VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.reasoning import ReasoningParserManager
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
# any model with a chat template defined in tokenizer_config should work here
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@pytest.fixture(scope="module")
def default_server_args():
return [
# use half precision for speed and memory savings in CI environment
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
]
@pytest.fixture(scope="module")
def server(default_server_args):
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions(
server: RemoteOpenAIServer, model_name: str
) -> None:
conversations = [
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
[{"role": "user", "content": "Reply with exactly the word: beta"}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
choices = data["choices"]
assert len(choices) == 2
indices = {choice["index"] for choice in choices}
assert indices == {0, 1}
# Each conversation should produce a non-empty text response.
for choice in choices:
assert choice["message"]["content"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions_with_json_schema(
server: RemoteOpenAIServer, model_name: str
) -> None:
schema = {
"type": "object",
"properties": {
"answer": {"type": "string", "enum": ["yes", "no"]},
},
"required": ["answer"],
}
conversations = [
[{"role": "user", "content": "Is the sky blue? Answer in JSON."}],
[{"role": "user", "content": "Is fire cold? Answer in JSON."}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
"response_format": {
"type": "json_schema",
"json_schema": {"name": "answer", "schema": schema, "strict": True},
},
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
choices = data["choices"]
assert len(choices) == 2
for choice in choices:
parsed = json.loads(choice["message"]["content"])
assert "answer" in parsed
assert parsed["answer"] in ("yes", "no")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions_logprobs_not_token_id_placeholders(
server: RemoteOpenAIServer, model_name: str
) -> None:
# Regression test: requesting `return_token_ids` alongside logprobs must not
# corrupt the logprob `token` fields into "token_id:{id}" placeholders. That
# placeholder rendering is controlled by `return_tokens_as_token_ids`, which
# this request leaves unset.
conversations = [
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
"logprobs": True,
"top_logprobs": 1,
"return_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
content = data["choices"][0]["logprobs"]["content"]
assert content
for entry in content:
assert not entry["token"].startswith("token_id:")
for top in entry["top_logprobs"]:
assert not top["token"].startswith("token_id:")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batched_chat_completions_return_tokens_as_token_ids(
server: RemoteOpenAIServer, model_name: str
) -> None:
# Complementary check: when `return_tokens_as_token_ids` is explicitly set,
# the logprob tokens *should* be rendered as "token_id:{id}" placeholders,
# proving the new field is actually wired through.
conversations = [
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": model_name,
"messages": conversations,
"logprobs": True,
"top_logprobs": 1,
"return_tokens_as_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
data = response.json()
content = data["choices"][0]["logprobs"]["content"]
assert content
assert all(entry["token"].startswith("token_id:") for entry in content)
@pytest.mark.asyncio
async def test_batched_chat_completions_logprob_token_ids(
server: RemoteOpenAIServer,
) -> None:
conversations = [[{"role": "user", "content": "Hello"}]]
async with httpx.AsyncClient() as http_client:
response = await http_client.post(
f"{server.url_for('v1/chat/completions/batch')}",
json={
"model": MODEL_NAME,
"messages": conversations,
"max_tokens": 1,
"temperature": 0,
"logprobs": True,
"top_logprobs": 5,
"logprob_token_ids": [100, 1000, 5000],
"return_tokens_as_token_ids": True,
},
timeout=60,
)
assert response.status_code == 200, response.text
content = response.json()["choices"][0]["logprobs"]["content"]
assert content
sampled_token = content[0]["token"]
assert {entry["token"] for entry in content[0]["top_logprobs"]} == {
"token_id:100",
"token_id:1000",
"token_id:5000",
sampled_token,
}
def _make_request_output(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> RequestOutput:
return RequestOutput(
request_id=f"req-{prompt_idx}",
prompt=None,
prompt_token_ids=prompt_token_ids or [1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=[4, 5],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
)
async def _generator(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text, prompt_token_ids)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
"""Echo must only echo the assistant turn, never the user's prompt.
With `add_generation_prompt` (the default) the response role is `assistant`
while the last conversation message is the user's, so there is nothing to
echo; without the role check the user prompt leaks into the answer.
"""
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
conversations = [[{"role": "user", "content": "USER PROMPT"}]]
request = BatchChatCompletionRequest(
model="test-model",
messages=conversations,
echo=True,
)
response = await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "ASSISTANT ANSWER")],
request_id="req-echo",
model_name="test-model",
all_conversations=conversations,
tokenizer=None,
request_metadata=RequestResponseMetadata(request_id="req-echo"),
)
assert response.choices[0].message.content == "ASSISTANT ANSWER"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
"""A trailing assistant turn is a real prefix and must still be echoed."""
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
conversations = [
[
{"role": "user", "content": "USER PROMPT"},
{"role": "assistant", "content": "PREFIX "},
]
]
request = BatchChatCompletionRequest(
model="test-model",
messages=conversations,
echo=True,
add_generation_prompt=False,
continue_final_message=True,
)
response = await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "ASSISTANT ANSWER")],
request_id="req-echo-2",
model_name="test-model",
all_conversations=conversations,
tokenizer=None,
request_metadata=RequestResponseMetadata(request_id="req-echo-2"),
)
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
"""Harmony must constrain only the final channel of every conversation."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.model_config = MockModelConfig()
mock_engine.errored = False
mock_engine.input_processor = MagicMock()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.renderer = _build_renderer(mock_engine.model_config)
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
online_renderer = OnlineRenderer(
model_config=mock_engine.model_config,
renderer=mock_engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="openai_gptoss",
)
serving = OpenAIServingChatBatch(
mock_engine,
models,
response_role="assistant",
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
request_logger=None,
)
request = BatchChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=[
[{"role": "user", "content": "France?"}],
[{"role": "user", "content": "Japan?"}],
],
response_format={
"type": "json_schema",
"json_schema": {"name": "city", "schema": {"type": "object"}},
},
)
await serving.create_batch_chat_completion(request)
calls = mock_engine.generate.call_args_list
assert len(calls) == 2
for call in calls:
assert call.args[1].structured_outputs.structural_tag is not None
@pytest.mark.skip_global_cleanup
def test_batch_rejects_kv_transfer_prompt_token_ids():
"""One pre-tokenized prompt cannot stand in for every conversation."""
with pytest.raises(
VLLMValidationError, match=r"parameter=kv_transfer_params\.prompt_token_ids"
):
BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_parser_receives_each_prompt_prefix() -> None:
prefixes = []
class RecordingParser:
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = prompt_token_ids
def parse(self, model_output, **kwargs):
prefixes.append(self.prompt_token_ids)
return None, model_output, None
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request = BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
)
await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
request_id="req-prefix",
model_name="test-model",
all_conversations=request.messages,
tokenizer=object(),
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
parser_cls=RecordingParser,
)
assert prefixes == [[1], [2]]
REASONING_PARSER = "kimi_k3"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_reasoning_wiring_matches_single_chat() -> None:
"""`/v1/chat/completions/batch` must hand the engine the same reasoning
wiring as `/v1/chat/completions`.
Both `reasoning_ended` and `reasoning_parser_kwargs` are carried on the
request into `EngineCoreRequest`, where
`StructuredOutputManager._get_reasoner()` rebuilds the request-local
reasoning parser from the forwarded `chat_template_kwargs` -- "so the
structured-output gate observes the same template kwargs used by the
frontend". A divergence makes a template-kwarg-driven parser fall back to
its default (`thinking` on) for a request that disabled thinking, so
`is_reasoning_end()` keeps reporting False, the structured-output gate
never opens, and the grammar is silently never applied.
Both frontends run over one and the same model and renderer, so the only
thing that can differ between the two calls is the serving code itself.
"""
mock_model_config = MockModelConfig()
shared_renderer = _build_renderer(mock_model_config)
def _build_serving(cls):
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.model_config = mock_model_config
mock_engine.errored = False
mock_engine.input_processor = MagicMock()
mock_engine.renderer = shared_renderer
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
online_renderer = OnlineRenderer(
model_config=mock_model_config,
renderer=mock_engine.renderer,
request_logger=None,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
reasoning_parser=REASONING_PARSER,
)
return mock_engine, cls(
mock_engine,
models,
response_role="assistant",
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
reasoning_parser=REASONING_PARSER,
)
conversation = [{"role": "user", "content": "What is the capital of France?"}]
chat_template_kwargs = {"enable_thinking": False}
single_engine, single_serving = _build_serving(OpenAIServingChat)
await single_serving.create_chat_completion(
ChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=conversation,
chat_template_kwargs=chat_template_kwargs,
)
)
single_call = single_engine.generate.call_args_list[-1]
batch_engine, batch_serving = _build_serving(OpenAIServingChatBatch)
await batch_serving.create_batch_chat_completion(
BatchChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=[conversation],
chat_template_kwargs=chat_template_kwargs,
)
)
batch_call = batch_engine.generate.call_args_list[-1]
# Guard against a vacuous comparison: the reference path must be wired up.
assert single_call.kwargs["reasoning_ended"] is not None
assert single_call.kwargs["reasoning_parser_kwargs"] is not None
assert batch_call.kwargs["reasoning_ended"] == single_call.kwargs["reasoning_ended"]
assert (
batch_call.kwargs["reasoning_parser_kwargs"]
== single_call.kwargs["reasoning_parser_kwargs"]
)
# ...and the forwarded kwargs must actually turn thinking off for the
# engine-side parser, i.e. the structured-output gate opens right away.
engine_chat_template_kwargs = batch_call.kwargs["reasoning_parser_kwargs"][
"chat_template_kwargs"
]
reasoner = ReasoningParserManager.get_reasoning_parser(REASONING_PARSER)(
tokenizer=batch_engine.renderer.tokenizer,
chat_template_kwargs=engine_chat_template_kwargs,
)
assert reasoner.is_reasoning_end([]) is True
@@ -1,298 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for `prompt_embeds` content parts in the Chat Completions API."""
import asyncio
import io
import openai
import pybase64 as base64
import pytest
import pytest_asyncio
import torch
from openai import BadRequestError
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm.platforms import current_platform
MODEL_NAME = "facebook/opt-125m"
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
# Matches `--dtype` in `server_args` to avoid an implicit cast in
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
# model's dtype automatically, we match here just to skip the conversion).
SERVER_DTYPE: torch.dtype = torch.bfloat16
@pytest.fixture(scope="module")
def server_args() -> list[str]:
return [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
"--chat-template",
str(CHAT_TEMPLATE),
# Prompt Embeds server args
"--enable-prompt-embeds",
]
@pytest.fixture(scope="module")
def server(server_args, request):
if current_platform.is_rocm():
# Materialize HF embeddings before the server reserves ROCm VRAM.
request.getfixturevalue("prompt_embeds_b64")
request.getfixturevalue("aligned_content_and_embeds_b64")
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
def _encode_embeds(embeds: torch.Tensor) -> str:
buf = io.BytesIO()
torch.save(embeds, buf)
return base64.b64encode(buf.getvalue()).decode("utf-8")
@pytest.fixture(scope="module")
def prompt_embeds_b64(hf_runner) -> list[str]:
"""Pre-compute embeddings for two short prompts and return as base64."""
prompts = ["Hello, my name is", "What is an LLM?"]
with hf_runner(MODEL_NAME) as hf_model:
embeddings = hf_model.get_prompt_embeddings(prompts)
# Cast to the server's dtype so `safe_load_prompt_embeds` doesn't need to
# convert on its own, the function accepts any floating-point dtype and
# will cast to the model's dtype, but matching up front skips the work.
return [_encode_embeds(e.to(SERVER_DTYPE)) for e in embeddings]
@pytest.mark.asyncio
async def test_single_prompt_embeds_part(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""A user message with one prompt_embeds part + text."""
b64 = prompt_embeds_b64[0]
chat = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": "Continue:"},
],
}
],
)
assert chat.choices[0].message.content is not None
assert len(chat.choices[0].message.content) > 0
@pytest.mark.asyncio
async def test_multiple_prompt_embeds_parts(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""Multiple prompt_embeds parts in a single message."""
b64_a, b64_b = prompt_embeds_b64
chat = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64_a},
{"type": "text", "text": " and "},
{"type": "prompt_embeds", "data": b64_b},
],
}
],
)
assert chat.choices[0].message.content is not None
assert len(chat.choices[0].message.content) > 0
@pytest.mark.asyncio
async def test_multi_message_conversation(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""prompt_embeds in both system and user messages."""
b64_sys, b64_usr = prompt_embeds_b64
chat = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "system",
"content": [
{"type": "text", "text": "You are helpful."},
{"type": "prompt_embeds", "data": b64_sys},
],
},
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64_usr},
{"type": "text", "text": "Summarize."},
],
},
],
)
assert chat.choices[0].message.content is not None
assert len(chat.choices[0].message.content) > 0
@pytest.mark.asyncio
async def test_streaming(
client: openai.AsyncOpenAI,
prompt_embeds_b64: list[str],
):
"""Streaming chat completion with prompt_embeds."""
b64 = prompt_embeds_b64[0]
# Non-streaming baseline.
baseline = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": "Continue:"},
],
}
],
)
expected = baseline.choices[0].message.content
# Streaming.
stream = await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
temperature=0.0,
stream=True,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": "Continue:"},
],
}
],
)
chunks: list[str] = []
async for chunk in stream:
delta = chunk.choices[0].delta.content
if delta:
chunks.append(delta)
assert "".join(chunks) == expected
@pytest.fixture(scope="module")
def aligned_content_and_embeds_b64(hf_runner) -> tuple[str, str]:
"""Return `(content, base64_embeds)` where the embeddings are the model's
embedding of `content` tokenized WITHOUT special tokens.
"""
content = "Hello, my name is"
with hf_runner(MODEL_NAME) as hf_model:
ids = hf_model.tokenizer(
content, add_special_tokens=False, return_tensors="pt"
).input_ids
ids = hf_model.wrap_device({"input_ids": ids})["input_ids"]
embed_layer = hf_model.model.get_input_embeddings()
embeds = embed_layer(ids).squeeze(0).to(SERVER_DTYPE).cpu()
return content, _encode_embeds(embeds)
@pytest.mark.asyncio
async def test_text_content_and_prompt_embeds_match(
client: openai.AsyncOpenAI,
aligned_content_and_embeds_b64: tuple[str, str],
):
"""Equal content in text and `prompt_embeds` should yield identical
Chat Completions output under greedy decoding.
"""
content, encoded_embeds = aligned_content_and_embeds_b64
text_resp, embeds_resp = await asyncio.gather(
client.chat.completions.create(
model=MODEL_NAME,
max_tokens=10,
temperature=0.0,
messages=[{"role": "user", "content": content}],
),
client.chat.completions.create(
model=MODEL_NAME,
max_tokens=10,
temperature=0.0,
messages=[
{
"role": "user",
"content": [{"type": "prompt_embeds", "data": encoded_embeds}],
}
],
),
)
text_out = text_resp.choices[0].message.content
embeds_out = embeds_resp.choices[0].message.content
assert text_out is not None and len(text_out) > 0
assert embeds_out is not None and len(embeds_out) > 0
assert text_out == embeds_out
@pytest.mark.asyncio
async def test_missing_data_field(
client: openai.AsyncOpenAI,
):
"""A prompt_embeds part without `data` should return a clear error."""
with pytest.raises(BadRequestError):
await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
messages=[
{
"role": "user",
"content": [{"type": "prompt_embeds"}],
}
],
)
@pytest.mark.asyncio
async def test_invalid_base64(
client: openai.AsyncOpenAI,
):
"""Invalid base64 in the `data` field should return a clear error."""
with pytest.raises(BadRequestError):
await client.chat.completions.create(
model=MODEL_NAME,
max_tokens=5,
messages=[
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": "not_valid_base64!!"},
],
}
],
)
File diff suppressed because it is too large Load Diff
@@ -1,341 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for ``thinking_token_budget`` with reasoning models.
Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
"""
import asyncio
import json
from typing import Literal
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
from vllm.platforms import current_platform
from vllm.tokenizers import get_tokenizer
MODEL_NAME = "Qwen/Qwen3-0.6B"
QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
THINK_BUDGET = 5
REASONING_START_STR = "<think>"
REASONING_END_STR = "</think>"
def _count_reasoning_decode_token_ids_between_markers(
full_token_ids: list[int],
reasoning_start_ids: list[int],
reasoning_end_ids: list[int],
) -> int | None:
"""Count decode tokens in the thinking span (after last start, before first end)."""
if not reasoning_start_ids or not reasoning_end_ids:
raise ValueError("reasoning marker token id lists must be non-empty")
def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
n = len(needle)
if n > len(haystack):
return -1
for i in range(len(haystack) - n, -1, -1):
if haystack[i : i + n] == needle:
return i
return -1
last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
if last_start < 0:
return None
pos_after_start = last_start + len(reasoning_start_ids)
end_n = len(reasoning_end_ids)
for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
if full_token_ids[j : j + end_n] == reasoning_end_ids:
return j - pos_after_start
return len(full_token_ids) - pos_after_start
@pytest.fixture(scope="module")
def server():
args = [
"--reasoning-parser",
"qwen3",
"--reasoning-config",
'{"reasoning_start_str": "<think>", "reasoning_end_str": "</think>"}',
"--max-model-len",
"2048",
"--enforce-eager",
"--gpu-memory-utilization",
"0.4",
"--no-async-scheduling",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def server_with_auto_reasoning_config():
args = [
"--reasoning-parser",
"qwen3",
"--max-model-len",
"2048",
"--enforce-eager",
"--gpu-memory-utilization",
"0.4",
"--no-async-scheduling",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def server_qwen35_fp8_mtp_tp2():
"""Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
if current_platform.device_count() < 2:
pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
if not current_platform.supports_fp8():
pytest.skip("FP8 is not supported on this platform")
spec_cfg = {
"method": "mtp",
"num_speculative_tokens": 2,
"max_model_len": 32768,
}
args = [
"--tensor-parallel-size",
"2",
"--max-model-len",
"32768",
"--speculative-config",
json.dumps(spec_cfg),
"--reasoning-parser",
"qwen3",
"--reasoning-config",
json.dumps(
{
"reasoning_start_str": REASONING_START_STR,
"reasoning_end_str": REASONING_END_STR,
}
),
]
env_dict: dict[str, str] = {}
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
# on 0,1 do not exhaust memory on the same devices as this worker.
if current_platform.device_count() >= 4:
env_dict["CUDA_VISIBLE_DEVICES"] = "2,3"
with RemoteOpenAIServer(
QWEN35_FP8_MTP_MODEL,
args,
max_wait_seconds=3000,
env_dict=env_dict,
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(request, server, server_with_auto_reasoning_config):
server_map = {
"default": server,
"auto_config": server_with_auto_reasoning_config,
}
target_server = server_map[request.param]
async with target_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
"""Test that mixed requests (some with thinking_token_budget, some without)
complete successfully without errors."""
response_with_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
extra_body={"thinking_token_budget": THINK_BUDGET},
)
response_without_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
)
msg_with = response_with_budget.choices[0].message
msg_without = response_without_budget.choices[0].message
assert msg_with.content or getattr(msg_with, "reasoning", None)
assert msg_without.content or getattr(msg_without, "reasoning", None)
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
"""Test that thinking_token_budget limits the number of reasoning tokens.
Counts reasoning decode tokens by id, which is robust to how tokens are
grouped into streamed chunks (a single chunk can carry several tokens under
async scheduling / stream_interval > 1). Counting chunks under-counts.
"""
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
prompt_token_ids: list[int] = []
decode_token_ids: list[int] = []
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
stream=True,
extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True},
)
async for chunk in stream:
if not chunk.choices:
continue
if getattr(chunk, "prompt_token_ids", None):
prompt_token_ids = list(chunk.prompt_token_ids)
delta_ids = getattr(chunk.choices[0], "token_ids", None)
if delta_ids:
decode_token_ids.extend(delta_ids)
reasoning_token_count = _count_reasoning_decode_token_ids_between_markers(
prompt_token_ids + decode_token_ids, start_ids, end_ids
)
assert reasoning_token_count is not None, "missing reasoning start marker in ids"
assert reasoning_token_count == THINK_BUDGET, (
f"reasoning tokens ({reasoning_token_count}) != "
f"thinking_token_budget ({THINK_BUDGET})"
)
@pytest.mark.asyncio
@multi_gpu_only(num_gpus=2)
@requires_fp8
async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
server_qwen35_fp8_mtp_tp2,
):
"""Concurrent chat requests: some with ``thinking_token_budget``, some without.
Exercises the scheduler / input processor under a mixed batch on the same
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
"""
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
("budget", 1),
("budget", 12),
("plain", None),
("budget", 20),
("budget", 14),
("plain", None),
("plain", None),
("budget", 12),
("plain", None),
]
tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
async def budgeted_call(expected_budget: int):
return await client.chat.completions.create(
model=QWEN35_FP8_MTP_MODEL,
messages=MESSAGES,
max_tokens=256,
stream=False,
extra_body={
"thinking_token_budget": expected_budget,
"return_token_ids": True,
},
)
async def plain_call():
return await client.chat.completions.create(
model=QWEN35_FP8_MTP_MODEL,
messages=MESSAGES,
max_tokens=256,
stream=False,
)
coros = []
for row in _batch_spec:
if row[0] == "budget":
b = row[1]
assert isinstance(b, int)
coros.append(budgeted_call(b))
else:
coros.append(plain_call())
results = await asyncio.gather(*coros)
for i, (response, (kind, expected_budget)) in enumerate(
zip(results, _batch_spec, strict=True)
):
msg = response.choices[0].message
assert msg.content or getattr(msg, "reasoning", None), (
f"index {i} ({kind}): empty message"
)
if kind == "budget":
assert expected_budget is not None
assert response.prompt_token_ids is not None
assert response.choices[0].token_ids is not None
full_ids = list(response.prompt_token_ids) + list(
response.choices[0].token_ids
)
n_reason = _count_reasoning_decode_token_ids_between_markers(
full_ids, start_ids, end_ids
)
assert n_reason is not None, f"index {i}: missing reasoning start in ids"
assert n_reason == expected_budget, (
f"index {i}: reasoning decode token ids ({n_reason}) != "
f"thinking_token_budget ({expected_budget})"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_streaming_with_thinking_disabled_stays_in_content(
client: openai.AsyncOpenAI,
):
request_kwargs = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": "Which is larger, 4 or 12?"
" Output exactly one token: 4 or 12.",
}
],
"max_tokens": 16,
"temperature": 0.0,
"extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
}
response = await client.chat.completions.create(**request_kwargs)
message = response.choices[0].message
assert message.content is not None and message.content.strip() != ""
assert getattr(message, "reasoning", None) in (None, "")
stream = await client.chat.completions.create(
**request_kwargs,
stream=True,
)
content_chunks = []
reasoning_chunks = []
async for chunk in stream:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if getattr(delta, "content", None):
content_chunks.append(delta.content)
if getattr(delta, "reasoning", None):
reasoning_chunks.append(delta.reasoning)
assert "".join(content_chunks).strip() != ""
assert reasoning_chunks == []
@@ -1,830 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import regex as re
from openai import BadRequestError
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.sampling_params import SamplingParams
from vllm.tokenizers import get_tokenizer
# any model with a chat template should work here
MODEL_NAME = "facebook/opt-125m"
@pytest.fixture(scope="module")
def default_server_args():
return [
"--dtype",
"float32",
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
"--enable-prompt-tokens-details",
"--no-enable-prefix-caching",
]
@pytest.fixture(scope="module")
def server(default_server_args):
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) -> None:
completion = await client.completions.create(
model=model_name, prompt="Hello, my name is", max_tokens=5, temperature=0.0
)
assert completion.id is not None
assert completion.choices is not None and len(completion.choices) == 1
choice = completion.choices[0]
assert len(choice.text) >= 5
assert choice.finish_reason == "length"
assert completion.usage is not None
assert completion.usage.completion_tokens == 5
assert completion.usage.prompt_tokens == 6
assert completion.usage.total_tokens == 11
assert completion.usage.prompt_tokens_details is not None
assert completion.usage.prompt_tokens_details.cached_tokens == 0
# test using token IDs
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
)
assert len(completion.choices[0].text) >= 1
assert completion.choices[0].prompt_logprobs is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_completion_truncation_side_controls_prompt_truncation(
client: openai.AsyncOpenAI, model_name: str
) -> None:
prompt_token_ids = list(range(8))
right_completion = await client.completions.create(
model=model_name,
prompt=prompt_token_ids,
max_tokens=1,
temperature=0.0,
extra_body={
"return_token_ids": True,
"truncate_prompt_tokens": 4,
"truncation_side": "right",
},
)
assert right_completion.choices[0].prompt_token_ids == prompt_token_ids[:4]
left_completion = await client.completions.create(
model=model_name,
prompt=prompt_token_ids,
max_tokens=1,
temperature=0.0,
extra_body={
"return_token_ids": True,
"truncate_prompt_tokens": 4,
"truncation_side": "left",
},
)
assert left_completion.choices[0].prompt_token_ids == prompt_token_ids[-4:]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
# test using token IDs
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
logprobs=None,
)
choice = completion.choices[0]
assert choice.logprobs is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
# test using token IDs
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
logprobs=0,
)
choice = completion.choices[0]
assert choice.logprobs is not None
assert choice.logprobs.token_logprobs is not None
assert choice.logprobs.top_logprobs is not None
assert len(choice.logprobs.top_logprobs[0]) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
# test using token IDs
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
logprobs=5,
)
choice = completion.choices[0]
assert choice.logprobs is not None
assert choice.logprobs.token_logprobs is not None
assert choice.logprobs.top_logprobs is not None
assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
@pytest.mark.asyncio
async def test_logprob_token_ids(client: openai.AsyncOpenAI):
completion = await client.completions.create(
model=MODEL_NAME,
prompt="Hello",
max_tokens=1,
temperature=0.0,
logprobs=5,
extra_body={
"logprob_token_ids": [5000],
"allowed_token_ids": [42],
"return_tokens_as_token_ids": True,
},
)
choice = completion.choices[0]
assert choice.logprobs is not None
assert choice.logprobs.tokens == ["token_id:42"]
assert choice.logprobs.top_logprobs is not None
assert set(choice.logprobs.top_logprobs[0]) == {
"token_id:42",
"token_id:5000",
}
@pytest.mark.asyncio
async def test_logprob_token_ids_stream(client: openai.AsyncOpenAI):
stream = await client.completions.create(
model=MODEL_NAME,
prompt="Hello",
max_tokens=1,
temperature=0.0,
logprobs=5,
stream=True,
extra_body={
"logprob_token_ids": [5000],
"allowed_token_ids": [42],
"return_tokens_as_token_ids": True,
},
)
returned_top_logprobs: list[dict[str, float]] = []
async for chunk in stream:
logprobs = chunk.choices[0].logprobs
if logprobs is not None and logprobs.top_logprobs is not None:
returned_top_logprobs.extend(
top_logprobs
for top_logprobs in logprobs.top_logprobs
if top_logprobs is not None
)
assert len(returned_top_logprobs) == 1
assert set(returned_top_logprobs[0]) == {"token_id:42", "token_id:5000"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_too_many_completion_logprobs(
client: openai.AsyncOpenAI, model_name: str
) -> None:
with pytest.raises(
(openai.BadRequestError, openai.APIError)
): # test using token IDs
await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
# vLLM has higher default max_logprobs (20 instead of 5) to support
# both Completion API and Chat Completion API
logprobs=21,
)
...
with pytest.raises(
(openai.BadRequestError, openai.APIError)
): # test using token IDs
stream = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
# vLLM has higher default max_logprobs (20 instead of 5) to support
# both Completion API and Chat Completion API
logprobs=30,
stream=True,
)
async for chunk in stream:
...
# the server should still work afterwards
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
)
assert len(completion.choices[0].text) >= 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name, prompt_logprobs",
[(MODEL_NAME, -1), (MODEL_NAME, 0), (MODEL_NAME, 1), (MODEL_NAME, None)],
)
async def test_prompt_logprobs_completion(
client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: int | None
):
params: dict = {
"prompt": ["A robot may not injure another robot", "My name is"],
"model": model_name,
}
if prompt_logprobs is not None:
params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
if prompt_logprobs is not None and prompt_logprobs < 0:
with pytest.raises(BadRequestError):
await client.completions.create(**params)
else:
completion = await client.completions.create(**params)
if prompt_logprobs is not None:
assert completion.choices[0].prompt_logprobs is not None
assert len(completion.choices[0].prompt_logprobs) > 0
assert completion.choices[1].prompt_logprobs is not None
assert len(completion.choices[1].prompt_logprobs) > 0
else:
assert completion.choices[0].prompt_logprobs is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_completion_streaming(
client: openai.AsyncOpenAI, model_name: str
) -> None:
prompt = "What is an LLM?"
single_completion = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
)
single_output = single_completion.choices[0].text
stream = await client.completions.create(
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
)
chunks: list[str] = []
finish_reason_count = 0
async for chunk in stream:
chunks.append(chunk.choices[0].text)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
# finish reason should only return in last block
assert finish_reason_count == 1
assert chunk.choices[0].finish_reason == "length"
assert chunk.choices[0].text
assert "".join(chunks) == single_output
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str):
"""Parallel sampling without streaming.
A single request output contains a list of completions.
"""
prompt = "What is an LLM?"
n = 3
max_tokens = 50
# High temperature to maximize chance of unique completions.
completion = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=max_tokens,
n=n,
temperature=1.0,
stream=False,
logprobs=0,
seed=42,
)
# Assert `n` completions
num_completions = len(completion.choices)
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
completion_repeats: dict[str, int] = {}
for idx, choice in enumerate(completion.choices):
# Assert correct completion index & some finish reason.
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
assert choice.finish_reason is not None, "None finish_reason is invalid."
text = choice.text
completion_repeats[text] = completion_repeats.get(text, 0) + 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
repeats = {txt: num for (txt, num) in completion_repeats.items() if num > 1}
raise AssertionError(
f"Expected {n} unique completions, got {num_unique}; repeats: {repeats}."
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
"""Streaming for parallel sampling.
The tokens from multiple samples, are flattened into a single stream,
with an index to indicate which sample the token belongs to.
"""
prompt = "What is an LLM?"
n = 3
max_tokens = 50
stream = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=max_tokens,
n=n,
temperature=1.0,
stream=True,
seed=42,
)
chunks: list[list[str]] = [[] for _ in range(n)]
finish_reason_count = 0
async for chunk in stream:
index = chunk.choices[0].index
text = chunk.choices[0].text
chunks[index].append(text)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
# Assert `n` completions with correct finish reasons
assert finish_reason_count == n, (
f"Expected {n} completions with valid indices and finish_reason."
)
completion_repeats: dict[str, int] = {}
for chunk in chunks:
chunk_len = len(chunk)
# Assert correct number of completion tokens
assert chunk_len <= max_tokens, (
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
)
text = "".join(chunk)
completion_repeats[text] = completion_repeats.get(text, 0) + 1
print(text)
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
repeats = {txt: num for (txt, num) in completion_repeats.items() if num > 1}
raise AssertionError(
f"{num_unique} unique completions, expected {n}; repeats: {repeats}"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_completion_stream_options(client: openai.AsyncOpenAI, model_name: str):
prompt = "What is the capital of France?"
# Test stream=True, stream_options=
# {"include_usage": False, "continuous_usage_stats": False}
stream = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=True,
stream_options={
"include_usage": False,
"continuous_usage_stats": False,
},
)
async for chunk in stream:
assert chunk.usage is None
# Test stream=True, stream_options=
# {"include_usage": False, "continuous_usage_stats": True}
stream = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=True,
stream_options={
"include_usage": False,
"continuous_usage_stats": True,
},
)
async for chunk in stream:
assert chunk.usage is None
# Test stream=True, stream_options=
# {"include_usage": True, "continuous_usage_stats": False}
stream = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=True,
stream_options={
"include_usage": True,
"continuous_usage_stats": False,
},
)
async for chunk in stream:
if chunk.choices[0].finish_reason is None:
assert chunk.usage is None
else:
assert chunk.usage is None
final_chunk = await anext(stream)
assert final_chunk.usage is not None
assert final_chunk.usage.prompt_tokens > 0
assert final_chunk.usage.completion_tokens > 0
assert final_chunk.usage.total_tokens == (
final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
)
assert final_chunk.choices == []
# Test stream=True, stream_options=
# {"include_usage": True, "continuous_usage_stats": True}
stream = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=True,
stream_options={
"include_usage": True,
"continuous_usage_stats": True,
},
)
async for chunk in stream:
assert chunk.usage is not None
assert chunk.usage.prompt_tokens > 0
assert chunk.usage.completion_tokens > 0
assert chunk.usage.total_tokens == (
chunk.usage.prompt_tokens + chunk.usage.completion_tokens
)
if chunk.choices[0].finish_reason is not None:
final_chunk = await anext(stream)
assert final_chunk.usage is not None
assert final_chunk.usage.prompt_tokens > 0
assert final_chunk.usage.completion_tokens > 0
assert final_chunk.usage.total_tokens == (
final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
)
assert final_chunk.choices == []
# Test stream=True, stream_options={}
stream = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=True,
stream_options={},
)
async for chunk in stream:
assert chunk.usage is None
# Test stream=False, stream_options=
# {"include_usage": None}
with pytest.raises(BadRequestError):
await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=False,
stream_options={"include_usage": None},
)
# Test stream=False, stream_options=
# {"include_usage": True}
with pytest.raises(BadRequestError):
await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=False,
stream_options={"include_usage": True},
)
# Test stream=False, stream_options=
# {"continuous_usage_stats": None}
with pytest.raises(BadRequestError):
await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=False,
stream_options={"continuous_usage_stats": None},
)
# Test stream=False, stream_options=
# {"continuous_usage_stats": True}
with pytest.raises(BadRequestError):
await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
stream=False,
stream_options={"continuous_usage_stats": True},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
# test both text and token IDs
for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
# test simple list
batch = await client.completions.create(
model=model_name,
prompt=prompts,
max_tokens=5,
temperature=0.0,
)
assert len(batch.choices) == 2
assert batch.choices[0].text == batch.choices[1].text
# test n = 2
batch = await client.completions.create(
model=model_name,
prompt=prompts,
n=2,
max_tokens=5,
temperature=0.0,
extra_body=dict(
# NOTE: this has to be true for n > 1 in vLLM, but
# not necessary for official client.
use_beam_search=True
),
)
assert len(batch.choices) == 4
assert batch.choices[0].text != batch.choices[1].text, (
"beam search should be different"
)
assert batch.choices[0].text == batch.choices[2].text, (
"two copies of the same prompt should be the same"
)
assert batch.choices[1].text == batch.choices[3].text, (
"two copies of the same prompt should be the same"
)
# test streaming
batch = await client.completions.create(
model=model_name,
prompt=prompts,
max_tokens=5,
temperature=0.0,
stream=True,
)
texts = [""] * 2
async for chunk in batch:
assert len(chunk.choices) == 1
choice = chunk.choices[0]
texts[choice.index] += choice.text
assert texts[0] == texts[1]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
@pytest.mark.parametrize("logprobs_arg", [1, 0])
async def test_echo_logprob_completion(
client: openai.AsyncOpenAI, model_name: str, logprobs_arg: int
):
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
# test using text and token IDs
for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
completion = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
echo=True,
logprobs=logprobs_arg,
)
prompt_text = tokenizer.decode(prompt) if isinstance(prompt, list) else prompt
assert re.search(r"^" + prompt_text, completion.choices[0].text)
logprobs = completion.choices[0].logprobs
assert logprobs is not None
assert len(logprobs.text_offset) > 5
assert len(logprobs.token_logprobs) > 5 and logprobs.token_logprobs[0] is None
assert len(logprobs.top_logprobs) > 5 and logprobs.top_logprobs[0] is None
for top_logprobs in logprobs.top_logprobs[1:]:
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
assert len(logprobs.tokens) > 5
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_invalid_json_schema(client: openai.AsyncOpenAI, model_name: str) -> None:
invalid_json_schema = {
"$defs": {
"CarType": {
"enum": ["sedan", "SUV", "Truck", "Coupe"],
"title": "CarType",
"type": "string",
}
},
"properties": {
"brand": {"title": "Brand", "type": "string"},
"model": {"title": "Model", "type": "string"},
"car_type": {"$ref": "#/$defs/CarType"},
"foo": "bar",
},
"required": ["brand", "model", "car_type"],
"title": "CarDescription",
"type": "object",
}
prompt = (
"Generate a JSON with the brand, model and car_type of"
"the most iconic car from the 90's"
)
with pytest.raises((openai.BadRequestError, openai.APIError)):
await client.completions.create(
model=model_name,
prompt=prompt,
extra_body={"structured_outputs": {"json": invalid_json_schema}},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_invalid_regex(client: openai.AsyncOpenAI, model_name: str):
prompt = (
"Generate an email address for Alan Turing, who works in Enigma."
"End in .com and new line. Example result:"
"alan.turing@enigma.com\n"
)
with pytest.raises((openai.BadRequestError, openai.APIError)):
await client.completions.create(
model=model_name,
prompt=prompt,
extra_body={"structured_outputs": {"regex": r"[.*"}, "stop": ["\n"]},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_invalid_grammar(client: openai.AsyncOpenAI, model_name: str):
invalid_simplified_sql_grammar = """
root ::= select_statementinvalidsyntax
select_statement ::= "SELECT " column " from " table " where " condition
column ::= "col_1 " | "col_2 "
table ::= "table_1 " | "table_2 "
condition ::= column "= " number
number ::= "1 " | "2 "
"""
prompt = (
"Generate an SQL query to show the 'username' and 'email'"
"from the 'users' table."
)
with pytest.raises((openai.BadRequestError, openai.APIError)):
await client.completions.create(
model=model_name,
prompt=prompt,
extra_body={
"structured_outputs": {"grammar": invalid_simplified_sql_grammar}
},
)
# Unit tests for bad_words in CompletionRequest.to_sampling_params()
def test_completion_request_bad_words_to_sampling_params():
"""bad_words should be forwarded to SamplingParams (parity with chat)."""
request = CompletionRequest(
model="test-model",
prompt="Hello",
bad_words=["foo", "bar"],
max_tokens=10,
)
sampling_params = request.to_sampling_params(
max_tokens=10,
default_sampling_params={},
)
assert isinstance(sampling_params, SamplingParams)
assert sampling_params.bad_words == ["foo", "bar"]
def test_completion_request_bad_words_default_empty():
"""bad_words defaults to an empty list, matching the chat endpoint."""
request = CompletionRequest(
model="test-model",
prompt="Hello",
max_tokens=10,
)
assert request.bad_words == []
sampling_params = request.to_sampling_params(
max_tokens=10,
default_sampling_params={},
)
assert sampling_params.bad_words == []
def test_completion_request_forwards_routed_experts_prompt_start():
request = CompletionRequest(
model="test-model",
prompt="Hello",
max_tokens=10,
routed_experts_prompt_start=3,
)
sampling_params = request.to_sampling_params(
max_tokens=10,
default_sampling_params={},
)
assert sampling_params.routed_experts_prompt_start == 3
File diff suppressed because it is too large Load Diff
@@ -1,103 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import os
import tempfile
import openai
import pytest
import pytest_asyncio
import torch.cuda
from tests.utils import RemoteOpenAIServer
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.model_loader.tensorizer import (
TensorizerConfig,
tensorize_vllm_model,
)
from vllm.platforms import current_platform
MODEL_NAME = "unsloth/llama-3.2-1b-Instruct"
def _cleanup():
gc.collect()
torch.accelerator.empty_cache()
@pytest.fixture(autouse=True)
def cleanup():
_cleanup()
@pytest.fixture(scope="module")
def tmp_dir():
with tempfile.TemporaryDirectory() as path:
yield path
@pytest.fixture(scope="module")
def model_uri(tmp_dir):
yield f"{tmp_dir}/model.tensors"
@pytest.fixture(scope="module")
def tensorize_model(model_uri):
tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri)
args = EngineArgs(model=MODEL_NAME)
tensorize_vllm_model(args, tensorizer_config)
# Manually invoke a _cleanup() here, as the cleanup()
# fixture won't be guaranteed to be called after this
# when this fixture is used for a test
_cleanup()
yield
@pytest.fixture(scope="module")
def server(model_uri, tensorize_model):
# In this case, model_uri is a directory with a model.tensors
# file and all necessary model artifacts, particularly a
# HF `config.json` file. In this case, Tensorizer can infer the
# `TensorizerConfig` so --model-loader-extra-config can be completely
# omitted.
## Start OpenAI API server
args = [
"--load-format",
"tensorizer",
"--served-model-name",
MODEL_NAME,
]
if current_platform.is_rocm():
args += ["--attention-backend", "TRITON_ATTN"]
model_dir = os.path.dirname(model_uri)
with RemoteOpenAIServer(model_dir, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
_cleanup()
completion = await client.completions.create(
model=model_name, prompt="Hello, my name is", max_tokens=5, temperature=0.0
)
assert completion.id is not None
assert completion.choices is not None and len(completion.choices) == 1
assert completion.model == MODEL_NAME
assert len(completion.choices) == 1
assert len(completion.choices[0].text) >= 5
assert completion.choices[0].finish_reason == "length"
assert completion.usage == openai.types.CompletionUsage(
completion_tokens=5, prompt_tokens=6, total_tokens=11
)
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import tempfile
import pytest
from tests.utils import RemoteOpenAIServer
from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
from vllm.tokenizers import get_tokenizer
MODEL_NAME = "Qwen/Qwen3-0.6B"
MODEL_PATH = os.path.join(tempfile.gettempdir(), "qwen3_06b")
@pytest.fixture(scope="module")
def server():
global MODEL_PATH
MODEL_PATH = download_weights_from_hf(
MODEL_NAME,
allow_patterns=["*"],
cache_dir=MODEL_PATH,
ignore_patterns=["tokenizer*", "vocab*", "*.safetensors"],
)
args = [
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
"--skip-tokenizer-init",
"--load-format",
"dummy",
]
with RemoteOpenAIServer(MODEL_PATH, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_token_in_token_out_and_logprobs(server):
"""Test token-in-token-out and token_ids align with prompt_logprobs
& logprobs when return_tokens_as_token_ids is enabled.
"""
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
text = "Hello, world! How are you today?"
token_ids = tokenizer.encode(text)
async with server.get_async_client() as client:
# Test with both return_token_ids and return_tokens_as_token_ids enabled
completion = await client.completions.create(
model=MODEL_PATH,
prompt=token_ids,
max_tokens=20,
temperature=0,
echo=True,
extra_body={
"return_token_ids": True,
},
)
# Verify all fields are present
assert (
completion.choices[0].token_ids is not None
and 0 < len(completion.choices[0].token_ids) <= 20
)
assert completion.choices[0].prompt_token_ids is not None
# Decode prompt tokens
if completion.choices[0].prompt_token_ids:
prompt_text = tokenizer.decode(completion.choices[0].prompt_token_ids)
# The decoded prompt should match or close to original prompt
assert prompt_text == text
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file test accuracy of the vLLM server via LMEval.
It uses local-completions, which interacts with vLLM
through the OAI API with N concurrent connections.
This simulates real work usage of the API and makes
sure that the zmq frontend mp RPC message passing and
AsyncLLMEngine are working correctly.
"""
import lm_eval
from vllm.platforms import current_platform
from ....utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
NUM_CONCURRENT = 500
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.03
EXPECTED_VALUE = 0.54
DEFAULT_ARGS = ["--max-model-len", "4096"]
MORE_ARGS_LIST = [
[], # Default
["--enable-chunked-prefill"], # Chunked
]
MAX_WAIT_SECONDS = None
if current_platform.is_tpu():
MORE_ARGS_LIST = [
[], # Default
]
MAX_WAIT_SECONDS = 600
def run_test(more_args):
"""Run the end to end accuracy test."""
args = list(DEFAULT_ARGS)
args.extend(more_args)
print(f"Running with: {args}")
with RemoteOpenAIServer(
MODEL_NAME, args, max_wait_seconds=MAX_WAIT_SECONDS
) as remote_server:
url = f"{remote_server.url_for('v1')}/completions"
model_args = (
f"model={MODEL_NAME},"
f"base_url={url},"
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
)
results = lm_eval.simple_evaluate(
model="local-completions",
model_args=model_args,
tasks=TASK,
)
measured_value = results["results"][TASK][FILTER]
assert (
measured_value - RTOL < EXPECTED_VALUE
and measured_value + RTOL > EXPECTED_VALUE
), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
def test_lm_eval_accuracy_v1_engine():
"""Run with the V1 Engine."""
more_args = []
# Limit compilation time for V1 on TPU
# Avoid OOM on XPU
if current_platform.is_tpu() or current_platform.is_xpu():
more_args = ["--max-num-seqs", "64"]
run_test(more_args)
File diff suppressed because it is too large Load Diff
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from argparse import Namespace
from typing import Final
import pytest
import schemathesis
from fastapi.testclient import TestClient
from hypothesis import HealthCheck, settings
from jsonschema import Draft202012Validator
from schemathesis import GenerationMode
from schemathesis.checks import not_a_server_error
from schemathesis.config import (
ChecksConfig,
CoveragePhaseConfig,
GenerationConfig,
PhasesConfig,
PositiveDataAcceptanceConfig,
ProjectConfig,
ProjectsConfig,
SchemathesisConfig,
)
from vllm.platforms import current_platform
from ...utils import RemoteOpenAIServer
MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
MAXIMUM_IMAGES = 2
_ROCM_TIMEOUT_MULTIPLIER = 3 if current_platform.is_rocm() else 1
DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 * _ROCM_TIMEOUT_MULTIPLIER
LONG_TIMEOUT_SECONDS: Final[int] = 60 * _ROCM_TIMEOUT_MULTIPLIER
def test_openapi_exposes_completion_response_schemas() -> None:
"""The published 200 contract of both completion APIs must reference the
real response models, for the JSON body and for each SSE data chunk.
Uses the production app: FastAPI renames models used as both input and
output (-Input/-Output), which a router-only app cannot show.
"""
from vllm.entrypoints.launchers.app import build_app
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
CompletionStreamResponse,
)
args = Namespace(
disable_fastapi_docs=False,
enable_offline_docs=False,
root_path="",
allowed_origins=["*"],
allow_credentials=False,
allowed_methods=["*"],
allowed_headers=["*"],
api_key=None,
enable_request_id_headers=False,
enable_fault_tolerance=False,
middleware=[],
log_error_stack=False,
)
# No lifespan: schema generation does not need an inference engine.
client = TestClient(build_app(args, supported_tasks=("generate",)))
document = client.get("/openapi.json").json()
message = {"role": "assistant", "content": "hi"}
for path, unary, unary_choice, chunk, chunk_choice in (
(
"/v1/chat/completions",
ChatCompletionResponse,
{"index": 0, "message": message},
ChatCompletionStreamResponse,
{"index": 0, "delta": {"content": "hi"}},
),
(
"/v1/completions",
CompletionResponse,
{"index": 0, "text": "hi"},
CompletionStreamResponse,
{"index": 0, "text": "hi"},
),
):
content = document["paths"][path]["post"]["responses"]["200"]["content"]
data = content["text/event-stream"]["schema"]["properties"]["data"]
assert data["anyOf"][0] == {"const": "[DONE]"}
for schema, model, choice in (
(content["application/json"]["schema"], unary, unary_choice),
(data["anyOf"][1]["contentSchema"], chunk, chunk_choice),
):
assert schema == {"$ref": f"#/components/schemas/{model.__name__}"}
validator = Draft202012Validator(
{**schema, "components": document["components"]}
)
payload = model(model="m", choices=[choice], usage={}).model_dump(
mode="json"
)
validator.validate(payload)
payload["choices"] = "not an array"
assert not validator.is_valid(payload)
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"generate",
"--max-model-len",
"2048",
"--max-num-seqs",
"5",
"--enforce-eager",
"--trust-remote-code",
"--limit-mm-per-prompt",
json.dumps({"image": MAXIMUM_IMAGES}),
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def get_schema(server):
# avoid generating null (\x00) bytes in strings during test case generation
return schemathesis.openapi.from_url(
f"{server.url_root}/openapi.json",
config=SchemathesisConfig(
projects=ProjectsConfig(
default=ProjectConfig(
generation=GenerationConfig(
allow_x00=False,
modes=[GenerationMode.POSITIVE],
),
checks=ChecksConfig(
positive_data_acceptance=PositiveDataAcceptanceConfig(
enabled=False,
),
),
phases=PhasesConfig(
coverage=CoveragePhaseConfig(enabled=False),
),
),
),
),
)
schema = schemathesis.pytest.from_fixture("get_schema")
@schemathesis.hook
def before_generate_case(context: schemathesis.HookContext, strategy):
op = context.operation
assert op is not None
def no_invalid_types(case: schemathesis.Case):
"""Skips tool_calls with `"type": "custom"` which schemathesis incorrectly
generates instead of the valid `"type": "function"`.
Example test case that is skipped:
curl -X POST -H 'Content-Type: application/json' \
-d '{"messages": [{"role": "assistant", "tool_calls": [{"custom": {"input": "", "name": ""}, "id": "", "type": "custom"}]}]}' \
http://localhost:8000/v1/chat/completions
""" # noqa: E501
if (
hasattr(case, "body")
and isinstance(case.body, dict)
and "messages" in case.body
and isinstance(case.body["messages"], list)
and len(case.body["messages"]) > 0
):
for message in case.body["messages"]:
if not isinstance(message, dict):
continue
tool_calls = message.get("tool_calls", [])
if isinstance(tool_calls, list):
for tool_call in tool_calls:
if isinstance(tool_call, dict):
if tool_call.get("type") != "function":
return False
if "custom" in tool_call:
return False
return True
return strategy.filter(no_invalid_types)
@schema.parametrize()
@settings(
deadline=LONG_TIMEOUT_SECONDS * 1000,
max_examples=50,
# Under CI's derandomized hypothesis seed, the schemathesis strategy
# for /v1/chat/completions/batch's nested-message body, combined with
# the no_invalid_types filter (notably the grammar=="" rule), exceeds
# the default filtered-vs-good ratio. The filter is intentional, so
# suppress the health check rather than drop the filter — dropping it
# exposes pre-existing server bugs out of scope here.
# The same nested schema can also trip Hypothesis' entropy budget while
# generating large-but-valid request bodies before vLLM is called.
suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.data_too_large],
)
def test_openapi_stateless(case: schemathesis.Case):
key = (
case.operation.method.upper(),
case.operation.path,
)
if case.operation.path.startswith("/v1/responses"):
# Skip responses API as it is meant to be stateful.
return
# Skip weight transfer endpoints as they require special setup
# (weight_transfer_config) and are meant to be stateful.
if case.operation.path in (
"/init_weight_transfer_engine",
"/start_weight_update",
"/start_draft_weight_update",
"/update_weights",
"/finish_weight_update",
"/update_weight_version",
):
return
timeout = {
# requires a longer timeout
("POST", "/v1/chat/completions"): LONG_TIMEOUT_SECONDS,
("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS,
("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS,
("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS,
("POST", "/inference/v1/generate"): LONG_TIMEOUT_SECONDS,
}.get(key, DEFAULT_TIMEOUT_SECONDS)
# No need to verify SSL certificate for localhost
unsupported_decisions = case.operation.path in {
"/v1/systemone",
"/v1/decisions",
}
response = case.call_and_validate(
verify=False,
timeout=timeout,
headers={"Content-Type": "application/json"},
excluded_checks=[not_a_server_error] if unsupported_decisions else None,
)
if unsupported_decisions:
# SmolVLM does not support either decision endpoint.
assert response.status_code < 500 or response.status_code == 501
@@ -1,368 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.tokenizers import get_tokenizer
from ...utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enable-auto-tool-choice",
"--tool-call-parser",
"hermes",
"--enforce-eager",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize("return_token_ids", [True, False, None])
async def test_basic_completion_with_emoji(server, return_token_ids: bool | None):
"""Test basic completion with emoji to verify token_ids field."""
extra_body = None
if return_token_ids is not None:
extra_body = {"return_token_ids": return_token_ids}
async with server.get_async_client() as client:
# Test with return_token_ids enabled
completion = await client.completions.create(
model=MODEL_NAME,
prompt="Complete this sentence with emojis: I love coding 🚀",
max_tokens=10,
temperature=0,
logprobs=1,
extra_body=extra_body,
)
# Check the raw response to see the structure
completion_dict = completion.model_dump()
# Verify prompt_token_ids field is present in the completion response
assert "prompt_token_ids" in completion_dict["choices"][0]
if not return_token_ids:
# If return_token_ids is False, token_ids should not be present
assert completion_dict["choices"][0].get("token_ids") is None
assert completion_dict["choices"][0].get("prompt_token_ids") is None
# Skip further checks
return
assert isinstance(completion.choices[0].prompt_token_ids, list)
# Check against the expected prompt token IDs
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
encoded_tokens = tokenizer.encode(
"Complete this sentence with emojis: I love coding 🚀"
)
# Check that encoded_tokens is a subsequence of prompt_token_ids
assert any(
completion.choices[0].prompt_token_ids[i : i + len(encoded_tokens)]
== encoded_tokens
for i in range(
len(completion.choices[0].prompt_token_ids) - len(encoded_tokens) + 1
)
)
# Verify token_ids field is present in the choice
assert completion.choices[0].token_ids is not None
assert isinstance(completion.choices[0].token_ids, list)
assert len(completion.choices[0].token_ids) > 0
# Verify decoding works correctly
decoded_text = tokenizer.decode(completion.choices[0].token_ids)
# The decoded text should contain a <|im_end|> at the end
assert decoded_text.startswith(completion.choices[0].text)
# Test without return_token_ids (should be None)
completion_without = await client.completions.create(
model=MODEL_NAME,
prompt="Complete this sentence with emojis: I love coding 🚀",
max_tokens=10,
temperature=0,
logprobs=1,
extra_body={"return_token_ids": False},
)
completion_without_dict = completion_without.model_dump()
assert completion_without_dict["choices"][0].get("token_ids") is None
assert completion_without_dict.get("prompt_token_ids") is None
@pytest.mark.asyncio
async def test_chat_completion_with_tool_use(server):
"""Test chat completion with tool use (get_weather function)."""
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
"description": "The unit of temperature",
},
},
"required": ["location"],
},
},
}
]
async with server.get_async_client() as client:
# Test with return_token_ids enabled
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What's the weather like in Paris?"},
],
tools=tools,
tool_choice="auto",
max_tokens=100,
temperature=0,
logprobs=True,
extra_body={"return_token_ids": True},
)
# Verify token_ids field is present in choices
assert response.choices[0].token_ids is not None
assert isinstance(response.choices[0].token_ids, list)
# Verify prompt_token_ids field is present
assert response.prompt_token_ids is not None
assert isinstance(response.prompt_token_ids, list)
# Verify the prompt texts and response texts
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
prompt_text = tokenizer.decode(response.prompt_token_ids)
assert prompt_text.startswith(
"<|im_start|>system\nYou are a helpful assistant."
)
assert prompt_text.endswith(
"What's the weather like in Paris?<|im_end|>\n<|im_start|>assistant\n"
)
response_text = tokenizer.decode(response.choices[0].token_ids)
assert response_text.startswith('<tool_call>\n{"name": "get_weather"')
assert response_text.endswith("</tool_call><|im_end|>")
# If tool call was made, verify the response structure
if response.choices[0].message.tool_calls:
assert len(response.choices[0].message.tool_calls) > 0
tool_call = response.choices[0].message.tool_calls[0]
assert tool_call.function.name == "get_weather"
# Test without return_token_ids
response_without = await client.chat.completions.create(
model=MODEL_NAME,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What's the weather like in Paris?"},
],
tools=tools,
tool_choice="auto",
max_tokens=100,
temperature=0,
logprobs=True,
extra_body={"return_token_ids": False},
)
assert response_without.choices[0].token_ids is None
assert response_without.prompt_token_ids is None
@pytest.mark.asyncio
async def test_comparison_with_prompt_logprobs_and_logprobs(server):
"""Test that token_ids align with prompt_logprobs and
logprobs when return_tokens_as_token_ids is enabled.
"""
async with server.get_async_client() as client:
# Test with both return_token_ids and return_tokens_as_token_ids enabled
completion = await client.completions.create(
model=MODEL_NAME,
prompt="Hello, world! How are you today?",
max_tokens=20,
temperature=0,
echo=True,
logprobs=1,
extra_body={
"return_token_ids": True,
"return_tokens_as_token_ids": True,
"prompt_logprobs": 1,
},
)
# Verify all fields are present
assert completion.choices[0].token_ids is not None
assert completion.choices[0].prompt_token_ids is not None
assert completion.choices[0].prompt_logprobs is not None
assert completion.choices[0].logprobs is not None
# Extract token IDs from logprobs
# (when return_tokens_as_token_ids is True)
logprobs_token_ids = []
for token_str in completion.choices[0].logprobs.tokens:
# Token format is "token_id:12345" when
# return_tokens_as_token_ids is True
if token_str.startswith("token_id:"):
token_id = int(token_str.removeprefix("token_id:"))
logprobs_token_ids.append(token_id)
# When echo=True, the logprobs include both prompt and response tokens
# The token_ids field should match the suffix of response portion
# The prompt_token_ids should match the prompt portion
assert len(completion.choices[0].token_ids) < len(logprobs_token_ids)
response_token_ids_length = len(completion.choices[0].token_ids)
assert (
logprobs_token_ids[-response_token_ids_length:]
== completion.choices[0].token_ids
)
# Verify tokenizer consistency
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
# Decode prompt tokens
if completion.choices[0].prompt_token_ids:
prompt_text = tokenizer.decode(completion.choices[0].prompt_token_ids)
# The decoded prompt should match or close to original prompt
assert "Hello, world" in prompt_text
# Decode response tokens
if completion.choices[0].token_ids:
response_text = tokenizer.decode(completion.choices[0].token_ids)
assert completion.choices[0].text.endswith(response_text)
# Test streaming mode
stream = await client.completions.create(
model=MODEL_NAME,
prompt="Tell me a short fact about Python:",
max_tokens=30,
temperature=0,
stream=True,
echo=False,
logprobs=1,
extra_body={"return_token_ids": True, "return_tokens_as_token_ids": True},
)
# Collect streamed tokens
streamed_prompt_token_ids = []
streamed_token_ids = []
streamed_logprob_token_ids = []
first_chunk = True
async for chunk in stream:
for token_str in chunk.choices[0].logprobs.tokens:
# Token format is "token_id:12345" when
# return_tokens_as_token_ids is True
if token_str.startswith("token_id:"):
token_id = int(token_str.removeprefix("token_id:"))
streamed_logprob_token_ids.append(token_id)
if first_chunk:
streamed_prompt_token_ids = chunk.choices[0].prompt_token_ids
first_chunk = False
streamed_token_ids += chunk.choices[0].token_ids
# Verify we collected some tokens and first chunk had prompt_token_ids
assert len(streamed_prompt_token_ids) > 0
assert streamed_token_ids == streamed_logprob_token_ids
@pytest.mark.asyncio
async def test_chat_completion_with_emoji_and_token_ids(server):
"""Test chat completion with emojis to verify token_ids handling."""
chat_messages = [
{"role": "system", "content": "You like to use emojis in your responses."},
{"role": "user", "content": "Repeat after me: I love cats 🐱"},
]
async with server.get_async_client() as client:
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=chat_messages,
max_tokens=50,
temperature=0,
logprobs=True,
extra_body={"return_token_ids": True},
)
# Verify token_ids are present
response_dict = response.model_dump()
assert response.choices[0].token_ids is not None
assert "prompt_token_ids" in response_dict
# Verify the response contains the expected fields
assert response.choices[0].message.content is not None
# Decode token_ids and verify consistency
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
decoded_prompt = tokenizer.decode(response.prompt_token_ids)
assert decoded_prompt.startswith(
"<|im_start|>system\nYou like to use emojis in your responses."
)
assert decoded_prompt.endswith(
"I love cats 🐱<|im_end|>\n<|im_start|>assistant\n"
)
decoded_response = tokenizer.decode(response.choices[0].token_ids)
# The content should match the response text
# except the ending <|im_end|>
assert decoded_response == response.choices[0].message.content + "<|im_end|>"
# Test with streaming
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=chat_messages,
max_tokens=50,
temperature=0,
stream=True,
extra_body={"return_token_ids": True},
)
collected_content = ""
collected_token_ids = []
first_chunk = True
async for chunk in stream:
if first_chunk:
assert chunk.prompt_token_ids is not None
assert isinstance(chunk.prompt_token_ids, list)
# Check the prompt_token_ids match the initial prompt
decoded_prompt_stream = tokenizer.decode(chunk.prompt_token_ids)
assert decoded_prompt_stream == decoded_prompt
first_chunk = False
else:
chunk_dump = chunk.model_dump()
assert "prompt_token_ids" not in chunk_dump, (
"Subsequent chunks should not have prompt_token_ids"
)
if chunk.choices:
if chunk.choices[0].delta.content:
collected_content += chunk.choices[0].delta.content
# token_ids may not present in all chunks
choice_dump = chunk.choices[0].model_dump()
if "token_ids" in choice_dump:
collected_token_ids.extend(chunk.choices[0].token_ids)
# Verify we got response and token_ids
assert len(collected_content) > 0
assert len(collected_token_ids) > 0
# Verify token_ids decode properly
decoded_response = tokenizer.decode(collected_token_ids)
assert decoded_response == collected_content + "<|im_end|>"
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Separate these tests out from test_completion and test_chat, because they
# require launching a second server with a different flag. Running both servers
# at the same time on a single node will OOM.
import pytest
from vllm.tokenizers import get_tokenizer
from ...utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def default_server_args(qwen3_lora_files):
return [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--max-num-seqs",
"128",
"--enforce-eager",
# lora config
"--enable-lora",
"--lora-modules",
f"qwen3-lora={qwen3_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
]
@pytest.fixture(scope="module")
def server_fixture(request, default_server_args):
use_server_flag = request.param
if use_server_flag:
args_with_flag = default_server_args + ["--return-tokens-as-token-ids"]
with RemoteOpenAIServer(MODEL_NAME, args_with_flag) as remote_server:
yield (remote_server, True)
else:
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
yield (remote_server, False)
@pytest.mark.asyncio
@pytest.mark.parametrize("server_fixture", [True, False], indirect=True)
async def test_completion_return_tokens_as_token_ids_completion(server_fixture):
server, use_server_flag = server_fixture
request_args = {}
if not use_server_flag:
request_args["return_tokens_as_token_ids"] = True
async with server.get_async_client() as client:
completion = await client.completions.create(
model=MODEL_NAME,
# Include Unicode characters to test for dividing a single
# character across multiple tokens: 🎉 is [28705, 31862] for the
# Zephyr tokenizer
prompt="Say 'Hello, world! 🎉'",
echo=True,
temperature=0,
max_tokens=10,
logprobs=1,
extra_body=request_args,
)
text = completion.choices[0].text
token_strs = completion.choices[0].logprobs.tokens
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
# Check that the token representations are consistent between raw
# tokens and top_logprobs
# Slice off the first one, because there's no scoring associated
# with BOS
top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
top_logprob_keys = [
next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
]
assert token_strs[1:] == top_logprob_keys
# Check that decoding the tokens gives the expected text
tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
assert text == tokenizer.decode(tokens, skip_special_tokens=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("server_fixture", [True, False], indirect=True)
async def test_chat_return_tokens_as_token_ids_completion(server_fixture):
server, use_server_flag = server_fixture
request_args = {}
if not use_server_flag:
request_args["return_tokens_as_token_ids"] = True
async with server.get_async_client() as client:
response = await client.chat.completions.create(
model=MODEL_NAME,
# Include Unicode characters to test for dividing a single
# character across multiple tokens: 🎉 is [28705, 31862] for the
# Zephyr tokenizer
messages=[
{
"role": "system",
"content": "You like to respond in only emojis, like 🎉",
},
{"role": "user", "content": "Please write some emojis: 🐱🐶🎉"},
],
temperature=0,
max_tokens=8,
logprobs=True,
extra_body=request_args,
)
text = response.choices[0].message.content
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
token_ids = []
for logprob_content in response.choices[0].logprobs.content:
token_ids.append(int(logprob_content.token.removeprefix("token_id:")))
assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
def test_responses_api_logprobs_with_return_tokens_as_token_ids():
"""Test that return_tokens_as_token_ids works in Responses API logprobs."""
from unittest.mock import MagicMock
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
from vllm.logprobs import Logprob as SampleLogprob
serving = MagicMock(spec=OpenAIServingResponses)
serving.return_tokens_as_token_ids = True
serving._get_decoded_token = GenerateBaseServing._get_decoded_token
tokenizer = MagicMock()
tokenizer.decode = lambda token_id: "decoded"
token_ids = [100, 200, 300]
sample_logprobs = [
{100: SampleLogprob(logprob=-0.5, decoded_token="hello")},
{200: SampleLogprob(logprob=-1.2, decoded_token="world")},
{300: SampleLogprob(logprob=-0.8, decoded_token="!")},
]
result = OpenAIServingResponses._create_response_logprobs(
serving,
token_ids=token_ids,
logprobs=sample_logprobs,
tokenizer=tokenizer,
top_logprobs=1,
)
assert len(result) == 3
assert result[0].token == "token_id:100"
assert result[1].token == "token_id:200"
assert result[2].token == "token_id:300"
assert result[0].logprob == -0.5
assert result[1].logprob == -1.2
assert result[2].logprob == -0.8
@@ -1,249 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
import pytest
from fastapi import UploadFile
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.speech_to_text.transcription.protocol import (
TranscriptionRequest,
)
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
from vllm.sampling_params import StructuredOutputsParams
def test_chat_request_preserves_watermarking_opt_out():
request = ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}], watermarking=False
)
params = request.to_sampling_params(max_tokens=1, default_sampling_params={})
assert params.watermarking is False
def test_completion_request_preserves_watermarking_opt_out():
request = CompletionRequest(prompt="hello", watermarking=False)
params = request.to_sampling_params(max_tokens=1)
assert params.watermarking is False
def test_responses_request_preserves_watermarking_opt_out():
request = ResponsesRequest(input="hello", watermarking=False)
params = request.to_sampling_params(default_max_tokens=1)
assert params.watermarking is False
def test_batch_chat_request_preserves_watermarking_opt_out():
request = BatchChatCompletionRequest(
messages=[[{"role": "user", "content": "hello"}]], watermarking=False
)
converted = request.to_chat_completion_request(request.messages[0])
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
assert params.watermarking is False
@pytest.mark.parametrize(
"api_request,conversion,kwargs",
[
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}],
use_beam_search=True,
watermarking=False,
),
"to_beam_search_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=False),
"to_beam_search_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
TranscriptionRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranscriptionRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_beam_search_params",
{"default_max_tokens": 1},
),
(
TranslationRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranslationRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=False,
),
"to_beam_search_params",
{"default_max_tokens": 1},
),
],
)
def test_request_preserves_watermarking_opt_out(api_request, conversion, kwargs):
params = getattr(api_request, conversion)(**kwargs)
assert params.watermarking is False
@pytest.mark.parametrize("request_cls", [TranscriptionRequest, TranslationRequest])
def test_speech_requests_leave_watermarking_unspecified(request_cls):
request = request_cls(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
temperature=0.8,
)
sampling_params = request.to_sampling_params(default_max_tokens=1)
beam_params = request.to_beam_search_params(default_max_tokens=1)
assert request.watermarking is None
assert sampling_params.watermarking is None
assert beam_params.watermarking is None
@pytest.mark.parametrize(
"api_request,kwargs",
[
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "choose A or B"}],
temperature=0.8,
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
),
{"max_tokens": 8, "default_sampling_params": {}},
),
(
CompletionRequest(
prompt="choose A or B",
temperature=0.8,
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
),
{"max_tokens": 8},
),
(
ResponsesRequest(
input="choose A or B",
temperature=0.8,
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
),
{"default_max_tokens": 8},
),
],
)
def test_request_preserves_unspecified_watermarking_with_structured_outputs(
api_request, kwargs
):
params = api_request.to_sampling_params(**kwargs)
assert params.watermarking is None
assert params.structured_outputs is not None
assert params.structured_outputs.choice == ["A", "B"]
@pytest.mark.parametrize(
"api_request,conversion,kwargs",
[
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}], watermarking=True
),
"to_sampling_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}],
use_beam_search=True,
watermarking=True,
),
"to_beam_search_params",
{"max_tokens": 1, "default_sampling_params": {}},
),
(
CompletionRequest(prompt="hello", watermarking=True),
"to_sampling_params",
{"max_tokens": 1},
),
(
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=True),
"to_beam_search_params",
{"max_tokens": 1},
),
(
ResponsesRequest(input="hello", watermarking=True),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranscriptionRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=True,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
(
TranslationRequest(
file=UploadFile(file=BytesIO(), filename="audio.wav"),
watermarking=True,
),
"to_sampling_params",
{"default_max_tokens": 1},
),
],
)
def test_request_preserves_explicit_watermarking(api_request, conversion, kwargs):
params = getattr(api_request, conversion)(**kwargs)
assert params.watermarking is True
def test_batch_chat_request_preserves_explicit_watermarking():
request = BatchChatCompletionRequest(
messages=[[{"role": "user", "content": "hello"}]], watermarking=True
)
converted = request.to_chat_completion_request(request.messages[0])
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
assert converted.watermarking is True
assert params.watermarking is True
def test_batch_chat_request_inherits_watermarking():
request = BatchChatCompletionRequest(
messages=[[{"role": "user", "content": "hello"}]]
)
converted = request.to_chat_completion_request(request.messages[0])
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
assert converted.watermarking is None
assert params.watermarking is None