[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,558 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
BASE_MODEL_PATHS,
|
||||
CHAT_TEMPLATE,
|
||||
MockHFConfig,
|
||||
MockModelConfig,
|
||||
_build_renderer,
|
||||
)
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
MODEL_NAME as MOCK_MODEL_NAME,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
# any model with a chat template defined in tokenizer_config should work here
|
||||
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
[{"role": "user", "content": "Reply with exactly the word: beta"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
choices = data["choices"]
|
||||
assert len(choices) == 2
|
||||
|
||||
indices = {choice["index"] for choice in choices}
|
||||
assert indices == {0, 1}
|
||||
|
||||
# Each conversation should produce a non-empty text response.
|
||||
for choice in choices:
|
||||
assert choice["message"]["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_with_json_schema(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"answer": {"type": "string", "enum": ["yes", "no"]},
|
||||
},
|
||||
"required": ["answer"],
|
||||
}
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Is the sky blue? Answer in JSON."}],
|
||||
[{"role": "user", "content": "Is fire cold? Answer in JSON."}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "answer", "schema": schema, "strict": True},
|
||||
},
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
choices = data["choices"]
|
||||
assert len(choices) == 2
|
||||
|
||||
for choice in choices:
|
||||
parsed = json.loads(choice["message"]["content"])
|
||||
assert "answer" in parsed
|
||||
assert parsed["answer"] in ("yes", "no")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_logprobs_not_token_id_placeholders(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
# Regression test: requesting `return_token_ids` alongside logprobs must not
|
||||
# corrupt the logprob `token` fields into "token_id:{id}" placeholders. That
|
||||
# placeholder rendering is controlled by `return_tokens_as_token_ids`, which
|
||||
# this request leaves unset.
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 1,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
for entry in content:
|
||||
assert not entry["token"].startswith("token_id:")
|
||||
for top in entry["top_logprobs"]:
|
||||
assert not top["token"].startswith("token_id:")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_return_tokens_as_token_ids(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
# Complementary check: when `return_tokens_as_token_ids` is explicitly set,
|
||||
# the logprob tokens *should* be rendered as "token_id:{id}" placeholders,
|
||||
# proving the new field is actually wired through.
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 1,
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
assert all(entry["token"].startswith("token_id:") for entry in content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batched_chat_completions_logprob_token_ids(
|
||||
server: RemoteOpenAIServer,
|
||||
) -> None:
|
||||
conversations = [[{"role": "user", "content": "Hello"}]]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": conversations,
|
||||
"max_tokens": 1,
|
||||
"temperature": 0,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 5,
|
||||
"logprob_token_ids": [100, 1000, 5000],
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
content = response.json()["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
sampled_token = content[0]["token"]
|
||||
assert {entry["token"] for entry in content[0]["top_logprobs"]} == {
|
||||
"token_id:100",
|
||||
"token_id:1000",
|
||||
"token_id:5000",
|
||||
sampled_token,
|
||||
}
|
||||
|
||||
|
||||
def _make_request_output(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=f"req-{prompt_idx}",
|
||||
prompt=None,
|
||||
prompt_token_ids=prompt_token_ids or [1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=[4, 5],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
async def _generator(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _make_request_output(prompt_idx, text, prompt_token_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
|
||||
"""Echo must only echo the assistant turn, never the user's prompt.
|
||||
|
||||
With `add_generation_prompt` (the default) the response role is `assistant`
|
||||
while the last conversation message is the user's, so there is nothing to
|
||||
echo; without the role check the user prompt leaks into the answer.
|
||||
"""
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
|
||||
conversations = [[{"role": "user", "content": "USER PROMPT"}]]
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=conversations,
|
||||
echo=True,
|
||||
)
|
||||
|
||||
response = await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "ASSISTANT ANSWER")],
|
||||
request_id="req-echo",
|
||||
model_name="test-model",
|
||||
all_conversations=conversations,
|
||||
tokenizer=None,
|
||||
request_metadata=RequestResponseMetadata(request_id="req-echo"),
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "ASSISTANT ANSWER"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
|
||||
"""A trailing assistant turn is a real prefix and must still be echoed."""
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
|
||||
conversations = [
|
||||
[
|
||||
{"role": "user", "content": "USER PROMPT"},
|
||||
{"role": "assistant", "content": "PREFIX "},
|
||||
]
|
||||
]
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=conversations,
|
||||
echo=True,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
|
||||
response = await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "ASSISTANT ANSWER")],
|
||||
request_id="req-echo-2",
|
||||
model_name="test-model",
|
||||
all_conversations=conversations,
|
||||
tokenizer=None,
|
||||
request_metadata=RequestResponseMetadata(request_id="req-echo-2"),
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
|
||||
"""Harmony must constrain only the final channel of every conversation."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.errored = False
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
|
||||
|
||||
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="openai_gptoss",
|
||||
)
|
||||
serving = OpenAIServingChatBatch(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
)
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=[
|
||||
[{"role": "user", "content": "France?"}],
|
||||
[{"role": "user", "content": "Japan?"}],
|
||||
],
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "city", "schema": {"type": "object"}},
|
||||
},
|
||||
)
|
||||
await serving.create_batch_chat_completion(request)
|
||||
|
||||
calls = mock_engine.generate.call_args_list
|
||||
assert len(calls) == 2
|
||||
for call in calls:
|
||||
assert call.args[1].structured_outputs.structural_tag is not None
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_batch_rejects_kv_transfer_prompt_token_ids():
|
||||
"""One pre-tokenized prompt cannot stand in for every conversation."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match=r"parameter=kv_transfer_params\.prompt_token_ids"
|
||||
):
|
||||
BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
[{"role": "user", "content": "first"}],
|
||||
[{"role": "user", "content": "second"}],
|
||||
],
|
||||
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_parser_receives_each_prompt_prefix() -> None:
|
||||
prefixes = []
|
||||
|
||||
class RecordingParser:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prompt_token_ids = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids):
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
|
||||
def parse(self, model_output, **kwargs):
|
||||
prefixes.append(self.prompt_token_ids)
|
||||
return None, model_output, None
|
||||
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
serving.chat_template = None
|
||||
serving.chat_template_content_format = "auto"
|
||||
serving.default_chat_template_kwargs = {}
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
[{"role": "user", "content": "first"}],
|
||||
[{"role": "user", "content": "second"}],
|
||||
],
|
||||
)
|
||||
await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
|
||||
request_id="req-prefix",
|
||||
model_name="test-model",
|
||||
all_conversations=request.messages,
|
||||
tokenizer=object(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
|
||||
parser_cls=RecordingParser,
|
||||
)
|
||||
|
||||
assert prefixes == [[1], [2]]
|
||||
|
||||
|
||||
REASONING_PARSER = "kimi_k3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_reasoning_wiring_matches_single_chat() -> None:
|
||||
"""`/v1/chat/completions/batch` must hand the engine the same reasoning
|
||||
wiring as `/v1/chat/completions`.
|
||||
|
||||
Both `reasoning_ended` and `reasoning_parser_kwargs` are carried on the
|
||||
request into `EngineCoreRequest`, where
|
||||
`StructuredOutputManager._get_reasoner()` rebuilds the request-local
|
||||
reasoning parser from the forwarded `chat_template_kwargs` -- "so the
|
||||
structured-output gate observes the same template kwargs used by the
|
||||
frontend". A divergence makes a template-kwarg-driven parser fall back to
|
||||
its default (`thinking` on) for a request that disabled thinking, so
|
||||
`is_reasoning_end()` keeps reporting False, the structured-output gate
|
||||
never opens, and the grammar is silently never applied.
|
||||
|
||||
Both frontends run over one and the same model and renderer, so the only
|
||||
thing that can differ between the two calls is the serving code itself.
|
||||
"""
|
||||
mock_model_config = MockModelConfig()
|
||||
shared_renderer = _build_renderer(mock_model_config)
|
||||
|
||||
def _build_serving(cls):
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.model_config = mock_model_config
|
||||
mock_engine.errored = False
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = shared_renderer
|
||||
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
|
||||
|
||||
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser=REASONING_PARSER,
|
||||
)
|
||||
return mock_engine, cls(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
reasoning_parser=REASONING_PARSER,
|
||||
)
|
||||
|
||||
conversation = [{"role": "user", "content": "What is the capital of France?"}]
|
||||
chat_template_kwargs = {"enable_thinking": False}
|
||||
|
||||
single_engine, single_serving = _build_serving(OpenAIServingChat)
|
||||
await single_serving.create_chat_completion(
|
||||
ChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=conversation,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
single_call = single_engine.generate.call_args_list[-1]
|
||||
|
||||
batch_engine, batch_serving = _build_serving(OpenAIServingChatBatch)
|
||||
await batch_serving.create_batch_chat_completion(
|
||||
BatchChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=[conversation],
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
batch_call = batch_engine.generate.call_args_list[-1]
|
||||
|
||||
# Guard against a vacuous comparison: the reference path must be wired up.
|
||||
assert single_call.kwargs["reasoning_ended"] is not None
|
||||
assert single_call.kwargs["reasoning_parser_kwargs"] is not None
|
||||
|
||||
assert batch_call.kwargs["reasoning_ended"] == single_call.kwargs["reasoning_ended"]
|
||||
assert (
|
||||
batch_call.kwargs["reasoning_parser_kwargs"]
|
||||
== single_call.kwargs["reasoning_parser_kwargs"]
|
||||
)
|
||||
|
||||
# ...and the forwarded kwargs must actually turn thinking off for the
|
||||
# engine-side parser, i.e. the structured-output gate opens right away.
|
||||
engine_chat_template_kwargs = batch_call.kwargs["reasoning_parser_kwargs"][
|
||||
"chat_template_kwargs"
|
||||
]
|
||||
reasoner = ReasoningParserManager.get_reasoning_parser(REASONING_PARSER)(
|
||||
tokenizer=batch_engine.renderer.tokenizer,
|
||||
chat_template_kwargs=engine_chat_template_kwargs,
|
||||
)
|
||||
assert reasoner.is_reasoning_end([]) is True
|
||||
@@ -1,298 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for `prompt_embeds` content parts in the Chat Completions API."""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
|
||||
import openai
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch
|
||||
from openai import BadRequestError
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
# Matches `--dtype` in `server_args` to avoid an implicit cast in
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically, we match here just to skip the conversion).
|
||||
SERVER_DTYPE: torch.dtype = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--chat-template",
|
||||
str(CHAT_TEMPLATE),
|
||||
# Prompt Embeds server args
|
||||
"--enable-prompt-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args, request):
|
||||
if current_platform.is_rocm():
|
||||
# Materialize HF embeddings before the server reserves ROCm VRAM.
|
||||
request.getfixturevalue("prompt_embeds_b64")
|
||||
request.getfixturevalue("aligned_content_and_embeds_b64")
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
def _encode_embeds(embeds: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(embeds, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def prompt_embeds_b64(hf_runner) -> list[str]:
|
||||
"""Pre-compute embeddings for two short prompts and return as base64."""
|
||||
prompts = ["Hello, my name is", "What is an LLM?"]
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
embeddings = hf_model.get_prompt_embeddings(prompts)
|
||||
# Cast to the server's dtype so `safe_load_prompt_embeds` doesn't need to
|
||||
# convert on its own, the function accepts any floating-point dtype and
|
||||
# will cast to the model's dtype, but matching up front skips the work.
|
||||
return [_encode_embeds(e.to(SERVER_DTYPE)) for e in embeddings]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_prompt_embeds_part(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""A user message with one prompt_embeds part + text."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_prompt_embeds_parts(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Multiple prompt_embeds parts in a single message."""
|
||||
b64_a, b64_b = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_a},
|
||||
{"type": "text", "text": " and "},
|
||||
{"type": "prompt_embeds", "data": b64_b},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_message_conversation(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""prompt_embeds in both system and user messages."""
|
||||
b64_sys, b64_usr = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": b64_sys},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_usr},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Streaming chat completion with prompt_embeds."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
|
||||
# Non-streaming baseline.
|
||||
baseline = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
expected = baseline.choices[0].message.content
|
||||
|
||||
# Streaming.
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
chunks: list[str] = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta.content
|
||||
if delta:
|
||||
chunks.append(delta)
|
||||
assert "".join(chunks) == expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def aligned_content_and_embeds_b64(hf_runner) -> tuple[str, str]:
|
||||
"""Return `(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
"""
|
||||
content = "Hello, my name is"
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
ids = hf_model.tokenizer(
|
||||
content, add_special_tokens=False, return_tensors="pt"
|
||||
).input_ids
|
||||
ids = hf_model.wrap_device({"input_ids": ids})["input_ids"]
|
||||
embed_layer = hf_model.model.get_input_embeddings()
|
||||
embeds = embed_layer(ids).squeeze(0).to(SERVER_DTYPE).cpu()
|
||||
return content, _encode_embeds(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_content_and_prompt_embeds_match(
|
||||
client: openai.AsyncOpenAI,
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
):
|
||||
"""Equal content in text and `prompt_embeds` should yield identical
|
||||
Chat Completions output under greedy decoding.
|
||||
"""
|
||||
content, encoded_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
text_resp, embeds_resp = await asyncio.gather(
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
),
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": encoded_embeds}],
|
||||
}
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_data_field(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""A prompt_embeds part without `data` should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_base64(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""Invalid base64 in the `data` field should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": "not_valid_base64!!"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for ``thinking_token_budget`` with reasoning models.
|
||||
|
||||
Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Literal
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
|
||||
THINK_BUDGET = 5
|
||||
|
||||
REASONING_START_STR = "<think>"
|
||||
REASONING_END_STR = "</think>"
|
||||
|
||||
|
||||
def _count_reasoning_decode_token_ids_between_markers(
|
||||
full_token_ids: list[int],
|
||||
reasoning_start_ids: list[int],
|
||||
reasoning_end_ids: list[int],
|
||||
) -> int | None:
|
||||
"""Count decode tokens in the thinking span (after last start, before first end)."""
|
||||
if not reasoning_start_ids or not reasoning_end_ids:
|
||||
raise ValueError("reasoning marker token id lists must be non-empty")
|
||||
|
||||
def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
|
||||
n = len(needle)
|
||||
if n > len(haystack):
|
||||
return -1
|
||||
for i in range(len(haystack) - n, -1, -1):
|
||||
if haystack[i : i + n] == needle:
|
||||
return i
|
||||
return -1
|
||||
|
||||
last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
|
||||
if last_start < 0:
|
||||
return None
|
||||
|
||||
pos_after_start = last_start + len(reasoning_start_ids)
|
||||
end_n = len(reasoning_end_ids)
|
||||
for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
|
||||
if full_token_ids[j : j + end_n] == reasoning_end_ids:
|
||||
return j - pos_after_start
|
||||
return len(full_token_ids) - pos_after_start
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--reasoning-config",
|
||||
'{"reasoning_start_str": "<think>", "reasoning_end_str": "</think>"}',
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_with_auto_reasoning_config():
|
||||
args = [
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_qwen35_fp8_mtp_tp2():
|
||||
"""Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
|
||||
if current_platform.device_count() < 2:
|
||||
pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
|
||||
if not current_platform.supports_fp8():
|
||||
pytest.skip("FP8 is not supported on this platform")
|
||||
|
||||
spec_cfg = {
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 2,
|
||||
"max_model_len": 32768,
|
||||
}
|
||||
args = [
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--max-model-len",
|
||||
"32768",
|
||||
"--speculative-config",
|
||||
json.dumps(spec_cfg),
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--reasoning-config",
|
||||
json.dumps(
|
||||
{
|
||||
"reasoning_start_str": REASONING_START_STR,
|
||||
"reasoning_end_str": REASONING_END_STR,
|
||||
}
|
||||
),
|
||||
]
|
||||
env_dict: dict[str, str] = {}
|
||||
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
|
||||
# on 0,1 do not exhaust memory on the same devices as this worker.
|
||||
if current_platform.device_count() >= 4:
|
||||
env_dict["CUDA_VISIBLE_DEVICES"] = "2,3"
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
QWEN35_FP8_MTP_MODEL,
|
||||
args,
|
||||
max_wait_seconds=3000,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(request, server, server_with_auto_reasoning_config):
|
||||
server_map = {
|
||||
"default": server,
|
||||
"auto_config": server_with_auto_reasoning_config,
|
||||
}
|
||||
target_server = server_map[request.param]
|
||||
async with target_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||
"""Test that mixed requests (some with thinking_token_budget, some without)
|
||||
complete successfully without errors."""
|
||||
response_with_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET},
|
||||
)
|
||||
response_without_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
)
|
||||
|
||||
msg_with = response_with_budget.choices[0].message
|
||||
msg_without = response_without_budget.choices[0].message
|
||||
|
||||
assert msg_with.content or getattr(msg_with, "reasoning", None)
|
||||
assert msg_without.content or getattr(msg_without, "reasoning", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
|
||||
"""Test that thinking_token_budget limits the number of reasoning tokens.
|
||||
|
||||
Counts reasoning decode tokens by id, which is robust to how tokens are
|
||||
grouped into streamed chunks (a single chunk can carry several tokens under
|
||||
async scheduling / stream_interval > 1). Counting chunks under-counts.
|
||||
"""
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
|
||||
prompt_token_ids: list[int] = []
|
||||
decode_token_ids: list[int] = []
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
stream=True,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True},
|
||||
)
|
||||
async for chunk in stream:
|
||||
if not chunk.choices:
|
||||
continue
|
||||
if getattr(chunk, "prompt_token_ids", None):
|
||||
prompt_token_ids = list(chunk.prompt_token_ids)
|
||||
delta_ids = getattr(chunk.choices[0], "token_ids", None)
|
||||
if delta_ids:
|
||||
decode_token_ids.extend(delta_ids)
|
||||
|
||||
reasoning_token_count = _count_reasoning_decode_token_ids_between_markers(
|
||||
prompt_token_ids + decode_token_ids, start_ids, end_ids
|
||||
)
|
||||
assert reasoning_token_count is not None, "missing reasoning start marker in ids"
|
||||
assert reasoning_token_count == THINK_BUDGET, (
|
||||
f"reasoning tokens ({reasoning_token_count}) != "
|
||||
f"thinking_token_budget ({THINK_BUDGET})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
@requires_fp8
|
||||
async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
|
||||
server_qwen35_fp8_mtp_tp2,
|
||||
):
|
||||
"""Concurrent chat requests: some with ``thinking_token_budget``, some without.
|
||||
|
||||
Exercises the scheduler / input processor under a mixed batch on the same
|
||||
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
|
||||
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
|
||||
"""
|
||||
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
|
||||
("budget", 1),
|
||||
("budget", 12),
|
||||
("plain", None),
|
||||
("budget", 20),
|
||||
("budget", 14),
|
||||
("plain", None),
|
||||
("plain", None),
|
||||
("budget", 12),
|
||||
("plain", None),
|
||||
]
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
|
||||
async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
|
||||
|
||||
async def budgeted_call(expected_budget: int):
|
||||
return await client.chat.completions.create(
|
||||
model=QWEN35_FP8_MTP_MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=256,
|
||||
stream=False,
|
||||
extra_body={
|
||||
"thinking_token_budget": expected_budget,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
)
|
||||
|
||||
async def plain_call():
|
||||
return await client.chat.completions.create(
|
||||
model=QWEN35_FP8_MTP_MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=256,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
coros = []
|
||||
for row in _batch_spec:
|
||||
if row[0] == "budget":
|
||||
b = row[1]
|
||||
assert isinstance(b, int)
|
||||
coros.append(budgeted_call(b))
|
||||
else:
|
||||
coros.append(plain_call())
|
||||
results = await asyncio.gather(*coros)
|
||||
|
||||
for i, (response, (kind, expected_budget)) in enumerate(
|
||||
zip(results, _batch_spec, strict=True)
|
||||
):
|
||||
msg = response.choices[0].message
|
||||
assert msg.content or getattr(msg, "reasoning", None), (
|
||||
f"index {i} ({kind}): empty message"
|
||||
)
|
||||
|
||||
if kind == "budget":
|
||||
assert expected_budget is not None
|
||||
assert response.prompt_token_ids is not None
|
||||
assert response.choices[0].token_ids is not None
|
||||
full_ids = list(response.prompt_token_ids) + list(
|
||||
response.choices[0].token_ids
|
||||
)
|
||||
n_reason = _count_reasoning_decode_token_ids_between_markers(
|
||||
full_ids, start_ids, end_ids
|
||||
)
|
||||
assert n_reason is not None, f"index {i}: missing reasoning start in ids"
|
||||
assert n_reason == expected_budget, (
|
||||
f"index {i}: reasoning decode token ids ({n_reason}) != "
|
||||
f"thinking_token_budget ({expected_budget})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_streaming_with_thinking_disabled_stays_in_content(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
request_kwargs = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Which is larger, 4 or 12?"
|
||||
" Output exactly one token: 4 or 12.",
|
||||
}
|
||||
],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0.0,
|
||||
"extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
|
||||
}
|
||||
|
||||
response = await client.chat.completions.create(**request_kwargs)
|
||||
message = response.choices[0].message
|
||||
assert message.content is not None and message.content.strip() != ""
|
||||
assert getattr(message, "reasoning", None) in (None, "")
|
||||
|
||||
stream = await client.chat.completions.create(
|
||||
**request_kwargs,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
content_chunks = []
|
||||
reasoning_chunks = []
|
||||
async for chunk in stream:
|
||||
if not chunk.choices:
|
||||
continue
|
||||
delta = chunk.choices[0].delta
|
||||
if getattr(delta, "content", None):
|
||||
content_chunks.append(delta.content)
|
||||
if getattr(delta, "reasoning", None):
|
||||
reasoning_chunks.append(delta.reasoning)
|
||||
|
||||
assert "".join(content_chunks).strip() != ""
|
||||
assert reasoning_chunks == []
|
||||
@@ -1,830 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import regex as re
|
||||
from openai import BadRequestError
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
"--dtype",
|
||||
"float32",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--enable-prompt-tokens-details",
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_single_completion(client: openai.AsyncOpenAI, model_name: str) -> None:
|
||||
completion = await client.completions.create(
|
||||
model=model_name, prompt="Hello, my name is", max_tokens=5, temperature=0.0
|
||||
)
|
||||
|
||||
assert completion.id is not None
|
||||
assert completion.choices is not None and len(completion.choices) == 1
|
||||
|
||||
choice = completion.choices[0]
|
||||
assert len(choice.text) >= 5
|
||||
assert choice.finish_reason == "length"
|
||||
assert completion.usage is not None
|
||||
assert completion.usage.completion_tokens == 5
|
||||
assert completion.usage.prompt_tokens == 6
|
||||
assert completion.usage.total_tokens == 11
|
||||
assert completion.usage.prompt_tokens_details is not None
|
||||
assert completion.usage.prompt_tokens_details.cached_tokens == 0
|
||||
|
||||
# test using token IDs
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert len(completion.choices[0].text) >= 1
|
||||
assert completion.choices[0].prompt_logprobs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_truncation_side_controls_prompt_truncation(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
) -> None:
|
||||
prompt_token_ids = list(range(8))
|
||||
|
||||
right_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_token_ids,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"truncate_prompt_tokens": 4,
|
||||
"truncation_side": "right",
|
||||
},
|
||||
)
|
||||
assert right_completion.choices[0].prompt_token_ids == prompt_token_ids[:4]
|
||||
|
||||
left_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt_token_ids,
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"truncate_prompt_tokens": 4,
|
||||
"truncation_side": "left",
|
||||
},
|
||||
)
|
||||
assert left_completion.choices[0].prompt_token_ids == prompt_token_ids[-4:]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
|
||||
# test using token IDs
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
logprobs=None,
|
||||
)
|
||||
choice = completion.choices[0]
|
||||
assert choice.logprobs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
|
||||
# test using token IDs
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
logprobs=0,
|
||||
)
|
||||
choice = completion.choices[0]
|
||||
assert choice.logprobs is not None
|
||||
assert choice.logprobs.token_logprobs is not None
|
||||
assert choice.logprobs.top_logprobs is not None
|
||||
assert len(choice.logprobs.top_logprobs[0]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
|
||||
# test using token IDs
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
logprobs=5,
|
||||
)
|
||||
choice = completion.choices[0]
|
||||
assert choice.logprobs is not None
|
||||
assert choice.logprobs.token_logprobs is not None
|
||||
assert choice.logprobs.top_logprobs is not None
|
||||
assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprob_token_ids(client: openai.AsyncOpenAI):
|
||||
completion = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Hello",
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
logprobs=5,
|
||||
extra_body={
|
||||
"logprob_token_ids": [5000],
|
||||
"allowed_token_ids": [42],
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
)
|
||||
|
||||
choice = completion.choices[0]
|
||||
assert choice.logprobs is not None
|
||||
assert choice.logprobs.tokens == ["token_id:42"]
|
||||
assert choice.logprobs.top_logprobs is not None
|
||||
assert set(choice.logprobs.top_logprobs[0]) == {
|
||||
"token_id:42",
|
||||
"token_id:5000",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprob_token_ids_stream(client: openai.AsyncOpenAI):
|
||||
stream = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Hello",
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
logprobs=5,
|
||||
stream=True,
|
||||
extra_body={
|
||||
"logprob_token_ids": [5000],
|
||||
"allowed_token_ids": [42],
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
)
|
||||
|
||||
returned_top_logprobs: list[dict[str, float]] = []
|
||||
async for chunk in stream:
|
||||
logprobs = chunk.choices[0].logprobs
|
||||
if logprobs is not None and logprobs.top_logprobs is not None:
|
||||
returned_top_logprobs.extend(
|
||||
top_logprobs
|
||||
for top_logprobs in logprobs.top_logprobs
|
||||
if top_logprobs is not None
|
||||
)
|
||||
|
||||
assert len(returned_top_logprobs) == 1
|
||||
assert set(returned_top_logprobs[0]) == {"token_id:42", "token_id:5000"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_too_many_completion_logprobs(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
) -> None:
|
||||
with pytest.raises(
|
||||
(openai.BadRequestError, openai.APIError)
|
||||
): # test using token IDs
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
# vLLM has higher default max_logprobs (20 instead of 5) to support
|
||||
# both Completion API and Chat Completion API
|
||||
logprobs=21,
|
||||
)
|
||||
...
|
||||
with pytest.raises(
|
||||
(openai.BadRequestError, openai.APIError)
|
||||
): # test using token IDs
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
# vLLM has higher default max_logprobs (20 instead of 5) to support
|
||||
# both Completion API and Chat Completion API
|
||||
logprobs=30,
|
||||
stream=True,
|
||||
)
|
||||
async for chunk in stream:
|
||||
...
|
||||
|
||||
# the server should still work afterwards
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert len(completion.choices[0].text) >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, prompt_logprobs",
|
||||
[(MODEL_NAME, -1), (MODEL_NAME, 0), (MODEL_NAME, 1), (MODEL_NAME, None)],
|
||||
)
|
||||
async def test_prompt_logprobs_completion(
|
||||
client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: int | None
|
||||
):
|
||||
params: dict = {
|
||||
"prompt": ["A robot may not injure another robot", "My name is"],
|
||||
"model": model_name,
|
||||
}
|
||||
if prompt_logprobs is not None:
|
||||
params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
|
||||
|
||||
if prompt_logprobs is not None and prompt_logprobs < 0:
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.completions.create(**params)
|
||||
else:
|
||||
completion = await client.completions.create(**params)
|
||||
if prompt_logprobs is not None:
|
||||
assert completion.choices[0].prompt_logprobs is not None
|
||||
assert len(completion.choices[0].prompt_logprobs) > 0
|
||||
|
||||
assert completion.choices[1].prompt_logprobs is not None
|
||||
assert len(completion.choices[1].prompt_logprobs) > 0
|
||||
|
||||
else:
|
||||
assert completion.choices[0].prompt_logprobs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_streaming(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
) -> None:
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
single_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
stream = await client.completions.create(
|
||||
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
# finish reason should only return in last block
|
||||
assert finish_reason_count == 1
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
assert chunk.choices[0].text
|
||||
assert "".join(chunks) == single_output
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
"""Parallel sampling without streaming.
|
||||
A single request output contains a list of completions.
|
||||
"""
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50
|
||||
|
||||
# High temperature to maximize chance of unique completions.
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
n=n,
|
||||
temperature=1.0,
|
||||
stream=False,
|
||||
logprobs=0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
# Assert `n` completions
|
||||
num_completions = len(completion.choices)
|
||||
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
|
||||
completion_repeats: dict[str, int] = {}
|
||||
for idx, choice in enumerate(completion.choices):
|
||||
# Assert correct completion index & some finish reason.
|
||||
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
|
||||
assert choice.finish_reason is not None, "None finish_reason is invalid."
|
||||
text = choice.text
|
||||
completion_repeats[text] = completion_repeats.get(text, 0) + 1
|
||||
# Assert `n` unique completions
|
||||
num_unique = len(completion_repeats)
|
||||
if num_unique != n:
|
||||
repeats = {txt: num for (txt, num) in completion_repeats.items() if num > 1}
|
||||
raise AssertionError(
|
||||
f"Expected {n} unique completions, got {num_unique}; repeats: {repeats}."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
"""Streaming for parallel sampling.
|
||||
The tokens from multiple samples, are flattened into a single stream,
|
||||
with an index to indicate which sample the token belongs to.
|
||||
"""
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50
|
||||
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
n=n,
|
||||
temperature=1.0,
|
||||
stream=True,
|
||||
seed=42,
|
||||
)
|
||||
chunks: list[list[str]] = [[] for _ in range(n)]
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
index = chunk.choices[0].index
|
||||
text = chunk.choices[0].text
|
||||
chunks[index].append(text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
# Assert `n` completions with correct finish reasons
|
||||
assert finish_reason_count == n, (
|
||||
f"Expected {n} completions with valid indices and finish_reason."
|
||||
)
|
||||
completion_repeats: dict[str, int] = {}
|
||||
for chunk in chunks:
|
||||
chunk_len = len(chunk)
|
||||
# Assert correct number of completion tokens
|
||||
assert chunk_len <= max_tokens, (
|
||||
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
|
||||
)
|
||||
text = "".join(chunk)
|
||||
completion_repeats[text] = completion_repeats.get(text, 0) + 1
|
||||
print(text)
|
||||
# Assert `n` unique completions
|
||||
num_unique = len(completion_repeats)
|
||||
if num_unique != n:
|
||||
repeats = {txt: num for (txt, num) in completion_repeats.items() if num > 1}
|
||||
raise AssertionError(
|
||||
f"{num_unique} unique completions, expected {n}; repeats: {repeats}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_stream_options(client: openai.AsyncOpenAI, model_name: str):
|
||||
prompt = "What is the capital of France?"
|
||||
|
||||
# Test stream=True, stream_options=
|
||||
# {"include_usage": False, "continuous_usage_stats": False}
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
stream_options={
|
||||
"include_usage": False,
|
||||
"continuous_usage_stats": False,
|
||||
},
|
||||
)
|
||||
|
||||
async for chunk in stream:
|
||||
assert chunk.usage is None
|
||||
|
||||
# Test stream=True, stream_options=
|
||||
# {"include_usage": False, "continuous_usage_stats": True}
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
stream_options={
|
||||
"include_usage": False,
|
||||
"continuous_usage_stats": True,
|
||||
},
|
||||
)
|
||||
async for chunk in stream:
|
||||
assert chunk.usage is None
|
||||
|
||||
# Test stream=True, stream_options=
|
||||
# {"include_usage": True, "continuous_usage_stats": False}
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
stream_options={
|
||||
"include_usage": True,
|
||||
"continuous_usage_stats": False,
|
||||
},
|
||||
)
|
||||
async for chunk in stream:
|
||||
if chunk.choices[0].finish_reason is None:
|
||||
assert chunk.usage is None
|
||||
else:
|
||||
assert chunk.usage is None
|
||||
final_chunk = await anext(stream)
|
||||
assert final_chunk.usage is not None
|
||||
assert final_chunk.usage.prompt_tokens > 0
|
||||
assert final_chunk.usage.completion_tokens > 0
|
||||
assert final_chunk.usage.total_tokens == (
|
||||
final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
|
||||
)
|
||||
assert final_chunk.choices == []
|
||||
|
||||
# Test stream=True, stream_options=
|
||||
# {"include_usage": True, "continuous_usage_stats": True}
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
stream_options={
|
||||
"include_usage": True,
|
||||
"continuous_usage_stats": True,
|
||||
},
|
||||
)
|
||||
async for chunk in stream:
|
||||
assert chunk.usage is not None
|
||||
assert chunk.usage.prompt_tokens > 0
|
||||
assert chunk.usage.completion_tokens > 0
|
||||
assert chunk.usage.total_tokens == (
|
||||
chunk.usage.prompt_tokens + chunk.usage.completion_tokens
|
||||
)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
final_chunk = await anext(stream)
|
||||
assert final_chunk.usage is not None
|
||||
assert final_chunk.usage.prompt_tokens > 0
|
||||
assert final_chunk.usage.completion_tokens > 0
|
||||
assert final_chunk.usage.total_tokens == (
|
||||
final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
|
||||
)
|
||||
assert final_chunk.choices == []
|
||||
|
||||
# Test stream=True, stream_options={}
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
stream_options={},
|
||||
)
|
||||
async for chunk in stream:
|
||||
assert chunk.usage is None
|
||||
|
||||
# Test stream=False, stream_options=
|
||||
# {"include_usage": None}
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=False,
|
||||
stream_options={"include_usage": None},
|
||||
)
|
||||
|
||||
# Test stream=False, stream_options=
|
||||
# {"include_usage": True}
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=False,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
# Test stream=False, stream_options=
|
||||
# {"continuous_usage_stats": None}
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=False,
|
||||
stream_options={"continuous_usage_stats": None},
|
||||
)
|
||||
|
||||
# Test stream=False, stream_options=
|
||||
# {"continuous_usage_stats": True}
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=False,
|
||||
stream_options={"continuous_usage_stats": True},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
|
||||
# test both text and token IDs
|
||||
for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
|
||||
# test simple list
|
||||
batch = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompts,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert len(batch.choices) == 2
|
||||
assert batch.choices[0].text == batch.choices[1].text
|
||||
|
||||
# test n = 2
|
||||
batch = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompts,
|
||||
n=2,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
extra_body=dict(
|
||||
# NOTE: this has to be true for n > 1 in vLLM, but
|
||||
# not necessary for official client.
|
||||
use_beam_search=True
|
||||
),
|
||||
)
|
||||
assert len(batch.choices) == 4
|
||||
assert batch.choices[0].text != batch.choices[1].text, (
|
||||
"beam search should be different"
|
||||
)
|
||||
assert batch.choices[0].text == batch.choices[2].text, (
|
||||
"two copies of the same prompt should be the same"
|
||||
)
|
||||
assert batch.choices[1].text == batch.choices[3].text, (
|
||||
"two copies of the same prompt should be the same"
|
||||
)
|
||||
|
||||
# test streaming
|
||||
batch = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompts,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
)
|
||||
texts = [""] * 2
|
||||
async for chunk in batch:
|
||||
assert len(chunk.choices) == 1
|
||||
choice = chunk.choices[0]
|
||||
texts[choice.index] += choice.text
|
||||
assert texts[0] == texts[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
@pytest.mark.parametrize("logprobs_arg", [1, 0])
|
||||
async def test_echo_logprob_completion(
|
||||
client: openai.AsyncOpenAI, model_name: str, logprobs_arg: int
|
||||
):
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
# test using text and token IDs
|
||||
for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
echo=True,
|
||||
logprobs=logprobs_arg,
|
||||
)
|
||||
|
||||
prompt_text = tokenizer.decode(prompt) if isinstance(prompt, list) else prompt
|
||||
assert re.search(r"^" + prompt_text, completion.choices[0].text)
|
||||
logprobs = completion.choices[0].logprobs
|
||||
assert logprobs is not None
|
||||
assert len(logprobs.text_offset) > 5
|
||||
assert len(logprobs.token_logprobs) > 5 and logprobs.token_logprobs[0] is None
|
||||
assert len(logprobs.top_logprobs) > 5 and logprobs.top_logprobs[0] is None
|
||||
for top_logprobs in logprobs.top_logprobs[1:]:
|
||||
assert max(logprobs_arg, 1) <= len(top_logprobs) <= logprobs_arg + 1
|
||||
assert len(logprobs.tokens) > 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_invalid_json_schema(client: openai.AsyncOpenAI, model_name: str) -> None:
|
||||
invalid_json_schema = {
|
||||
"$defs": {
|
||||
"CarType": {
|
||||
"enum": ["sedan", "SUV", "Truck", "Coupe"],
|
||||
"title": "CarType",
|
||||
"type": "string",
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"brand": {"title": "Brand", "type": "string"},
|
||||
"model": {"title": "Model", "type": "string"},
|
||||
"car_type": {"$ref": "#/$defs/CarType"},
|
||||
"foo": "bar",
|
||||
},
|
||||
"required": ["brand", "model", "car_type"],
|
||||
"title": "CarDescription",
|
||||
"type": "object",
|
||||
}
|
||||
prompt = (
|
||||
"Generate a JSON with the brand, model and car_type of"
|
||||
"the most iconic car from the 90's"
|
||||
)
|
||||
with pytest.raises((openai.BadRequestError, openai.APIError)):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
extra_body={"structured_outputs": {"json": invalid_json_schema}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_invalid_regex(client: openai.AsyncOpenAI, model_name: str):
|
||||
prompt = (
|
||||
"Generate an email address for Alan Turing, who works in Enigma."
|
||||
"End in .com and new line. Example result:"
|
||||
"alan.turing@enigma.com\n"
|
||||
)
|
||||
|
||||
with pytest.raises((openai.BadRequestError, openai.APIError)):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
extra_body={"structured_outputs": {"regex": r"[.*"}, "stop": ["\n"]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_invalid_grammar(client: openai.AsyncOpenAI, model_name: str):
|
||||
invalid_simplified_sql_grammar = """
|
||||
root ::= select_statementinvalidsyntax
|
||||
|
||||
select_statement ::= "SELECT " column " from " table " where " condition
|
||||
|
||||
column ::= "col_1 " | "col_2 "
|
||||
|
||||
table ::= "table_1 " | "table_2 "
|
||||
|
||||
condition ::= column "= " number
|
||||
|
||||
number ::= "1 " | "2 "
|
||||
"""
|
||||
|
||||
prompt = (
|
||||
"Generate an SQL query to show the 'username' and 'email'"
|
||||
"from the 'users' table."
|
||||
)
|
||||
with pytest.raises((openai.BadRequestError, openai.APIError)):
|
||||
await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
extra_body={
|
||||
"structured_outputs": {"grammar": invalid_simplified_sql_grammar}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# Unit tests for bad_words in CompletionRequest.to_sampling_params()
|
||||
def test_completion_request_bad_words_to_sampling_params():
|
||||
"""bad_words should be forwarded to SamplingParams (parity with chat)."""
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="Hello",
|
||||
bad_words=["foo", "bar"],
|
||||
max_tokens=10,
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(
|
||||
max_tokens=10,
|
||||
default_sampling_params={},
|
||||
)
|
||||
|
||||
assert isinstance(sampling_params, SamplingParams)
|
||||
assert sampling_params.bad_words == ["foo", "bar"]
|
||||
|
||||
|
||||
def test_completion_request_bad_words_default_empty():
|
||||
"""bad_words defaults to an empty list, matching the chat endpoint."""
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="Hello",
|
||||
max_tokens=10,
|
||||
)
|
||||
|
||||
assert request.bad_words == []
|
||||
sampling_params = request.to_sampling_params(
|
||||
max_tokens=10,
|
||||
default_sampling_params={},
|
||||
)
|
||||
assert sampling_params.bad_words == []
|
||||
|
||||
|
||||
def test_completion_request_forwards_routed_experts_prompt_start():
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="Hello",
|
||||
max_tokens=10,
|
||||
routed_experts_prompt_start=3,
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(
|
||||
max_tokens=10,
|
||||
default_sampling_params={},
|
||||
)
|
||||
|
||||
assert sampling_params.routed_experts_prompt_start == 3
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch.cuda
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.model_loader.tensorizer import (
|
||||
TensorizerConfig,
|
||||
tensorize_vllm_model,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "unsloth/llama-3.2-1b-Instruct"
|
||||
|
||||
|
||||
def _cleanup():
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup():
|
||||
_cleanup()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tmp_dir():
|
||||
with tempfile.TemporaryDirectory() as path:
|
||||
yield path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def model_uri(tmp_dir):
|
||||
yield f"{tmp_dir}/model.tensors"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tensorize_model(model_uri):
|
||||
tensorizer_config = TensorizerConfig(tensorizer_uri=model_uri)
|
||||
args = EngineArgs(model=MODEL_NAME)
|
||||
|
||||
tensorize_vllm_model(args, tensorizer_config)
|
||||
|
||||
# Manually invoke a _cleanup() here, as the cleanup()
|
||||
# fixture won't be guaranteed to be called after this
|
||||
# when this fixture is used for a test
|
||||
_cleanup()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(model_uri, tensorize_model):
|
||||
# In this case, model_uri is a directory with a model.tensors
|
||||
# file and all necessary model artifacts, particularly a
|
||||
# HF `config.json` file. In this case, Tensorizer can infer the
|
||||
# `TensorizerConfig` so --model-loader-extra-config can be completely
|
||||
# omitted.
|
||||
|
||||
## Start OpenAI API server
|
||||
args = [
|
||||
"--load-format",
|
||||
"tensorizer",
|
||||
"--served-model-name",
|
||||
MODEL_NAME,
|
||||
]
|
||||
if current_platform.is_rocm():
|
||||
args += ["--attention-backend", "TRITON_ATTN"]
|
||||
|
||||
model_dir = os.path.dirname(model_uri)
|
||||
with RemoteOpenAIServer(model_dir, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
|
||||
_cleanup()
|
||||
completion = await client.completions.create(
|
||||
model=model_name, prompt="Hello, my name is", max_tokens=5, temperature=0.0
|
||||
)
|
||||
|
||||
assert completion.id is not None
|
||||
assert completion.choices is not None and len(completion.choices) == 1
|
||||
assert completion.model == MODEL_NAME
|
||||
assert len(completion.choices) == 1
|
||||
assert len(completion.choices[0].text) >= 5
|
||||
assert completion.choices[0].finish_reason == "length"
|
||||
assert completion.usage == openai.types.CompletionUsage(
|
||||
completion_tokens=5, prompt_tokens=6, total_tokens=11
|
||||
)
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
MODEL_PATH = os.path.join(tempfile.gettempdir(), "qwen3_06b")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
global MODEL_PATH
|
||||
MODEL_PATH = download_weights_from_hf(
|
||||
MODEL_NAME,
|
||||
allow_patterns=["*"],
|
||||
cache_dir=MODEL_PATH,
|
||||
ignore_patterns=["tokenizer*", "vocab*", "*.safetensors"],
|
||||
)
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--skip-tokenizer-init",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_PATH, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_in_token_out_and_logprobs(server):
|
||||
"""Test token-in-token-out and token_ids align with prompt_logprobs
|
||||
& logprobs when return_tokens_as_token_ids is enabled.
|
||||
"""
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
text = "Hello, world! How are you today?"
|
||||
token_ids = tokenizer.encode(text)
|
||||
async with server.get_async_client() as client:
|
||||
# Test with both return_token_ids and return_tokens_as_token_ids enabled
|
||||
completion = await client.completions.create(
|
||||
model=MODEL_PATH,
|
||||
prompt=token_ids,
|
||||
max_tokens=20,
|
||||
temperature=0,
|
||||
echo=True,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
},
|
||||
)
|
||||
|
||||
# Verify all fields are present
|
||||
assert (
|
||||
completion.choices[0].token_ids is not None
|
||||
and 0 < len(completion.choices[0].token_ids) <= 20
|
||||
)
|
||||
assert completion.choices[0].prompt_token_ids is not None
|
||||
|
||||
# Decode prompt tokens
|
||||
if completion.choices[0].prompt_token_ids:
|
||||
prompt_text = tokenizer.decode(completion.choices[0].prompt_token_ids)
|
||||
# The decoded prompt should match or close to original prompt
|
||||
assert prompt_text == text
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This file test accuracy of the vLLM server via LMEval.
|
||||
It uses local-completions, which interacts with vLLM
|
||||
through the OAI API with N concurrent connections.
|
||||
This simulates real work usage of the API and makes
|
||||
sure that the zmq frontend mp RPC message passing and
|
||||
AsyncLLMEngine are working correctly.
|
||||
"""
|
||||
|
||||
import lm_eval
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
|
||||
NUM_CONCURRENT = 500
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
EXPECTED_VALUE = 0.54
|
||||
DEFAULT_ARGS = ["--max-model-len", "4096"]
|
||||
MORE_ARGS_LIST = [
|
||||
[], # Default
|
||||
["--enable-chunked-prefill"], # Chunked
|
||||
]
|
||||
MAX_WAIT_SECONDS = None
|
||||
|
||||
if current_platform.is_tpu():
|
||||
MORE_ARGS_LIST = [
|
||||
[], # Default
|
||||
]
|
||||
MAX_WAIT_SECONDS = 600
|
||||
|
||||
|
||||
def run_test(more_args):
|
||||
"""Run the end to end accuracy test."""
|
||||
args = list(DEFAULT_ARGS)
|
||||
args.extend(more_args)
|
||||
print(f"Running with: {args}")
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, args, max_wait_seconds=MAX_WAIT_SECONDS
|
||||
) as remote_server:
|
||||
url = f"{remote_server.url_for('v1')}/completions"
|
||||
|
||||
model_args = (
|
||||
f"model={MODEL_NAME},"
|
||||
f"base_url={url},"
|
||||
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="local-completions",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
)
|
||||
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
assert (
|
||||
measured_value - RTOL < EXPECTED_VALUE
|
||||
and measured_value + RTOL > EXPECTED_VALUE
|
||||
), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
|
||||
|
||||
|
||||
def test_lm_eval_accuracy_v1_engine():
|
||||
"""Run with the V1 Engine."""
|
||||
more_args = []
|
||||
|
||||
# Limit compilation time for V1 on TPU
|
||||
# Avoid OOM on XPU
|
||||
if current_platform.is_tpu() or current_platform.is_xpu():
|
||||
more_args = ["--max-num-seqs", "64"]
|
||||
|
||||
run_test(more_args)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import schemathesis
|
||||
from fastapi.testclient import TestClient
|
||||
from hypothesis import HealthCheck, settings
|
||||
from jsonschema import Draft202012Validator
|
||||
from schemathesis import GenerationMode
|
||||
from schemathesis.checks import not_a_server_error
|
||||
from schemathesis.config import (
|
||||
ChecksConfig,
|
||||
CoveragePhaseConfig,
|
||||
GenerationConfig,
|
||||
PhasesConfig,
|
||||
PositiveDataAcceptanceConfig,
|
||||
ProjectConfig,
|
||||
ProjectsConfig,
|
||||
SchemathesisConfig,
|
||||
)
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
MAXIMUM_IMAGES = 2
|
||||
_ROCM_TIMEOUT_MULTIPLIER = 3 if current_platform.is_rocm() else 1
|
||||
DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 * _ROCM_TIMEOUT_MULTIPLIER
|
||||
LONG_TIMEOUT_SECONDS: Final[int] = 60 * _ROCM_TIMEOUT_MULTIPLIER
|
||||
|
||||
|
||||
def test_openapi_exposes_completion_response_schemas() -> None:
|
||||
"""The published 200 contract of both completion APIs must reference the
|
||||
real response models, for the JSON body and for each SSE data chunk.
|
||||
|
||||
Uses the production app: FastAPI renames models used as both input and
|
||||
output (-Input/-Output), which a router-only app cannot show.
|
||||
"""
|
||||
from vllm.entrypoints.launchers.app import build_app
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
|
||||
args = Namespace(
|
||||
disable_fastapi_docs=False,
|
||||
enable_offline_docs=False,
|
||||
root_path="",
|
||||
allowed_origins=["*"],
|
||||
allow_credentials=False,
|
||||
allowed_methods=["*"],
|
||||
allowed_headers=["*"],
|
||||
api_key=None,
|
||||
enable_request_id_headers=False,
|
||||
enable_fault_tolerance=False,
|
||||
middleware=[],
|
||||
log_error_stack=False,
|
||||
)
|
||||
# No lifespan: schema generation does not need an inference engine.
|
||||
client = TestClient(build_app(args, supported_tasks=("generate",)))
|
||||
document = client.get("/openapi.json").json()
|
||||
message = {"role": "assistant", "content": "hi"}
|
||||
for path, unary, unary_choice, chunk, chunk_choice in (
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
ChatCompletionResponse,
|
||||
{"index": 0, "message": message},
|
||||
ChatCompletionStreamResponse,
|
||||
{"index": 0, "delta": {"content": "hi"}},
|
||||
),
|
||||
(
|
||||
"/v1/completions",
|
||||
CompletionResponse,
|
||||
{"index": 0, "text": "hi"},
|
||||
CompletionStreamResponse,
|
||||
{"index": 0, "text": "hi"},
|
||||
),
|
||||
):
|
||||
content = document["paths"][path]["post"]["responses"]["200"]["content"]
|
||||
data = content["text/event-stream"]["schema"]["properties"]["data"]
|
||||
assert data["anyOf"][0] == {"const": "[DONE]"}
|
||||
for schema, model, choice in (
|
||||
(content["application/json"]["schema"], unary, unary_choice),
|
||||
(data["anyOf"][1]["contentSchema"], chunk, chunk_choice),
|
||||
):
|
||||
assert schema == {"$ref": f"#/components/schemas/{model.__name__}"}
|
||||
validator = Draft202012Validator(
|
||||
{**schema, "components": document["components"]}
|
||||
)
|
||||
payload = model(model="m", choices=[choice], usage={}).model_dump(
|
||||
mode="json"
|
||||
)
|
||||
validator.validate(payload)
|
||||
payload["choices"] = "not an array"
|
||||
assert not validator.is_valid(payload)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"generate",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"5",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"image": MAXIMUM_IMAGES}),
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def get_schema(server):
|
||||
# avoid generating null (\x00) bytes in strings during test case generation
|
||||
return schemathesis.openapi.from_url(
|
||||
f"{server.url_root}/openapi.json",
|
||||
config=SchemathesisConfig(
|
||||
projects=ProjectsConfig(
|
||||
default=ProjectConfig(
|
||||
generation=GenerationConfig(
|
||||
allow_x00=False,
|
||||
modes=[GenerationMode.POSITIVE],
|
||||
),
|
||||
checks=ChecksConfig(
|
||||
positive_data_acceptance=PositiveDataAcceptanceConfig(
|
||||
enabled=False,
|
||||
),
|
||||
),
|
||||
phases=PhasesConfig(
|
||||
coverage=CoveragePhaseConfig(enabled=False),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
schema = schemathesis.pytest.from_fixture("get_schema")
|
||||
|
||||
|
||||
@schemathesis.hook
|
||||
def before_generate_case(context: schemathesis.HookContext, strategy):
|
||||
op = context.operation
|
||||
assert op is not None
|
||||
|
||||
def no_invalid_types(case: schemathesis.Case):
|
||||
"""Skips tool_calls with `"type": "custom"` which schemathesis incorrectly
|
||||
generates instead of the valid `"type": "function"`.
|
||||
|
||||
Example test case that is skipped:
|
||||
curl -X POST -H 'Content-Type: application/json' \
|
||||
-d '{"messages": [{"role": "assistant", "tool_calls": [{"custom": {"input": "", "name": ""}, "id": "", "type": "custom"}]}]}' \
|
||||
http://localhost:8000/v1/chat/completions
|
||||
""" # noqa: E501
|
||||
if (
|
||||
hasattr(case, "body")
|
||||
and isinstance(case.body, dict)
|
||||
and "messages" in case.body
|
||||
and isinstance(case.body["messages"], list)
|
||||
and len(case.body["messages"]) > 0
|
||||
):
|
||||
for message in case.body["messages"]:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
|
||||
tool_calls = message.get("tool_calls", [])
|
||||
if isinstance(tool_calls, list):
|
||||
for tool_call in tool_calls:
|
||||
if isinstance(tool_call, dict):
|
||||
if tool_call.get("type") != "function":
|
||||
return False
|
||||
if "custom" in tool_call:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
return strategy.filter(no_invalid_types)
|
||||
|
||||
|
||||
@schema.parametrize()
|
||||
@settings(
|
||||
deadline=LONG_TIMEOUT_SECONDS * 1000,
|
||||
max_examples=50,
|
||||
# Under CI's derandomized hypothesis seed, the schemathesis strategy
|
||||
# for /v1/chat/completions/batch's nested-message body, combined with
|
||||
# the no_invalid_types filter (notably the grammar=="" rule), exceeds
|
||||
# the default filtered-vs-good ratio. The filter is intentional, so
|
||||
# suppress the health check rather than drop the filter — dropping it
|
||||
# exposes pre-existing server bugs out of scope here.
|
||||
# The same nested schema can also trip Hypothesis' entropy budget while
|
||||
# generating large-but-valid request bodies before vLLM is called.
|
||||
suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.data_too_large],
|
||||
)
|
||||
def test_openapi_stateless(case: schemathesis.Case):
|
||||
key = (
|
||||
case.operation.method.upper(),
|
||||
case.operation.path,
|
||||
)
|
||||
if case.operation.path.startswith("/v1/responses"):
|
||||
# Skip responses API as it is meant to be stateful.
|
||||
return
|
||||
|
||||
# Skip weight transfer endpoints as they require special setup
|
||||
# (weight_transfer_config) and are meant to be stateful.
|
||||
if case.operation.path in (
|
||||
"/init_weight_transfer_engine",
|
||||
"/start_weight_update",
|
||||
"/start_draft_weight_update",
|
||||
"/update_weights",
|
||||
"/finish_weight_update",
|
||||
"/update_weight_version",
|
||||
):
|
||||
return
|
||||
|
||||
timeout = {
|
||||
# requires a longer timeout
|
||||
("POST", "/v1/chat/completions"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/inference/v1/generate"): LONG_TIMEOUT_SECONDS,
|
||||
}.get(key, DEFAULT_TIMEOUT_SECONDS)
|
||||
|
||||
# No need to verify SSL certificate for localhost
|
||||
unsupported_decisions = case.operation.path in {
|
||||
"/v1/systemone",
|
||||
"/v1/decisions",
|
||||
}
|
||||
response = case.call_and_validate(
|
||||
verify=False,
|
||||
timeout=timeout,
|
||||
headers={"Content-Type": "application/json"},
|
||||
excluded_checks=[not_a_server_error] if unsupported_decisions else None,
|
||||
)
|
||||
if unsupported_decisions:
|
||||
# SmolVLM does not support either decision endpoint.
|
||||
assert response.status_code < 500 or response.status_code == 501
|
||||
@@ -1,368 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
from ...utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--enforce-eager",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("return_token_ids", [True, False, None])
|
||||
async def test_basic_completion_with_emoji(server, return_token_ids: bool | None):
|
||||
"""Test basic completion with emoji to verify token_ids field."""
|
||||
extra_body = None
|
||||
if return_token_ids is not None:
|
||||
extra_body = {"return_token_ids": return_token_ids}
|
||||
async with server.get_async_client() as client:
|
||||
# Test with return_token_ids enabled
|
||||
completion = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Complete this sentence with emojis: I love coding 🚀",
|
||||
max_tokens=10,
|
||||
temperature=0,
|
||||
logprobs=1,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
|
||||
# Check the raw response to see the structure
|
||||
completion_dict = completion.model_dump()
|
||||
|
||||
# Verify prompt_token_ids field is present in the completion response
|
||||
assert "prompt_token_ids" in completion_dict["choices"][0]
|
||||
if not return_token_ids:
|
||||
# If return_token_ids is False, token_ids should not be present
|
||||
assert completion_dict["choices"][0].get("token_ids") is None
|
||||
assert completion_dict["choices"][0].get("prompt_token_ids") is None
|
||||
# Skip further checks
|
||||
return
|
||||
assert isinstance(completion.choices[0].prompt_token_ids, list)
|
||||
|
||||
# Check against the expected prompt token IDs
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
encoded_tokens = tokenizer.encode(
|
||||
"Complete this sentence with emojis: I love coding 🚀"
|
||||
)
|
||||
# Check that encoded_tokens is a subsequence of prompt_token_ids
|
||||
assert any(
|
||||
completion.choices[0].prompt_token_ids[i : i + len(encoded_tokens)]
|
||||
== encoded_tokens
|
||||
for i in range(
|
||||
len(completion.choices[0].prompt_token_ids) - len(encoded_tokens) + 1
|
||||
)
|
||||
)
|
||||
|
||||
# Verify token_ids field is present in the choice
|
||||
assert completion.choices[0].token_ids is not None
|
||||
assert isinstance(completion.choices[0].token_ids, list)
|
||||
assert len(completion.choices[0].token_ids) > 0
|
||||
|
||||
# Verify decoding works correctly
|
||||
decoded_text = tokenizer.decode(completion.choices[0].token_ids)
|
||||
# The decoded text should contain a <|im_end|> at the end
|
||||
assert decoded_text.startswith(completion.choices[0].text)
|
||||
|
||||
# Test without return_token_ids (should be None)
|
||||
completion_without = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Complete this sentence with emojis: I love coding 🚀",
|
||||
max_tokens=10,
|
||||
temperature=0,
|
||||
logprobs=1,
|
||||
extra_body={"return_token_ids": False},
|
||||
)
|
||||
|
||||
completion_without_dict = completion_without.model_dump()
|
||||
assert completion_without_dict["choices"][0].get("token_ids") is None
|
||||
assert completion_without_dict.get("prompt_token_ids") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_with_tool_use(server):
|
||||
"""Test chat completion with tool use (get_weather function)."""
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
"description": "The unit of temperature",
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
async with server.get_async_client() as client:
|
||||
# Test with return_token_ids enabled
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What's the weather like in Paris?"},
|
||||
],
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
max_tokens=100,
|
||||
temperature=0,
|
||||
logprobs=True,
|
||||
extra_body={"return_token_ids": True},
|
||||
)
|
||||
|
||||
# Verify token_ids field is present in choices
|
||||
assert response.choices[0].token_ids is not None
|
||||
assert isinstance(response.choices[0].token_ids, list)
|
||||
|
||||
# Verify prompt_token_ids field is present
|
||||
assert response.prompt_token_ids is not None
|
||||
assert isinstance(response.prompt_token_ids, list)
|
||||
|
||||
# Verify the prompt texts and response texts
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
prompt_text = tokenizer.decode(response.prompt_token_ids)
|
||||
assert prompt_text.startswith(
|
||||
"<|im_start|>system\nYou are a helpful assistant."
|
||||
)
|
||||
assert prompt_text.endswith(
|
||||
"What's the weather like in Paris?<|im_end|>\n<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
response_text = tokenizer.decode(response.choices[0].token_ids)
|
||||
assert response_text.startswith('<tool_call>\n{"name": "get_weather"')
|
||||
assert response_text.endswith("</tool_call><|im_end|>")
|
||||
|
||||
# If tool call was made, verify the response structure
|
||||
if response.choices[0].message.tool_calls:
|
||||
assert len(response.choices[0].message.tool_calls) > 0
|
||||
tool_call = response.choices[0].message.tool_calls[0]
|
||||
assert tool_call.function.name == "get_weather"
|
||||
|
||||
# Test without return_token_ids
|
||||
response_without = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What's the weather like in Paris?"},
|
||||
],
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
max_tokens=100,
|
||||
temperature=0,
|
||||
logprobs=True,
|
||||
extra_body={"return_token_ids": False},
|
||||
)
|
||||
|
||||
assert response_without.choices[0].token_ids is None
|
||||
assert response_without.prompt_token_ids is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_comparison_with_prompt_logprobs_and_logprobs(server):
|
||||
"""Test that token_ids align with prompt_logprobs and
|
||||
logprobs when return_tokens_as_token_ids is enabled.
|
||||
"""
|
||||
async with server.get_async_client() as client:
|
||||
# Test with both return_token_ids and return_tokens_as_token_ids enabled
|
||||
completion = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Hello, world! How are you today?",
|
||||
max_tokens=20,
|
||||
temperature=0,
|
||||
echo=True,
|
||||
logprobs=1,
|
||||
extra_body={
|
||||
"return_token_ids": True,
|
||||
"return_tokens_as_token_ids": True,
|
||||
"prompt_logprobs": 1,
|
||||
},
|
||||
)
|
||||
|
||||
# Verify all fields are present
|
||||
assert completion.choices[0].token_ids is not None
|
||||
assert completion.choices[0].prompt_token_ids is not None
|
||||
assert completion.choices[0].prompt_logprobs is not None
|
||||
assert completion.choices[0].logprobs is not None
|
||||
|
||||
# Extract token IDs from logprobs
|
||||
# (when return_tokens_as_token_ids is True)
|
||||
logprobs_token_ids = []
|
||||
for token_str in completion.choices[0].logprobs.tokens:
|
||||
# Token format is "token_id:12345" when
|
||||
# return_tokens_as_token_ids is True
|
||||
if token_str.startswith("token_id:"):
|
||||
token_id = int(token_str.removeprefix("token_id:"))
|
||||
logprobs_token_ids.append(token_id)
|
||||
|
||||
# When echo=True, the logprobs include both prompt and response tokens
|
||||
# The token_ids field should match the suffix of response portion
|
||||
# The prompt_token_ids should match the prompt portion
|
||||
assert len(completion.choices[0].token_ids) < len(logprobs_token_ids)
|
||||
response_token_ids_length = len(completion.choices[0].token_ids)
|
||||
assert (
|
||||
logprobs_token_ids[-response_token_ids_length:]
|
||||
== completion.choices[0].token_ids
|
||||
)
|
||||
|
||||
# Verify tokenizer consistency
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
|
||||
# Decode prompt tokens
|
||||
if completion.choices[0].prompt_token_ids:
|
||||
prompt_text = tokenizer.decode(completion.choices[0].prompt_token_ids)
|
||||
# The decoded prompt should match or close to original prompt
|
||||
assert "Hello, world" in prompt_text
|
||||
|
||||
# Decode response tokens
|
||||
if completion.choices[0].token_ids:
|
||||
response_text = tokenizer.decode(completion.choices[0].token_ids)
|
||||
assert completion.choices[0].text.endswith(response_text)
|
||||
|
||||
# Test streaming mode
|
||||
stream = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Tell me a short fact about Python:",
|
||||
max_tokens=30,
|
||||
temperature=0,
|
||||
stream=True,
|
||||
echo=False,
|
||||
logprobs=1,
|
||||
extra_body={"return_token_ids": True, "return_tokens_as_token_ids": True},
|
||||
)
|
||||
|
||||
# Collect streamed tokens
|
||||
streamed_prompt_token_ids = []
|
||||
streamed_token_ids = []
|
||||
streamed_logprob_token_ids = []
|
||||
first_chunk = True
|
||||
async for chunk in stream:
|
||||
for token_str in chunk.choices[0].logprobs.tokens:
|
||||
# Token format is "token_id:12345" when
|
||||
# return_tokens_as_token_ids is True
|
||||
if token_str.startswith("token_id:"):
|
||||
token_id = int(token_str.removeprefix("token_id:"))
|
||||
streamed_logprob_token_ids.append(token_id)
|
||||
if first_chunk:
|
||||
streamed_prompt_token_ids = chunk.choices[0].prompt_token_ids
|
||||
first_chunk = False
|
||||
streamed_token_ids += chunk.choices[0].token_ids
|
||||
|
||||
# Verify we collected some tokens and first chunk had prompt_token_ids
|
||||
assert len(streamed_prompt_token_ids) > 0
|
||||
assert streamed_token_ids == streamed_logprob_token_ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_with_emoji_and_token_ids(server):
|
||||
"""Test chat completion with emojis to verify token_ids handling."""
|
||||
chat_messages = [
|
||||
{"role": "system", "content": "You like to use emojis in your responses."},
|
||||
{"role": "user", "content": "Repeat after me: I love cats 🐱"},
|
||||
]
|
||||
async with server.get_async_client() as client:
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=chat_messages,
|
||||
max_tokens=50,
|
||||
temperature=0,
|
||||
logprobs=True,
|
||||
extra_body={"return_token_ids": True},
|
||||
)
|
||||
|
||||
# Verify token_ids are present
|
||||
response_dict = response.model_dump()
|
||||
assert response.choices[0].token_ids is not None
|
||||
assert "prompt_token_ids" in response_dict
|
||||
|
||||
# Verify the response contains the expected fields
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
# Decode token_ids and verify consistency
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
|
||||
decoded_prompt = tokenizer.decode(response.prompt_token_ids)
|
||||
assert decoded_prompt.startswith(
|
||||
"<|im_start|>system\nYou like to use emojis in your responses."
|
||||
)
|
||||
assert decoded_prompt.endswith(
|
||||
"I love cats 🐱<|im_end|>\n<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
decoded_response = tokenizer.decode(response.choices[0].token_ids)
|
||||
# The content should match the response text
|
||||
# except the ending <|im_end|>
|
||||
assert decoded_response == response.choices[0].message.content + "<|im_end|>"
|
||||
|
||||
# Test with streaming
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=chat_messages,
|
||||
max_tokens=50,
|
||||
temperature=0,
|
||||
stream=True,
|
||||
extra_body={"return_token_ids": True},
|
||||
)
|
||||
|
||||
collected_content = ""
|
||||
collected_token_ids = []
|
||||
first_chunk = True
|
||||
|
||||
async for chunk in stream:
|
||||
if first_chunk:
|
||||
assert chunk.prompt_token_ids is not None
|
||||
assert isinstance(chunk.prompt_token_ids, list)
|
||||
# Check the prompt_token_ids match the initial prompt
|
||||
decoded_prompt_stream = tokenizer.decode(chunk.prompt_token_ids)
|
||||
assert decoded_prompt_stream == decoded_prompt
|
||||
first_chunk = False
|
||||
else:
|
||||
chunk_dump = chunk.model_dump()
|
||||
assert "prompt_token_ids" not in chunk_dump, (
|
||||
"Subsequent chunks should not have prompt_token_ids"
|
||||
)
|
||||
|
||||
if chunk.choices:
|
||||
if chunk.choices[0].delta.content:
|
||||
collected_content += chunk.choices[0].delta.content
|
||||
# token_ids may not present in all chunks
|
||||
choice_dump = chunk.choices[0].model_dump()
|
||||
if "token_ids" in choice_dump:
|
||||
collected_token_ids.extend(chunk.choices[0].token_ids)
|
||||
|
||||
# Verify we got response and token_ids
|
||||
assert len(collected_content) > 0
|
||||
assert len(collected_token_ids) > 0
|
||||
|
||||
# Verify token_ids decode properly
|
||||
decoded_response = tokenizer.decode(collected_token_ids)
|
||||
assert decoded_response == collected_content + "<|im_end|>"
|
||||
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Separate these tests out from test_completion and test_chat, because they
|
||||
# require launching a second server with a different flag. Running both servers
|
||||
# at the same time on a single node will OOM.
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
from ...utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args(qwen3_lora_files):
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
# lora config
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"qwen3-lora={qwen3_lora_files}",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_fixture(request, default_server_args):
|
||||
use_server_flag = request.param
|
||||
if use_server_flag:
|
||||
args_with_flag = default_server_args + ["--return-tokens-as-token-ids"]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args_with_flag) as remote_server:
|
||||
yield (remote_server, True)
|
||||
else:
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield (remote_server, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("server_fixture", [True, False], indirect=True)
|
||||
async def test_completion_return_tokens_as_token_ids_completion(server_fixture):
|
||||
server, use_server_flag = server_fixture
|
||||
request_args = {}
|
||||
if not use_server_flag:
|
||||
request_args["return_tokens_as_token_ids"] = True
|
||||
|
||||
async with server.get_async_client() as client:
|
||||
completion = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
# Include Unicode characters to test for dividing a single
|
||||
# character across multiple tokens: 🎉 is [28705, 31862] for the
|
||||
# Zephyr tokenizer
|
||||
prompt="Say 'Hello, world! 🎉'",
|
||||
echo=True,
|
||||
temperature=0,
|
||||
max_tokens=10,
|
||||
logprobs=1,
|
||||
extra_body=request_args,
|
||||
)
|
||||
|
||||
text = completion.choices[0].text
|
||||
token_strs = completion.choices[0].logprobs.tokens
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
# Check that the token representations are consistent between raw
|
||||
# tokens and top_logprobs
|
||||
# Slice off the first one, because there's no scoring associated
|
||||
# with BOS
|
||||
top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
|
||||
top_logprob_keys = [
|
||||
next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
|
||||
]
|
||||
assert token_strs[1:] == top_logprob_keys
|
||||
|
||||
# Check that decoding the tokens gives the expected text
|
||||
tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
|
||||
assert text == tokenizer.decode(tokens, skip_special_tokens=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("server_fixture", [True, False], indirect=True)
|
||||
async def test_chat_return_tokens_as_token_ids_completion(server_fixture):
|
||||
server, use_server_flag = server_fixture
|
||||
request_args = {}
|
||||
if not use_server_flag:
|
||||
request_args["return_tokens_as_token_ids"] = True
|
||||
|
||||
async with server.get_async_client() as client:
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
# Include Unicode characters to test for dividing a single
|
||||
# character across multiple tokens: 🎉 is [28705, 31862] for the
|
||||
# Zephyr tokenizer
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You like to respond in only emojis, like 🎉",
|
||||
},
|
||||
{"role": "user", "content": "Please write some emojis: 🐱🐶🎉"},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=8,
|
||||
logprobs=True,
|
||||
extra_body=request_args,
|
||||
)
|
||||
|
||||
text = response.choices[0].message.content
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
token_ids = []
|
||||
for logprob_content in response.choices[0].logprobs.content:
|
||||
token_ids.append(int(logprob_content.token.removeprefix("token_id:")))
|
||||
assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
|
||||
|
||||
|
||||
def test_responses_api_logprobs_with_return_tokens_as_token_ids():
|
||||
"""Test that return_tokens_as_token_ids works in Responses API logprobs."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
|
||||
serving = MagicMock(spec=OpenAIServingResponses)
|
||||
serving.return_tokens_as_token_ids = True
|
||||
serving._get_decoded_token = GenerateBaseServing._get_decoded_token
|
||||
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode = lambda token_id: "decoded"
|
||||
|
||||
token_ids = [100, 200, 300]
|
||||
sample_logprobs = [
|
||||
{100: SampleLogprob(logprob=-0.5, decoded_token="hello")},
|
||||
{200: SampleLogprob(logprob=-1.2, decoded_token="world")},
|
||||
{300: SampleLogprob(logprob=-0.8, decoded_token="!")},
|
||||
]
|
||||
|
||||
result = OpenAIServingResponses._create_response_logprobs(
|
||||
serving,
|
||||
token_ids=token_ids,
|
||||
logprobs=sample_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
top_logprobs=1,
|
||||
)
|
||||
|
||||
assert len(result) == 3
|
||||
assert result[0].token == "token_id:100"
|
||||
assert result[1].token == "token_id:200"
|
||||
assert result[2].token == "token_id:300"
|
||||
assert result[0].logprob == -0.5
|
||||
assert result[1].logprob == -1.2
|
||||
assert result[2].logprob == -0.8
|
||||
@@ -1,249 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from fastapi import UploadFile
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
|
||||
def test_chat_request_preserves_watermarking_opt_out():
|
||||
request = ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}], watermarking=False
|
||||
)
|
||||
|
||||
params = request.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
def test_completion_request_preserves_watermarking_opt_out():
|
||||
request = CompletionRequest(prompt="hello", watermarking=False)
|
||||
|
||||
params = request.to_sampling_params(max_tokens=1)
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
def test_responses_request_preserves_watermarking_opt_out():
|
||||
request = ResponsesRequest(input="hello", watermarking=False)
|
||||
|
||||
params = request.to_sampling_params(default_max_tokens=1)
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
def test_batch_chat_request_preserves_watermarking_opt_out():
|
||||
request = BatchChatCompletionRequest(
|
||||
messages=[[{"role": "user", "content": "hello"}]], watermarking=False
|
||||
)
|
||||
|
||||
converted = request.to_chat_completion_request(request.messages[0])
|
||||
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_request,conversion,kwargs",
|
||||
[
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
use_beam_search=True,
|
||||
watermarking=False,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=False),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
TranscriptionRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranscriptionRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranslationRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranslationRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=False,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_preserves_watermarking_opt_out(api_request, conversion, kwargs):
|
||||
params = getattr(api_request, conversion)(**kwargs)
|
||||
|
||||
assert params.watermarking is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_cls", [TranscriptionRequest, TranslationRequest])
|
||||
def test_speech_requests_leave_watermarking_unspecified(request_cls):
|
||||
request = request_cls(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
temperature=0.8,
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(default_max_tokens=1)
|
||||
beam_params = request.to_beam_search_params(default_max_tokens=1)
|
||||
|
||||
assert request.watermarking is None
|
||||
assert sampling_params.watermarking is None
|
||||
assert beam_params.watermarking is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_request,kwargs",
|
||||
[
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "choose A or B"}],
|
||||
temperature=0.8,
|
||||
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
|
||||
),
|
||||
{"max_tokens": 8, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
CompletionRequest(
|
||||
prompt="choose A or B",
|
||||
temperature=0.8,
|
||||
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
|
||||
),
|
||||
{"max_tokens": 8},
|
||||
),
|
||||
(
|
||||
ResponsesRequest(
|
||||
input="choose A or B",
|
||||
temperature=0.8,
|
||||
structured_outputs=StructuredOutputsParams(choice=["A", "B"]),
|
||||
),
|
||||
{"default_max_tokens": 8},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_preserves_unspecified_watermarking_with_structured_outputs(
|
||||
api_request, kwargs
|
||||
):
|
||||
params = api_request.to_sampling_params(**kwargs)
|
||||
|
||||
assert params.watermarking is None
|
||||
assert params.structured_outputs is not None
|
||||
assert params.structured_outputs.choice == ["A", "B"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_request,conversion,kwargs",
|
||||
[
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}], watermarking=True
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
use_beam_search=True,
|
||||
watermarking=True,
|
||||
),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1, "default_sampling_params": {}},
|
||||
),
|
||||
(
|
||||
CompletionRequest(prompt="hello", watermarking=True),
|
||||
"to_sampling_params",
|
||||
{"max_tokens": 1},
|
||||
),
|
||||
(
|
||||
CompletionRequest(prompt="hello", use_beam_search=True, watermarking=True),
|
||||
"to_beam_search_params",
|
||||
{"max_tokens": 1},
|
||||
),
|
||||
(
|
||||
ResponsesRequest(input="hello", watermarking=True),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranscriptionRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=True,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
(
|
||||
TranslationRequest(
|
||||
file=UploadFile(file=BytesIO(), filename="audio.wav"),
|
||||
watermarking=True,
|
||||
),
|
||||
"to_sampling_params",
|
||||
{"default_max_tokens": 1},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_preserves_explicit_watermarking(api_request, conversion, kwargs):
|
||||
params = getattr(api_request, conversion)(**kwargs)
|
||||
|
||||
assert params.watermarking is True
|
||||
|
||||
|
||||
def test_batch_chat_request_preserves_explicit_watermarking():
|
||||
request = BatchChatCompletionRequest(
|
||||
messages=[[{"role": "user", "content": "hello"}]], watermarking=True
|
||||
)
|
||||
|
||||
converted = request.to_chat_completion_request(request.messages[0])
|
||||
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert converted.watermarking is True
|
||||
assert params.watermarking is True
|
||||
|
||||
|
||||
def test_batch_chat_request_inherits_watermarking():
|
||||
request = BatchChatCompletionRequest(
|
||||
messages=[[{"role": "user", "content": "hello"}]]
|
||||
)
|
||||
|
||||
converted = request.to_chat_completion_request(request.messages[0])
|
||||
params = converted.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert converted.watermarking is None
|
||||
assert params.watermarking is None
|
||||
Reference in New Issue
Block a user