[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,558 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
BASE_MODEL_PATHS,
|
||||
CHAT_TEMPLATE,
|
||||
MockHFConfig,
|
||||
MockModelConfig,
|
||||
_build_renderer,
|
||||
)
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
MODEL_NAME as MOCK_MODEL_NAME,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
# any model with a chat template defined in tokenizer_config should work here
|
||||
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
[{"role": "user", "content": "Reply with exactly the word: beta"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
choices = data["choices"]
|
||||
assert len(choices) == 2
|
||||
|
||||
indices = {choice["index"] for choice in choices}
|
||||
assert indices == {0, 1}
|
||||
|
||||
# Each conversation should produce a non-empty text response.
|
||||
for choice in choices:
|
||||
assert choice["message"]["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_with_json_schema(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"answer": {"type": "string", "enum": ["yes", "no"]},
|
||||
},
|
||||
"required": ["answer"],
|
||||
}
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Is the sky blue? Answer in JSON."}],
|
||||
[{"role": "user", "content": "Is fire cold? Answer in JSON."}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "answer", "schema": schema, "strict": True},
|
||||
},
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
choices = data["choices"]
|
||||
assert len(choices) == 2
|
||||
|
||||
for choice in choices:
|
||||
parsed = json.loads(choice["message"]["content"])
|
||||
assert "answer" in parsed
|
||||
assert parsed["answer"] in ("yes", "no")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_logprobs_not_token_id_placeholders(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
# Regression test: requesting `return_token_ids` alongside logprobs must not
|
||||
# corrupt the logprob `token` fields into "token_id:{id}" placeholders. That
|
||||
# placeholder rendering is controlled by `return_tokens_as_token_ids`, which
|
||||
# this request leaves unset.
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 1,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
for entry in content:
|
||||
assert not entry["token"].startswith("token_id:")
|
||||
for top in entry["top_logprobs"]:
|
||||
assert not top["token"].startswith("token_id:")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_batched_chat_completions_return_tokens_as_token_ids(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
# Complementary check: when `return_tokens_as_token_ids` is explicitly set,
|
||||
# the logprob tokens *should* be rendered as "token_id:{id}" placeholders,
|
||||
# proving the new field is actually wired through.
|
||||
conversations = [
|
||||
[{"role": "user", "content": "Reply with exactly the word: alpha"}],
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": conversations,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 1,
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
assert all(entry["token"].startswith("token_id:") for entry in content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batched_chat_completions_logprob_token_ids(
|
||||
server: RemoteOpenAIServer,
|
||||
) -> None:
|
||||
conversations = [[{"role": "user", "content": "Hello"}]]
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
response = await http_client.post(
|
||||
f"{server.url_for('v1/chat/completions/batch')}",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": conversations,
|
||||
"max_tokens": 1,
|
||||
"temperature": 0,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 5,
|
||||
"logprob_token_ids": [100, 1000, 5000],
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
content = response.json()["choices"][0]["logprobs"]["content"]
|
||||
assert content
|
||||
sampled_token = content[0]["token"]
|
||||
assert {entry["token"] for entry in content[0]["top_logprobs"]} == {
|
||||
"token_id:100",
|
||||
"token_id:1000",
|
||||
"token_id:5000",
|
||||
sampled_token,
|
||||
}
|
||||
|
||||
|
||||
def _make_request_output(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=f"req-{prompt_idx}",
|
||||
prompt=None,
|
||||
prompt_token_ids=prompt_token_ids or [1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=[4, 5],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
async def _generator(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _make_request_output(prompt_idx, text, prompt_token_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
|
||||
"""Echo must only echo the assistant turn, never the user's prompt.
|
||||
|
||||
With `add_generation_prompt` (the default) the response role is `assistant`
|
||||
while the last conversation message is the user's, so there is nothing to
|
||||
echo; without the role check the user prompt leaks into the answer.
|
||||
"""
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
|
||||
conversations = [[{"role": "user", "content": "USER PROMPT"}]]
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=conversations,
|
||||
echo=True,
|
||||
)
|
||||
|
||||
response = await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "ASSISTANT ANSWER")],
|
||||
request_id="req-echo",
|
||||
model_name="test-model",
|
||||
all_conversations=conversations,
|
||||
tokenizer=None,
|
||||
request_metadata=RequestResponseMetadata(request_id="req-echo"),
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "ASSISTANT ANSWER"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
|
||||
"""A trailing assistant turn is a real prefix and must still be echoed."""
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
|
||||
conversations = [
|
||||
[
|
||||
{"role": "user", "content": "USER PROMPT"},
|
||||
{"role": "assistant", "content": "PREFIX "},
|
||||
]
|
||||
]
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=conversations,
|
||||
echo=True,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
|
||||
response = await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "ASSISTANT ANSWER")],
|
||||
request_id="req-echo-2",
|
||||
model_name="test-model",
|
||||
all_conversations=conversations,
|
||||
tokenizer=None,
|
||||
request_metadata=RequestResponseMetadata(request_id="req-echo-2"),
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
|
||||
"""Harmony must constrain only the final channel of every conversation."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.errored = False
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
|
||||
|
||||
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="openai_gptoss",
|
||||
)
|
||||
serving = OpenAIServingChatBatch(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
)
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=[
|
||||
[{"role": "user", "content": "France?"}],
|
||||
[{"role": "user", "content": "Japan?"}],
|
||||
],
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "city", "schema": {"type": "object"}},
|
||||
},
|
||||
)
|
||||
await serving.create_batch_chat_completion(request)
|
||||
|
||||
calls = mock_engine.generate.call_args_list
|
||||
assert len(calls) == 2
|
||||
for call in calls:
|
||||
assert call.args[1].structured_outputs.structural_tag is not None
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_batch_rejects_kv_transfer_prompt_token_ids():
|
||||
"""One pre-tokenized prompt cannot stand in for every conversation."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match=r"parameter=kv_transfer_params\.prompt_token_ids"
|
||||
):
|
||||
BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
[{"role": "user", "content": "first"}],
|
||||
[{"role": "user", "content": "second"}],
|
||||
],
|
||||
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_parser_receives_each_prompt_prefix() -> None:
|
||||
prefixes = []
|
||||
|
||||
class RecordingParser:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prompt_token_ids = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids):
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
|
||||
def parse(self, model_output, **kwargs):
|
||||
prefixes.append(self.prompt_token_ids)
|
||||
return None, model_output, None
|
||||
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
serving.chat_template = None
|
||||
serving.chat_template_content_format = "auto"
|
||||
serving.default_chat_template_kwargs = {}
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
[{"role": "user", "content": "first"}],
|
||||
[{"role": "user", "content": "second"}],
|
||||
],
|
||||
)
|
||||
await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
|
||||
request_id="req-prefix",
|
||||
model_name="test-model",
|
||||
all_conversations=request.messages,
|
||||
tokenizer=object(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
|
||||
parser_cls=RecordingParser,
|
||||
)
|
||||
|
||||
assert prefixes == [[1], [2]]
|
||||
|
||||
|
||||
REASONING_PARSER = "kimi_k3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_reasoning_wiring_matches_single_chat() -> None:
|
||||
"""`/v1/chat/completions/batch` must hand the engine the same reasoning
|
||||
wiring as `/v1/chat/completions`.
|
||||
|
||||
Both `reasoning_ended` and `reasoning_parser_kwargs` are carried on the
|
||||
request into `EngineCoreRequest`, where
|
||||
`StructuredOutputManager._get_reasoner()` rebuilds the request-local
|
||||
reasoning parser from the forwarded `chat_template_kwargs` -- "so the
|
||||
structured-output gate observes the same template kwargs used by the
|
||||
frontend". A divergence makes a template-kwarg-driven parser fall back to
|
||||
its default (`thinking` on) for a request that disabled thinking, so
|
||||
`is_reasoning_end()` keeps reporting False, the structured-output gate
|
||||
never opens, and the grammar is silently never applied.
|
||||
|
||||
Both frontends run over one and the same model and renderer, so the only
|
||||
thing that can differ between the two calls is the serving code itself.
|
||||
"""
|
||||
mock_model_config = MockModelConfig()
|
||||
shared_renderer = _build_renderer(mock_model_config)
|
||||
|
||||
def _build_serving(cls):
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.model_config = mock_model_config
|
||||
mock_engine.errored = False
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = shared_renderer
|
||||
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
|
||||
|
||||
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser=REASONING_PARSER,
|
||||
)
|
||||
return mock_engine, cls(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
reasoning_parser=REASONING_PARSER,
|
||||
)
|
||||
|
||||
conversation = [{"role": "user", "content": "What is the capital of France?"}]
|
||||
chat_template_kwargs = {"enable_thinking": False}
|
||||
|
||||
single_engine, single_serving = _build_serving(OpenAIServingChat)
|
||||
await single_serving.create_chat_completion(
|
||||
ChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=conversation,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
single_call = single_engine.generate.call_args_list[-1]
|
||||
|
||||
batch_engine, batch_serving = _build_serving(OpenAIServingChatBatch)
|
||||
await batch_serving.create_batch_chat_completion(
|
||||
BatchChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=[conversation],
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
batch_call = batch_engine.generate.call_args_list[-1]
|
||||
|
||||
# Guard against a vacuous comparison: the reference path must be wired up.
|
||||
assert single_call.kwargs["reasoning_ended"] is not None
|
||||
assert single_call.kwargs["reasoning_parser_kwargs"] is not None
|
||||
|
||||
assert batch_call.kwargs["reasoning_ended"] == single_call.kwargs["reasoning_ended"]
|
||||
assert (
|
||||
batch_call.kwargs["reasoning_parser_kwargs"]
|
||||
== single_call.kwargs["reasoning_parser_kwargs"]
|
||||
)
|
||||
|
||||
# ...and the forwarded kwargs must actually turn thinking off for the
|
||||
# engine-side parser, i.e. the structured-output gate opens right away.
|
||||
engine_chat_template_kwargs = batch_call.kwargs["reasoning_parser_kwargs"][
|
||||
"chat_template_kwargs"
|
||||
]
|
||||
reasoner = ReasoningParserManager.get_reasoning_parser(REASONING_PARSER)(
|
||||
tokenizer=batch_engine.renderer.tokenizer,
|
||||
chat_template_kwargs=engine_chat_template_kwargs,
|
||||
)
|
||||
assert reasoner.is_reasoning_end([]) is True
|
||||
@@ -1,298 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for `prompt_embeds` content parts in the Chat Completions API."""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
|
||||
import openai
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import torch
|
||||
from openai import BadRequestError
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
CHAT_TEMPLATE = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
# Matches `--dtype` in `server_args` to avoid an implicit cast in
|
||||
# `safe_load_prompt_embeds` (mismatched floating-point dtypes are cast to the
|
||||
# model's dtype automatically, we match here just to skip the conversion).
|
||||
SERVER_DTYPE: torch.dtype = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_args() -> list[str]:
|
||||
return [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--chat-template",
|
||||
str(CHAT_TEMPLATE),
|
||||
# Prompt Embeds server args
|
||||
"--enable-prompt-embeds",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(server_args, request):
|
||||
if current_platform.is_rocm():
|
||||
# Materialize HF embeddings before the server reserves ROCm VRAM.
|
||||
request.getfixturevalue("prompt_embeds_b64")
|
||||
request.getfixturevalue("aligned_content_and_embeds_b64")
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
def _encode_embeds(embeds: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(embeds, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def prompt_embeds_b64(hf_runner) -> list[str]:
|
||||
"""Pre-compute embeddings for two short prompts and return as base64."""
|
||||
prompts = ["Hello, my name is", "What is an LLM?"]
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
embeddings = hf_model.get_prompt_embeddings(prompts)
|
||||
# Cast to the server's dtype so `safe_load_prompt_embeds` doesn't need to
|
||||
# convert on its own, the function accepts any floating-point dtype and
|
||||
# will cast to the model's dtype, but matching up front skips the work.
|
||||
return [_encode_embeds(e.to(SERVER_DTYPE)) for e in embeddings]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_prompt_embeds_part(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""A user message with one prompt_embeds part + text."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_prompt_embeds_parts(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Multiple prompt_embeds parts in a single message."""
|
||||
b64_a, b64_b = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_a},
|
||||
{"type": "text", "text": " and "},
|
||||
{"type": "prompt_embeds", "data": b64_b},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_message_conversation(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""prompt_embeds in both system and user messages."""
|
||||
b64_sys, b64_usr = prompt_embeds_b64
|
||||
chat = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": b64_sys},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64_usr},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
assert chat.choices[0].message.content is not None
|
||||
assert len(chat.choices[0].message.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming(
|
||||
client: openai.AsyncOpenAI,
|
||||
prompt_embeds_b64: list[str],
|
||||
):
|
||||
"""Streaming chat completion with prompt_embeds."""
|
||||
b64 = prompt_embeds_b64[0]
|
||||
|
||||
# Non-streaming baseline.
|
||||
baseline = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
expected = baseline.choices[0].message.content
|
||||
|
||||
# Streaming.
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": "Continue:"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
chunks: list[str] = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta.content
|
||||
if delta:
|
||||
chunks.append(delta)
|
||||
assert "".join(chunks) == expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def aligned_content_and_embeds_b64(hf_runner) -> tuple[str, str]:
|
||||
"""Return `(content, base64_embeds)` where the embeddings are the model's
|
||||
embedding of `content` tokenized WITHOUT special tokens.
|
||||
"""
|
||||
content = "Hello, my name is"
|
||||
with hf_runner(MODEL_NAME) as hf_model:
|
||||
ids = hf_model.tokenizer(
|
||||
content, add_special_tokens=False, return_tensors="pt"
|
||||
).input_ids
|
||||
ids = hf_model.wrap_device({"input_ids": ids})["input_ids"]
|
||||
embed_layer = hf_model.model.get_input_embeddings()
|
||||
embeds = embed_layer(ids).squeeze(0).to(SERVER_DTYPE).cpu()
|
||||
return content, _encode_embeds(embeds)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_content_and_prompt_embeds_match(
|
||||
client: openai.AsyncOpenAI,
|
||||
aligned_content_and_embeds_b64: tuple[str, str],
|
||||
):
|
||||
"""Equal content in text and `prompt_embeds` should yield identical
|
||||
Chat Completions output under greedy decoding.
|
||||
"""
|
||||
content, encoded_embeds = aligned_content_and_embeds_b64
|
||||
|
||||
text_resp, embeds_resp = await asyncio.gather(
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
),
|
||||
client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=10,
|
||||
temperature=0.0,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": encoded_embeds}],
|
||||
}
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
text_out = text_resp.choices[0].message.content
|
||||
embeds_out = embeds_resp.choices[0].message.content
|
||||
assert text_out is not None and len(text_out) > 0
|
||||
assert embeds_out is not None and len(embeds_out) > 0
|
||||
assert text_out == embeds_out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_data_field(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""A prompt_embeds part without `data` should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_base64(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
"""Invalid base64 in the `data` field should return a clear error."""
|
||||
with pytest.raises(BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": "not_valid_base64!!"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for ``thinking_token_budget`` with reasoning models.
|
||||
|
||||
Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Literal
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
|
||||
THINK_BUDGET = 5
|
||||
|
||||
REASONING_START_STR = "<think>"
|
||||
REASONING_END_STR = "</think>"
|
||||
|
||||
|
||||
def _count_reasoning_decode_token_ids_between_markers(
|
||||
full_token_ids: list[int],
|
||||
reasoning_start_ids: list[int],
|
||||
reasoning_end_ids: list[int],
|
||||
) -> int | None:
|
||||
"""Count decode tokens in the thinking span (after last start, before first end)."""
|
||||
if not reasoning_start_ids or not reasoning_end_ids:
|
||||
raise ValueError("reasoning marker token id lists must be non-empty")
|
||||
|
||||
def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
|
||||
n = len(needle)
|
||||
if n > len(haystack):
|
||||
return -1
|
||||
for i in range(len(haystack) - n, -1, -1):
|
||||
if haystack[i : i + n] == needle:
|
||||
return i
|
||||
return -1
|
||||
|
||||
last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
|
||||
if last_start < 0:
|
||||
return None
|
||||
|
||||
pos_after_start = last_start + len(reasoning_start_ids)
|
||||
end_n = len(reasoning_end_ids)
|
||||
for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
|
||||
if full_token_ids[j : j + end_n] == reasoning_end_ids:
|
||||
return j - pos_after_start
|
||||
return len(full_token_ids) - pos_after_start
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--reasoning-config",
|
||||
'{"reasoning_start_str": "<think>", "reasoning_end_str": "</think>"}',
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_with_auto_reasoning_config():
|
||||
args = [
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_qwen35_fp8_mtp_tp2():
|
||||
"""Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
|
||||
if current_platform.device_count() < 2:
|
||||
pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
|
||||
if not current_platform.supports_fp8():
|
||||
pytest.skip("FP8 is not supported on this platform")
|
||||
|
||||
spec_cfg = {
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 2,
|
||||
"max_model_len": 32768,
|
||||
}
|
||||
args = [
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--max-model-len",
|
||||
"32768",
|
||||
"--speculative-config",
|
||||
json.dumps(spec_cfg),
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--reasoning-config",
|
||||
json.dumps(
|
||||
{
|
||||
"reasoning_start_str": REASONING_START_STR,
|
||||
"reasoning_end_str": REASONING_END_STR,
|
||||
}
|
||||
),
|
||||
]
|
||||
env_dict: dict[str, str] = {}
|
||||
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
|
||||
# on 0,1 do not exhaust memory on the same devices as this worker.
|
||||
if current_platform.device_count() >= 4:
|
||||
env_dict["CUDA_VISIBLE_DEVICES"] = "2,3"
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
QWEN35_FP8_MTP_MODEL,
|
||||
args,
|
||||
max_wait_seconds=3000,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(request, server, server_with_auto_reasoning_config):
|
||||
server_map = {
|
||||
"default": server,
|
||||
"auto_config": server_with_auto_reasoning_config,
|
||||
}
|
||||
target_server = server_map[request.param]
|
||||
async with target_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||
"""Test that mixed requests (some with thinking_token_budget, some without)
|
||||
complete successfully without errors."""
|
||||
response_with_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET},
|
||||
)
|
||||
response_without_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
)
|
||||
|
||||
msg_with = response_with_budget.choices[0].message
|
||||
msg_without = response_without_budget.choices[0].message
|
||||
|
||||
assert msg_with.content or getattr(msg_with, "reasoning", None)
|
||||
assert msg_without.content or getattr(msg_without, "reasoning", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
|
||||
"""Test that thinking_token_budget limits the number of reasoning tokens.
|
||||
|
||||
Counts reasoning decode tokens by id, which is robust to how tokens are
|
||||
grouped into streamed chunks (a single chunk can carry several tokens under
|
||||
async scheduling / stream_interval > 1). Counting chunks under-counts.
|
||||
"""
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
|
||||
prompt_token_ids: list[int] = []
|
||||
decode_token_ids: list[int] = []
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
stream=True,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET, "return_token_ids": True},
|
||||
)
|
||||
async for chunk in stream:
|
||||
if not chunk.choices:
|
||||
continue
|
||||
if getattr(chunk, "prompt_token_ids", None):
|
||||
prompt_token_ids = list(chunk.prompt_token_ids)
|
||||
delta_ids = getattr(chunk.choices[0], "token_ids", None)
|
||||
if delta_ids:
|
||||
decode_token_ids.extend(delta_ids)
|
||||
|
||||
reasoning_token_count = _count_reasoning_decode_token_ids_between_markers(
|
||||
prompt_token_ids + decode_token_ids, start_ids, end_ids
|
||||
)
|
||||
assert reasoning_token_count is not None, "missing reasoning start marker in ids"
|
||||
assert reasoning_token_count == THINK_BUDGET, (
|
||||
f"reasoning tokens ({reasoning_token_count}) != "
|
||||
f"thinking_token_budget ({THINK_BUDGET})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
@requires_fp8
|
||||
async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
|
||||
server_qwen35_fp8_mtp_tp2,
|
||||
):
|
||||
"""Concurrent chat requests: some with ``thinking_token_budget``, some without.
|
||||
|
||||
Exercises the scheduler / input processor under a mixed batch on the same
|
||||
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
|
||||
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
|
||||
"""
|
||||
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
|
||||
("budget", 1),
|
||||
("budget", 12),
|
||||
("plain", None),
|
||||
("budget", 20),
|
||||
("budget", 14),
|
||||
("plain", None),
|
||||
("plain", None),
|
||||
("budget", 12),
|
||||
("plain", None),
|
||||
]
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
|
||||
async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
|
||||
|
||||
async def budgeted_call(expected_budget: int):
|
||||
return await client.chat.completions.create(
|
||||
model=QWEN35_FP8_MTP_MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=256,
|
||||
stream=False,
|
||||
extra_body={
|
||||
"thinking_token_budget": expected_budget,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
)
|
||||
|
||||
async def plain_call():
|
||||
return await client.chat.completions.create(
|
||||
model=QWEN35_FP8_MTP_MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=256,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
coros = []
|
||||
for row in _batch_spec:
|
||||
if row[0] == "budget":
|
||||
b = row[1]
|
||||
assert isinstance(b, int)
|
||||
coros.append(budgeted_call(b))
|
||||
else:
|
||||
coros.append(plain_call())
|
||||
results = await asyncio.gather(*coros)
|
||||
|
||||
for i, (response, (kind, expected_budget)) in enumerate(
|
||||
zip(results, _batch_spec, strict=True)
|
||||
):
|
||||
msg = response.choices[0].message
|
||||
assert msg.content or getattr(msg, "reasoning", None), (
|
||||
f"index {i} ({kind}): empty message"
|
||||
)
|
||||
|
||||
if kind == "budget":
|
||||
assert expected_budget is not None
|
||||
assert response.prompt_token_ids is not None
|
||||
assert response.choices[0].token_ids is not None
|
||||
full_ids = list(response.prompt_token_ids) + list(
|
||||
response.choices[0].token_ids
|
||||
)
|
||||
n_reason = _count_reasoning_decode_token_ids_between_markers(
|
||||
full_ids, start_ids, end_ids
|
||||
)
|
||||
assert n_reason is not None, f"index {i}: missing reasoning start in ids"
|
||||
assert n_reason == expected_budget, (
|
||||
f"index {i}: reasoning decode token ids ({n_reason}) != "
|
||||
f"thinking_token_budget ({expected_budget})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_streaming_with_thinking_disabled_stays_in_content(
|
||||
client: openai.AsyncOpenAI,
|
||||
):
|
||||
request_kwargs = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Which is larger, 4 or 12?"
|
||||
" Output exactly one token: 4 or 12.",
|
||||
}
|
||||
],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0.0,
|
||||
"extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
|
||||
}
|
||||
|
||||
response = await client.chat.completions.create(**request_kwargs)
|
||||
message = response.choices[0].message
|
||||
assert message.content is not None and message.content.strip() != ""
|
||||
assert getattr(message, "reasoning", None) in (None, "")
|
||||
|
||||
stream = await client.chat.completions.create(
|
||||
**request_kwargs,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
content_chunks = []
|
||||
reasoning_chunks = []
|
||||
async for chunk in stream:
|
||||
if not chunk.choices:
|
||||
continue
|
||||
delta = chunk.choices[0].delta
|
||||
if getattr(delta, "content", None):
|
||||
content_chunks.append(delta.content)
|
||||
if getattr(delta, "reasoning", None):
|
||||
reasoning_chunks.append(delta.reasoning)
|
||||
|
||||
assert "".join(content_chunks).strip() != ""
|
||||
assert reasoning_chunks == []
|
||||
Reference in New Issue
Block a user