Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -32,8 +32,14 @@ from vllm.entrypoints.cohere.protocol import (
|
||||
AssistantMessageResponse,
|
||||
CohereChatV2Response,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
from vllm.entrypoints.cohere.serving import (
|
||||
_API_SERVER_ONLY_TEMPLATE_KWARGS,
|
||||
CohereServingChatV2,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
ReasoningParserKwargs,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.http import (
|
||||
http_exception_handler,
|
||||
@@ -97,6 +103,12 @@ class _RenderChatHandler:
|
||||
def to_chat_completion_request(self, request):
|
||||
return CohereServingChatV2._convert_v2_to_chat_completion(request)
|
||||
|
||||
def _engine_chat_template_kwargs(self, chat_template_kwargs):
|
||||
return CohereServingChatV2._engine_chat_template_kwargs(
|
||||
self, # type: ignore[arg-type]
|
||||
chat_template_kwargs,
|
||||
)
|
||||
|
||||
|
||||
class _RenderHandler:
|
||||
"""Minimal stand-in for :class:`ServingRender`.
|
||||
@@ -459,6 +471,24 @@ class TestRenderEndpoint:
|
||||
# The conversion lower-cases safety_mode for the renderer.
|
||||
assert kwargs["safety_mode"] == "strict"
|
||||
|
||||
def test_api_server_only_kwargs_not_forwarded(self):
|
||||
"""As on ``/cohere/v2/chat``, the engine's reasoning parser must not
|
||||
receive the template kwargs that only the API server reads."""
|
||||
result = _generate_request()
|
||||
result.reasoning_parser_kwargs = ReasoningParserKwargs(
|
||||
chat_template_kwargs={
|
||||
"safety_mode": "strict",
|
||||
**dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, "x"),
|
||||
}
|
||||
)
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(result))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.json()["reasoning_parser_kwargs"] == {
|
||||
"chat_template_kwargs": {"safety_mode": "strict"}
|
||||
}
|
||||
|
||||
def test_501_when_chat_handler_missing(self):
|
||||
app = _build_render_app(None, _RenderHandler(_generate_request()))
|
||||
with TestClient(app) as client:
|
||||
|
||||
@@ -38,6 +38,7 @@ def _build_responses_serving_render() -> ServingRender:
|
||||
OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
|
||||
)
|
||||
serving.online_renderer.trust_request_chat_template = True
|
||||
serving.online_renderer.parser = None
|
||||
serving._check_model = AsyncMock(return_value=None)
|
||||
return serving
|
||||
|
||||
@@ -135,6 +136,37 @@ async def test_render_responses_rejects_empty_token_ids():
|
||||
assert response.error.message == "No token_ids rendered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_carries_reasoning_parser_kwargs():
|
||||
"""/v1/responses gives the engine's reasoning parser the effective template
|
||||
kwargs, so the rendered request must carry them to generate."""
|
||||
serving = _build_responses_serving_render()
|
||||
online = serving.online_renderer
|
||||
online.parser = MagicMock(reasoning_parser_cls=object)
|
||||
online.chat_template = None
|
||||
online.chat_template_content_format = "auto"
|
||||
online.default_chat_template_kwargs = {"enable_thinking": True}
|
||||
online.effective_chat_template_kwargs = (
|
||||
OnlineRenderer.effective_chat_template_kwargs.__get__(online)
|
||||
)
|
||||
online.render_responses = AsyncMock(
|
||||
return_value=MagicMock(messages=[], engine_input={"prompt_token_ids": [7]})
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
model=MODEL_NAME,
|
||||
input="Test prompt",
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(request)
|
||||
|
||||
assert not isinstance(response, ErrorResponse)
|
||||
assert response.reasoning_parser_kwargs is not None
|
||||
kwargs = response.reasoning_parser_kwargs.chat_template_kwargs
|
||||
assert kwargs["enable_thinking"] is False
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = ["--trust-request-chat-template"]
|
||||
|
||||
@@ -124,6 +124,14 @@ def test_generate_request_rejects_placeholder_outside_prompt():
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_parser_kwargs_rejects_unknown_keys():
|
||||
"""The engine splats these into its reasoning parser's constructor, so a
|
||||
client must not be able to pass arbitrary keyword arguments."""
|
||||
payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
|
||||
with pytest.raises(ValidationError, match="tokenizer"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_output_mode_defaults_to_tokens():
|
||||
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
|
||||
|
||||
|
||||
@@ -589,6 +589,83 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
def _structured_chat_body(messages, **overrides) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 64,
|
||||
"temperature": 0.0,
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"count": {"type": "integer"}},
|
||||
"required": ["count"],
|
||||
},
|
||||
},
|
||||
},
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
|
||||
render_resp = await client.post("/v1/chat/completions/render", json=body)
|
||||
render_resp.raise_for_status()
|
||||
generate_request = render_resp.json()
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
|
||||
generate_resp.raise_for_status()
|
||||
return generate_request, generate_resp.json()["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
|
||||
# <think> tokens, so the engine gates structured outputs differently when
|
||||
# either reasoning field is dropped between render and generate.
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
|
||||
)
|
||||
async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
|
||||
"""With thinking on, structured outputs must wait for the reasoning to
|
||||
end on generate, the same as on chat."""
|
||||
body = _structured_chat_body(messages, return_token_ids=True)
|
||||
chat_resp = await client.post("/v1/chat/completions", json=body)
|
||||
chat_resp.raise_for_status()
|
||||
|
||||
generate_request, token_ids = await _render_then_generate(client, body)
|
||||
|
||||
assert generate_request["reasoning_ended"] is False
|
||||
assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
|
||||
)
|
||||
async def test_render_then_generate_forwards_reasoning_ended(
|
||||
client, tokenizer, messages
|
||||
):
|
||||
"""`include_reasoning=false` constrains from the first token on chat, so
|
||||
generate must too, even though the engine's parser expects thinking."""
|
||||
body = _structured_chat_body(messages, include_reasoning=False)
|
||||
|
||||
generate_request, token_ids = await _render_then_generate(client, body)
|
||||
|
||||
assert generate_request["reasoning_ended"] is True
|
||||
output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
|
||||
assert isinstance(output["count"], int)
|
||||
|
||||
|
||||
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
|
||||
return tokenizer.apply_chat_template(
|
||||
messages,
|
||||
|
||||
Reference in New Issue
Block a user