[Bugfix][Frontend] Apply model-default reasoning parser in GPU-less render server (#54835)
Signed-off-by: Shimi Bandiel <shimib@google.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude
Cyrus Leung
parent
68088ed392
commit
09e0ce3d82
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from starlette.datastructures import State
|
||||
|
||||
import vllm.entrypoints.launchers.render.app_state as app_state_mod
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class _CaptureKwargs:
|
||||
"""Stands in for OnlineRenderer/OnlineDerenderer; records init kwargs."""
|
||||
|
||||
captured: list[dict]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
type(self).captured.append(kwargs)
|
||||
|
||||
def warmup(self):
|
||||
pass
|
||||
|
||||
|
||||
def _render_cli_args(*argv: str) -> Namespace:
|
||||
"""Parse ``argv`` the way ``vllm launch render`` does, so new serve flags
|
||||
carry their defaults instead of needing a hand-maintained ``Namespace``."""
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(list(argv))
|
||||
if args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_app_state_uses_config_resolved_reasoning_parser(monkeypatch):
|
||||
"""gpt_oss defaults its reasoning parser through verify_and_update_config;
|
||||
the render server must pick up that resolved value like the main API
|
||||
server does, otherwise harmony markup leaks unparsed into derender output.
|
||||
"""
|
||||
model_config = ModelConfig("openai/gpt-oss-20b")
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
assert vllm_config.structured_outputs_config.reasoning_parser == "openai_gptoss"
|
||||
|
||||
class _Renderer(_CaptureKwargs):
|
||||
captured = []
|
||||
|
||||
class _Derenderer(_CaptureKwargs):
|
||||
captured = []
|
||||
|
||||
async def _noop_async(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_state_mod, "renderer_from_config", lambda cfg: object())
|
||||
monkeypatch.setattr(app_state_mod, "OnlineRenderer", _Renderer)
|
||||
monkeypatch.setattr(app_state_mod, "OnlineDerenderer", _Derenderer)
|
||||
monkeypatch.setattr(app_state_mod, "ServingTokenization", lambda *a, **kw: object())
|
||||
monkeypatch.setattr(app_state_mod, "init_render_state", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(app_state_mod, "init_endpoint_plugins_state", _noop_async)
|
||||
|
||||
args = _render_cli_args("openai/gpt-oss-20b")
|
||||
|
||||
await app_state_mod.init_render_app_state(vllm_config, State(), args)
|
||||
|
||||
assert _Renderer.captured[0]["reasoning_parser"] == "openai_gptoss"
|
||||
assert _Derenderer.captured[0]["reasoning_parser"] == "openai_gptoss"
|
||||
|
||||
|
||||
def test_explicit_reasoning_parser_flag_wins_over_model_default():
|
||||
"""An explicit --reasoning-parser must survive VllmConfig construction
|
||||
for models that define their own default (gpt_oss); the default only
|
||||
applies when the resolved value is empty."""
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
|
||||
engine_args = AsyncEngineArgs(
|
||||
model="openai/gpt-oss-20b", reasoning_parser="deepseek_r1"
|
||||
)
|
||||
model_config = engine_args.create_model_config()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
structured_outputs_config=engine_args.create_structured_outputs_config(),
|
||||
)
|
||||
|
||||
assert vllm_config.structured_outputs_config.reasoning_parser == "deepseek_r1"
|
||||
@@ -1298,6 +1298,72 @@ async def test_e2e_harmony_plain_roundtrip(harmony_client, harmony_tokenizer):
|
||||
assert "Four" in content
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def harmony_default_server():
|
||||
"""gpt-oss render server with NO parser flags.
|
||||
|
||||
Regression fixture: the model-default reasoning parser
|
||||
("openai_gptoss", applied by verify_and_update_config) must be
|
||||
resolved by the render server without any --reasoning-parser flag,
|
||||
otherwise harmony markup leaks unparsed into derender output.
|
||||
"""
|
||||
_ensure_harmony_vocab()
|
||||
with RemoteLaunchRenderServer(
|
||||
HARMONY_MODEL, ["--trust-remote-code"]
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def harmony_default_client(harmony_default_server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=harmony_default_server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_harmony_reasoning_default_parser(
|
||||
harmony_default_client, harmony_tokenizer
|
||||
):
|
||||
"""GPT-OSS reasoning parses on a bare launch (no parser flags)."""
|
||||
messages = [{"role": "user", "content": "Add 2 and 3."}]
|
||||
gen_req = await _e2e_render_chat(harmony_default_client, HARMONY_MODEL, messages)
|
||||
|
||||
reasoning_text = "The user wants 2 plus 3."
|
||||
answer_text = "The answer is 5."
|
||||
assistant_msg = {
|
||||
"role": "assistant",
|
||||
"thinking": reasoning_text,
|
||||
"content": answer_text,
|
||||
}
|
||||
output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg)
|
||||
|
||||
decoded = harmony_tokenizer.decode(output_ids)
|
||||
if reasoning_text not in decoded:
|
||||
pytest.skip("Harmony template did not render thinking")
|
||||
|
||||
resp = await harmony_default_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": HARMONY_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": HARMONY_MODEL,
|
||||
"messages": messages,
|
||||
"include_reasoning": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
msg = resp.json()["choices"][0]["message"]
|
||||
assert msg["reasoning"] is not None
|
||||
assert reasoning_text in msg["reasoning"]
|
||||
assert answer_text in (msg["content"] or "")
|
||||
assert "<|channel|>" not in (msg["content"] or "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer):
|
||||
"""GPT-OSS reasoning: analysis channel extracted."""
|
||||
|
||||
@@ -2136,6 +2136,25 @@ class EngineArgs:
|
||||
cfg = json.loads(cfg)
|
||||
return WatermarkConfig(**cfg)
|
||||
|
||||
def create_structured_outputs_config(self) -> StructuredOutputsConfig:
|
||||
"""Merge frontend parser flags into the structured outputs config.
|
||||
|
||||
Mutates `self.structured_outputs_config` in place and returns it
|
||||
(not a copy). Model-specific defaults (e.g. gpt_oss ->
|
||||
"openai_gptoss") are applied later by `verify_and_update_config`
|
||||
only when the resolved value is still empty, so explicit CLI flags
|
||||
take precedence.
|
||||
"""
|
||||
if self.reasoning_parser:
|
||||
self.structured_outputs_config.reasoning_parser = self.reasoning_parser
|
||||
|
||||
if self.reasoning_parser_plugin:
|
||||
self.structured_outputs_config.reasoning_parser_plugin = (
|
||||
self.reasoning_parser_plugin
|
||||
)
|
||||
|
||||
return self.structured_outputs_config
|
||||
|
||||
def create_observability_config(self) -> ObservabilityConfig:
|
||||
return ObservabilityConfig(
|
||||
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
|
||||
@@ -2728,13 +2747,7 @@ class EngineArgs:
|
||||
load_config = self.create_load_config()
|
||||
|
||||
# Pass reasoning_parser into StructuredOutputsConfig
|
||||
if self.reasoning_parser:
|
||||
self.structured_outputs_config.reasoning_parser = self.reasoning_parser
|
||||
|
||||
if self.reasoning_parser_plugin:
|
||||
self.structured_outputs_config.reasoning_parser_plugin = (
|
||||
self.reasoning_parser_plugin
|
||||
)
|
||||
self.create_structured_outputs_config()
|
||||
|
||||
observability_config = self.create_observability_config()
|
||||
|
||||
|
||||
@@ -50,6 +50,16 @@ async def init_render_app_state(
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = await init_tool_server(args)
|
||||
|
||||
# The config-resolved reasoning parser carries both the CLI flag (merged
|
||||
# in the entrypoint via `create_structured_outputs_config`) and any
|
||||
# model-specific default applied by `verify_and_update_config`
|
||||
# (e.g. "openai_gptoss" for gpt_oss), matching the main API server.
|
||||
# Keep `args.reasoning_parser` as the first source so callers that build
|
||||
# a VllmConfig without that merge still honor an explicit flag.
|
||||
reasoning_parser = (
|
||||
args.reasoning_parser or vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
@@ -62,7 +72,7 @@ async def init_render_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
reasoning_parser=reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
@@ -79,7 +89,7 @@ async def init_render_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
reasoning_parser=reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
@@ -87,9 +87,14 @@ async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||
# cache space warning from CpuPlatform.check_and_update_config.
|
||||
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
||||
|
||||
# Merge frontend parser flags (e.g. --reasoning-parser) into the
|
||||
# structured outputs config, mirroring `create_engine_config`, which this
|
||||
# GPU-less entrypoint bypasses. Model-specific defaults are then applied
|
||||
# by `verify_and_update_config` during VllmConfig construction.
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
logging_config=engine_args.create_logging_config(),
|
||||
structured_outputs_config=engine_args.create_structured_outputs_config(),
|
||||
)
|
||||
shutdown_task = await build_and_serve_renderer(
|
||||
vllm_config, listen_address, sock, args
|
||||
|
||||
Reference in New Issue
Block a user