From 09e0ce3d82f404e842fdfc9ab34ba27e76f95a16 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Tue, 6 Oct 2026 03:46:24 -0700 Subject: [PATCH] [Bugfix][Frontend] Apply model-default reasoning parser in GPU-less render server (#54835) Signed-off-by: Shimi Bandiel Co-authored-by: Claude Co-authored-by: Cyrus Leung --- .../entrypoints/launchers/render/__init__.py | 2 + .../launchers/render/test_app_state.py | 84 +++++++++++++++++++ .../scale_out/derender/test_derender.py | 66 +++++++++++++++ vllm/engine/arg_utils.py | 27 ++++-- .../entrypoints/launchers/render/app_state.py | 14 +++- vllm/entrypoints/launchers/render/entry.py | 5 ++ 6 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 tests/entrypoints/launchers/render/__init__.py create mode 100644 tests/entrypoints/launchers/render/test_app_state.py diff --git a/tests/entrypoints/launchers/render/__init__.py b/tests/entrypoints/launchers/render/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/entrypoints/launchers/render/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/entrypoints/launchers/render/test_app_state.py b/tests/entrypoints/launchers/render/test_app_state.py new file mode 100644 index 0000000000..f0f54d9baf --- /dev/null +++ b/tests/entrypoints/launchers/render/test_app_state.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from argparse import Namespace + +import pytest +from starlette.datastructures import State + +import vllm.entrypoints.launchers.render.app_state as app_state_mod +from vllm.config import ModelConfig, VllmConfig +from vllm.entrypoints.launchers.cli_args import make_arg_parser +from vllm.utils.argparse_utils import FlexibleArgumentParser + + +class _CaptureKwargs: + """Stands in for OnlineRenderer/OnlineDerenderer; records init kwargs.""" + + captured: list[dict] + + def __init__(self, **kwargs): + type(self).captured.append(kwargs) + + def warmup(self): + pass + + +def _render_cli_args(*argv: str) -> Namespace: + """Parse ``argv`` the way ``vllm launch render`` does, so new serve flags + carry their defaults instead of needing a hand-maintained ``Namespace``.""" + args = make_arg_parser(FlexibleArgumentParser()).parse_args(list(argv)) + if args.model_tag is not None: + args.model = args.model_tag + return args + + +@pytest.mark.asyncio +async def test_render_app_state_uses_config_resolved_reasoning_parser(monkeypatch): + """gpt_oss defaults its reasoning parser through verify_and_update_config; + the render server must pick up that resolved value like the main API + server does, otherwise harmony markup leaks unparsed into derender output. + """ + model_config = ModelConfig("openai/gpt-oss-20b") + vllm_config = VllmConfig(model_config=model_config) + assert vllm_config.structured_outputs_config.reasoning_parser == "openai_gptoss" + + class _Renderer(_CaptureKwargs): + captured = [] + + class _Derenderer(_CaptureKwargs): + captured = [] + + async def _noop_async(*args, **kwargs): + return None + + monkeypatch.setattr(app_state_mod, "renderer_from_config", lambda cfg: object()) + monkeypatch.setattr(app_state_mod, "OnlineRenderer", _Renderer) + monkeypatch.setattr(app_state_mod, "OnlineDerenderer", _Derenderer) + monkeypatch.setattr(app_state_mod, "ServingTokenization", lambda *a, **kw: object()) + monkeypatch.setattr(app_state_mod, "init_render_state", lambda *a, **kw: None) + monkeypatch.setattr(app_state_mod, "init_endpoint_plugins_state", _noop_async) + + args = _render_cli_args("openai/gpt-oss-20b") + + await app_state_mod.init_render_app_state(vllm_config, State(), args) + + assert _Renderer.captured[0]["reasoning_parser"] == "openai_gptoss" + assert _Derenderer.captured[0]["reasoning_parser"] == "openai_gptoss" + + +def test_explicit_reasoning_parser_flag_wins_over_model_default(): + """An explicit --reasoning-parser must survive VllmConfig construction + for models that define their own default (gpt_oss); the default only + applies when the resolved value is empty.""" + from vllm.engine.arg_utils import AsyncEngineArgs + + engine_args = AsyncEngineArgs( + model="openai/gpt-oss-20b", reasoning_parser="deepseek_r1" + ) + model_config = engine_args.create_model_config() + vllm_config = VllmConfig( + model_config=model_config, + structured_outputs_config=engine_args.create_structured_outputs_config(), + ) + + assert vllm_config.structured_outputs_config.reasoning_parser == "deepseek_r1" diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index 659aaed1aa..40b051304e 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -1298,6 +1298,72 @@ async def test_e2e_harmony_plain_roundtrip(harmony_client, harmony_tokenizer): assert "Four" in content +@pytest.fixture(scope="module") +def harmony_default_server(): + """gpt-oss render server with NO parser flags. + + Regression fixture: the model-default reasoning parser + ("openai_gptoss", applied by verify_and_update_config) must be + resolved by the render server without any --reasoning-parser flag, + otherwise harmony markup leaks unparsed into derender output. + """ + _ensure_harmony_vocab() + with RemoteLaunchRenderServer( + HARMONY_MODEL, ["--trust-remote-code"] + ) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def harmony_default_client(harmony_default_server): + async with httpx.AsyncClient( + base_url=harmony_default_server.url_for(""), timeout=60.0 + ) as http_client: + yield http_client + + +@pytest.mark.asyncio +async def test_e2e_harmony_reasoning_default_parser( + harmony_default_client, harmony_tokenizer +): + """GPT-OSS reasoning parses on a bare launch (no parser flags).""" + messages = [{"role": "user", "content": "Add 2 and 3."}] + gen_req = await _e2e_render_chat(harmony_default_client, HARMONY_MODEL, messages) + + reasoning_text = "The user wants 2 plus 3." + answer_text = "The answer is 5." + assistant_msg = { + "role": "assistant", + "thinking": reasoning_text, + "content": answer_text, + } + output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg) + + decoded = harmony_tokenizer.decode(output_ids) + if reasoning_text not in decoded: + pytest.skip("Harmony template did not render thinking") + + resp = await harmony_default_client.post( + "/v1/chat/completions/derender", + json={ + "model": HARMONY_MODEL, + "generate_response": _e2e_generate_response(output_ids), + "prompt_tokens": len(gen_req["token_ids"]), + "chat_request": { + "model": HARMONY_MODEL, + "messages": messages, + "include_reasoning": True, + }, + }, + ) + assert resp.status_code == 200, resp.text + msg = resp.json()["choices"][0]["message"] + assert msg["reasoning"] is not None + assert reasoning_text in msg["reasoning"] + assert answer_text in (msg["content"] or "") + assert "<|channel|>" not in (msg["content"] or "") + + @pytest.mark.asyncio async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer): """GPT-OSS reasoning: analysis channel extracted.""" diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 4f1f417289..98a0b7cd88 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -2136,6 +2136,25 @@ class EngineArgs: cfg = json.loads(cfg) return WatermarkConfig(**cfg) + def create_structured_outputs_config(self) -> StructuredOutputsConfig: + """Merge frontend parser flags into the structured outputs config. + + Mutates `self.structured_outputs_config` in place and returns it + (not a copy). Model-specific defaults (e.g. gpt_oss -> + "openai_gptoss") are applied later by `verify_and_update_config` + only when the resolved value is still empty, so explicit CLI flags + take precedence. + """ + if self.reasoning_parser: + self.structured_outputs_config.reasoning_parser = self.reasoning_parser + + if self.reasoning_parser_plugin: + self.structured_outputs_config.reasoning_parser_plugin = ( + self.reasoning_parser_plugin + ) + + return self.structured_outputs_config + def create_observability_config(self) -> ObservabilityConfig: return ObservabilityConfig( show_hidden_metrics_for_version=self.show_hidden_metrics_for_version, @@ -2728,13 +2747,7 @@ class EngineArgs: load_config = self.create_load_config() # Pass reasoning_parser into StructuredOutputsConfig - if self.reasoning_parser: - self.structured_outputs_config.reasoning_parser = self.reasoning_parser - - if self.reasoning_parser_plugin: - self.structured_outputs_config.reasoning_parser_plugin = ( - self.reasoning_parser_plugin - ) + self.create_structured_outputs_config() observability_config = self.create_observability_config() diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py index bbd9a61275..33aebb9bc3 100644 --- a/vllm/entrypoints/launchers/render/app_state.py +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -50,6 +50,16 @@ async def init_render_app_state( default_chat_template_kwargs = resolve_default_chat_template_kwargs(args) state.tool_server = await init_tool_server(args) + # The config-resolved reasoning parser carries both the CLI flag (merged + # in the entrypoint via `create_structured_outputs_config`) and any + # model-specific default applied by `verify_and_update_config` + # (e.g. "openai_gptoss" for gpt_oss), matching the main API server. + # Keep `args.reasoning_parser` as the first source so callers that build + # a VllmConfig without that merge still honor an explicit flag. + reasoning_parser = ( + args.reasoning_parser or vllm_config.structured_outputs_config.reasoning_parser + ) + state.online_renderer = OnlineRenderer( model_config=vllm_config.model_config, renderer=renderer, @@ -62,7 +72,7 @@ async def init_render_app_state( exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, tool_strict_level=args.tool_strict_level, - reasoning_parser=args.reasoning_parser, + reasoning_parser=reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) @@ -79,7 +89,7 @@ async def init_render_app_state( exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, tool_strict_level=args.tool_strict_level, - reasoning_parser=args.reasoning_parser, + reasoning_parser=reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) diff --git a/vllm/entrypoints/launchers/render/entry.py b/vllm/entrypoints/launchers/render/entry.py index b42e1f5911..92c29ea320 100644 --- a/vllm/entrypoints/launchers/render/entry.py +++ b/vllm/entrypoints/launchers/render/entry.py @@ -87,9 +87,14 @@ async def run_launch_fastapi(args: argparse.Namespace) -> None: # cache space warning from CpuPlatform.check_and_update_config. envs.VLLM_CPU_KVCACHE_SPACE = 0 + # Merge frontend parser flags (e.g. --reasoning-parser) into the + # structured outputs config, mirroring `create_engine_config`, which this + # GPU-less entrypoint bypasses. Model-specific defaults are then applied + # by `verify_and_update_config` during VllmConfig construction. vllm_config = VllmConfig( model_config=model_config, logging_config=engine_args.create_logging_config(), + structured_outputs_config=engine_args.create_structured_outputs_config(), ) shutdown_task = await build_and_serve_renderer( vllm_config, listen_address, sock, args