From 4882fd763282799e3570ca45baeca5c14c659e39 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Wed, 10 Jun 2026 14:58:19 +0200 Subject: [PATCH] [Bugfix][Reasoning] Nemotron V3: surface reasoning as content when thinking is unterminated (#39091) Signed-off-by: Andrii Skliar Co-authored-by: Andrii Skliar --- .../test_nemotron_v3_reasoning_parser.py | 120 +++++++++++++++++- vllm/parser/abstract_parser.py | 24 +++- .../reasoning/nemotron_v3_reasoning_parser.py | 29 ++++- 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/tests/reasoning/test_nemotron_v3_reasoning_parser.py b/tests/reasoning/test_nemotron_v3_reasoning_parser.py index c7ba95cb11..a22ce6aef7 100644 --- a/tests/reasoning/test_nemotron_v3_reasoning_parser.py +++ b/tests/reasoning/test_nemotron_v3_reasoning_parser.py @@ -8,7 +8,9 @@ import regex as re from tests.reasoning.utils import run_reasoning_extraction from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.parser.abstract_parser import DelegatingParser from vllm.reasoning import ReasoningParser, ReasoningParserManager +from vllm.reasoning.nemotron_v3_reasoning_parser import NemotronV3ReasoningParser parser_name = "nemotron_v3" @@ -106,7 +108,7 @@ def test_nemotron_v3_reasoning( assert content == param_dict["content"] -def test_nemotron_v3_without_thinking_returns_content( +def test_nemotron_v3_without_thinking_moves_into_content( tokenizer: FakeNemotronTokenizer, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) @@ -124,11 +126,13 @@ def test_nemotron_v3_without_thinking_returns_content( streaming=False, ) + # No real content followed the reasoning, so the trace is moved into + # content (reasoning left empty) — matching main's behavior. assert reasoning is None assert content == "This is plain content" -def test_nemotron_v3_force_nonempty_content_returns_content( +def test_nemotron_v3_force_nonempty_content_moves_into_content( tokenizer: FakeNemotronTokenizer, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) @@ -150,6 +154,30 @@ def test_nemotron_v3_force_nonempty_content_returns_content( assert content == "This is plain content" +def test_nemotron_v3_force_nonempty_keeps_real_content( + tokenizer: FakeNemotronTokenizer, +): + # When real content follows the closing tag nothing is promoted: the + # content after is returned as-is and reasoning stays separate. + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(tokenizer) + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + + reasoning, content = run_reasoning_extraction( + parser, + ["reasoning herereal answer"], + request=request, + streaming=False, + ) + + assert reasoning == "reasoning here" + assert content == "real answer" + + def test_nemotron_v3_with_thinking_keeps_truncated_reasoning( tokenizer: FakeNemotronTokenizer, ): @@ -170,3 +198,91 @@ def test_nemotron_v3_with_thinking_keeps_truncated_reasoning( assert reasoning == "This is truncated reasoning" assert content is None + + +_SPECIAL_TOKEN_IDS = {"": 1, "": 2} + + +def _token_id(token: str) -> int: + # Only the think markers need stable ids; everything else is non-special. + return _SPECIAL_TOKEN_IDS.get(token, 0) + + +def _make_reasoning_parser(tokenizer): + class _NemotronParser(DelegatingParser): + reasoning_parser_cls = NemotronV3ReasoningParser + tool_parser_cls = None + + return _NemotronParser(tokenizer) + + +def _run_parse_delta(parser, tokenizer, text, request): + tokens = tokenizer.tokenize(text) + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + for i, token in enumerate(tokens): + delta = parser.parse_delta( + delta_text=token, + delta_token_ids=[_token_id(token)], + request=request, + prompt_token_ids=[] if i == 0 else None, + finished=(i == len(tokens) - 1), + ) + if delta is None: + continue + if delta.reasoning: + reasoning_parts.append(delta.reasoning) + if delta.content: + content_parts.append(delta.content) + return "".join(reasoning_parts), "".join(content_parts) + + +def test_nemotron_v3_streaming_promotes_reasoning_to_content( + tokenizer: FakeNemotronTokenizer, +): + # Model never closes : reasoning streams normally AND is duplicated + # into content on the terminal delta. + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta(parser, tokenizer, "4", request) + + assert reasoning == "4" + assert content == "4" + + +def test_nemotron_v3_streaming_no_promotion_with_real_content( + tokenizer: FakeNemotronTokenizer, +): + request = ChatCompletionRequest( + model="test-model", + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta( + parser, tokenizer, "reasonreal answer", request + ) + + # Real content followed , so nothing is duplicated. + assert reasoning == "reason" + assert content == "real answer" + + +def test_nemotron_v3_streaming_no_promotion_without_opt_in( + tokenizer: FakeNemotronTokenizer, +): + # Without enable_thinking=False / force_nonempty_content the fallback must + # stay disabled: the response stays reasoning-only, content empty. + request = ChatCompletionRequest(model="test-model", messages=[]) + parser = _make_reasoning_parser(tokenizer) + + reasoning, content = _run_parse_delta(parser, tokenizer, "4", request) + + assert reasoning == "4" + assert content == "" diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 53809a126c..4b039f7322 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -776,6 +776,28 @@ class DelegatingParser(Parser): last_tc.function.arguments or "" ) + self._tool_parser.get_remaining_unstreamed_args() + def finalize_generation( + self, + delta_message: DeltaMessage | None, + request: ChatCompletionRequest | ResponsesRequest, + state: StreamState, + ) -> DeltaMessage | None: + """Finalize generation for cases where generation was incomplete. + For example, if streaming terminated before reasoning ended + """ + fallback_fn = getattr( + self._reasoning_parser, "get_streaming_fallback_content", None + ) + if fallback_fn is not None and not state.reasoning_ended: + promoted = fallback_fn(state.previous_text, request) + if promoted: + if delta_message is None: + delta_message = DeltaMessage() + delta_message.content = (delta_message.content or "") + promoted + + self._append_unstreamed_tool_args(delta_message) + return delta_message + def parse( self, model_output: str, @@ -883,6 +905,6 @@ class DelegatingParser(Parser): state.previous_token_ids = current_token_ids if finished: - self._append_unstreamed_tool_args(delta_message) + delta_message = self.finalize_generation(delta_message, request, state) return delta_message diff --git a/vllm/reasoning/nemotron_v3_reasoning_parser.py b/vllm/reasoning/nemotron_v3_reasoning_parser.py index 7256f0f128..635281f817 100644 --- a/vllm/reasoning/nemotron_v3_reasoning_parser.py +++ b/vllm/reasoning/nemotron_v3_reasoning_parser.py @@ -14,20 +14,35 @@ class NemotronV3ReasoningParser(DeepSeekR1ReasoningParser): Reasoning parser for Nemotron V3 models. """ - def extract_reasoning( - self, model_output: str, request: ChatCompletionRequest | ResponsesRequest - ) -> tuple[str | None, str | None]: - reasoning, final_content = super().extract_reasoning(model_output, request) + def _should_force_content( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> bool: chat_template_kwargs = getattr(request, "chat_template_kwargs", None) - - if ( + return bool( chat_template_kwargs and ( chat_template_kwargs.get("enable_thinking") is False or chat_template_kwargs.get("force_nonempty_content") is True ) - and (final_content is None or not final_content.strip()) + ) + + def extract_reasoning( + self, model_output: str, request: ChatCompletionRequest | ResponsesRequest + ) -> tuple[str | None, str | None]: + reasoning, final_content = super().extract_reasoning(model_output, request) + + if self._should_force_content(request) and ( + final_content is None or not final_content.strip() ): reasoning, final_content = final_content, reasoning return reasoning, final_content + + def get_streaming_fallback_content( + self, text: str, request: ChatCompletionRequest | ResponsesRequest + ) -> str | None: + """Reasoning to duplicate into content on the terminal streaming delta.""" + if not self._should_force_content(request): + return None + reasoning, _ = super().extract_reasoning(text, request) + return reasoning