[Bugfix][Reasoning] Nemotron V3: surface reasoning as content when thinking is unterminated (#39091)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-06-10 05:58:19 -07:00
committed by GitHub
co-authored by Andrii Skliar
parent 77f42d9725
commit 4882fd7632
3 changed files with 163 additions and 10 deletions
@@ -8,7 +8,9 @@ import regex as re
from tests.reasoning.utils import run_reasoning_extraction
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.reasoning.nemotron_v3_reasoning_parser import NemotronV3ReasoningParser
parser_name = "nemotron_v3"
@@ -106,7 +108,7 @@ def test_nemotron_v3_reasoning(
assert content == param_dict["content"]
def test_nemotron_v3_without_thinking_returns_content(
def test_nemotron_v3_without_thinking_moves_into_content(
tokenizer: FakeNemotronTokenizer,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
@@ -124,11 +126,13 @@ def test_nemotron_v3_without_thinking_returns_content(
streaming=False,
)
# No real content followed the reasoning, so the trace is moved into
# content (reasoning left empty) — matching main's behavior.
assert reasoning is None
assert content == "This is plain content"
def test_nemotron_v3_force_nonempty_content_returns_content(
def test_nemotron_v3_force_nonempty_content_moves_into_content(
tokenizer: FakeNemotronTokenizer,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
@@ -150,6 +154,30 @@ def test_nemotron_v3_force_nonempty_content_returns_content(
assert content == "This is plain content"
def test_nemotron_v3_force_nonempty_keeps_real_content(
tokenizer: FakeNemotronTokenizer,
):
# When real content follows the closing tag nothing is promoted: the
# content after </think> is returned as-is and reasoning stays separate.
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
request = ChatCompletionRequest(
model="test-model",
messages=[],
chat_template_kwargs={"force_nonempty_content": True},
)
reasoning, content = run_reasoning_extraction(
parser,
["<think>reasoning here</think>real answer"],
request=request,
streaming=False,
)
assert reasoning == "reasoning here"
assert content == "real answer"
def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
tokenizer: FakeNemotronTokenizer,
):
@@ -170,3 +198,91 @@ def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
assert reasoning == "This is truncated reasoning"
assert content is None
_SPECIAL_TOKEN_IDS = {"<think>": 1, "</think>": 2}
def _token_id(token: str) -> int:
# Only the think markers need stable ids; everything else is non-special.
return _SPECIAL_TOKEN_IDS.get(token, 0)
def _make_reasoning_parser(tokenizer):
class _NemotronParser(DelegatingParser):
reasoning_parser_cls = NemotronV3ReasoningParser
tool_parser_cls = None
return _NemotronParser(tokenizer)
def _run_parse_delta(parser, tokenizer, text, request):
tokens = tokenizer.tokenize(text)
reasoning_parts: list[str] = []
content_parts: list[str] = []
for i, token in enumerate(tokens):
delta = parser.parse_delta(
delta_text=token,
delta_token_ids=[_token_id(token)],
request=request,
prompt_token_ids=[] if i == 0 else None,
finished=(i == len(tokens) - 1),
)
if delta is None:
continue
if delta.reasoning:
reasoning_parts.append(delta.reasoning)
if delta.content:
content_parts.append(delta.content)
return "".join(reasoning_parts), "".join(content_parts)
def test_nemotron_v3_streaming_promotes_reasoning_to_content(
tokenizer: FakeNemotronTokenizer,
):
# Model never closes <think>: reasoning streams normally AND is duplicated
# into content on the terminal delta.
request = ChatCompletionRequest(
model="test-model",
messages=[],
chat_template_kwargs={"force_nonempty_content": True},
)
parser = _make_reasoning_parser(tokenizer)
reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
assert reasoning == "4"
assert content == "4"
def test_nemotron_v3_streaming_no_promotion_with_real_content(
tokenizer: FakeNemotronTokenizer,
):
request = ChatCompletionRequest(
model="test-model",
messages=[],
chat_template_kwargs={"force_nonempty_content": True},
)
parser = _make_reasoning_parser(tokenizer)
reasoning, content = _run_parse_delta(
parser, tokenizer, "<think>reason</think>real answer", request
)
# Real content followed </think>, so nothing is duplicated.
assert reasoning == "reason"
assert content == "real answer"
def test_nemotron_v3_streaming_no_promotion_without_opt_in(
tokenizer: FakeNemotronTokenizer,
):
# Without enable_thinking=False / force_nonempty_content the fallback must
# stay disabled: the response stays reasoning-only, content empty.
request = ChatCompletionRequest(model="test-model", messages=[])
parser = _make_reasoning_parser(tokenizer)
reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
assert reasoning == "4"
assert content == ""
+23 -1
View File
@@ -776,6 +776,28 @@ class DelegatingParser(Parser):
last_tc.function.arguments or ""
) + self._tool_parser.get_remaining_unstreamed_args()
def finalize_generation(
self,
delta_message: DeltaMessage | None,
request: ChatCompletionRequest | ResponsesRequest,
state: StreamState,
) -> DeltaMessage | None:
"""Finalize generation for cases where generation was incomplete.
For example, if streaming terminated before reasoning ended
"""
fallback_fn = getattr(
self._reasoning_parser, "get_streaming_fallback_content", None
)
if fallback_fn is not None and not state.reasoning_ended:
promoted = fallback_fn(state.previous_text, request)
if promoted:
if delta_message is None:
delta_message = DeltaMessage()
delta_message.content = (delta_message.content or "") + promoted
self._append_unstreamed_tool_args(delta_message)
return delta_message
def parse(
self,
model_output: str,
@@ -883,6 +905,6 @@ class DelegatingParser(Parser):
state.previous_token_ids = current_token_ids
if finished:
self._append_unstreamed_tool_args(delta_message)
delta_message = self.finalize_generation(delta_message, request, state)
return delta_message
+22 -7
View File
@@ -14,20 +14,35 @@ class NemotronV3ReasoningParser(DeepSeekR1ReasoningParser):
Reasoning parser for Nemotron V3 models.
"""
def extract_reasoning(
self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
) -> tuple[str | None, str | None]:
reasoning, final_content = super().extract_reasoning(model_output, request)
def _should_force_content(
self, request: ChatCompletionRequest | ResponsesRequest
) -> bool:
chat_template_kwargs = getattr(request, "chat_template_kwargs", None)
if (
return bool(
chat_template_kwargs
and (
chat_template_kwargs.get("enable_thinking") is False
or chat_template_kwargs.get("force_nonempty_content") is True
)
and (final_content is None or not final_content.strip())
)
def extract_reasoning(
self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
) -> tuple[str | None, str | None]:
reasoning, final_content = super().extract_reasoning(model_output, request)
if self._should_force_content(request) and (
final_content is None or not final_content.strip()
):
reasoning, final_content = final_content, reasoning
return reasoning, final_content
def get_streaming_fallback_content(
self, text: str, request: ChatCompletionRequest | ResponsesRequest
) -> str | None:
"""Reasoning to duplicate into content on the terminal streaming delta."""
if not self._should_force_content(request):
return None
reasoning, _ = super().extract_reasoning(text, request)
return reasoning