[Bugfix][Reasoning] Nemotron V3: surface reasoning as content when thinking is unterminated (#39091)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
77f42d9725
commit
4882fd7632
@@ -8,7 +8,9 @@ import regex as re
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.reasoning.nemotron_v3_reasoning_parser import NemotronV3ReasoningParser
|
||||
|
||||
parser_name = "nemotron_v3"
|
||||
|
||||
@@ -106,7 +108,7 @@ def test_nemotron_v3_reasoning(
|
||||
assert content == param_dict["content"]
|
||||
|
||||
|
||||
def test_nemotron_v3_without_thinking_returns_content(
|
||||
def test_nemotron_v3_without_thinking_moves_into_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
@@ -124,11 +126,13 @@ def test_nemotron_v3_without_thinking_returns_content(
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
# No real content followed the reasoning, so the trace is moved into
|
||||
# content (reasoning left empty) — matching main's behavior.
|
||||
assert reasoning is None
|
||||
assert content == "This is plain content"
|
||||
|
||||
|
||||
def test_nemotron_v3_force_nonempty_content_returns_content(
|
||||
def test_nemotron_v3_force_nonempty_content_moves_into_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
@@ -150,6 +154,30 @@ def test_nemotron_v3_force_nonempty_content_returns_content(
|
||||
assert content == "This is plain content"
|
||||
|
||||
|
||||
def test_nemotron_v3_force_nonempty_keeps_real_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
# When real content follows the closing tag nothing is promoted: the
|
||||
# content after </think> is returned as-is and reasoning stays separate.
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser,
|
||||
["<think>reasoning here</think>real answer"],
|
||||
request=request,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
assert reasoning == "reasoning here"
|
||||
assert content == "real answer"
|
||||
|
||||
|
||||
def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
@@ -170,3 +198,91 @@ def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
|
||||
|
||||
assert reasoning == "This is truncated reasoning"
|
||||
assert content is None
|
||||
|
||||
|
||||
_SPECIAL_TOKEN_IDS = {"<think>": 1, "</think>": 2}
|
||||
|
||||
|
||||
def _token_id(token: str) -> int:
|
||||
# Only the think markers need stable ids; everything else is non-special.
|
||||
return _SPECIAL_TOKEN_IDS.get(token, 0)
|
||||
|
||||
|
||||
def _make_reasoning_parser(tokenizer):
|
||||
class _NemotronParser(DelegatingParser):
|
||||
reasoning_parser_cls = NemotronV3ReasoningParser
|
||||
tool_parser_cls = None
|
||||
|
||||
return _NemotronParser(tokenizer)
|
||||
|
||||
|
||||
def _run_parse_delta(parser, tokenizer, text, request):
|
||||
tokens = tokenizer.tokenize(text)
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
for i, token in enumerate(tokens):
|
||||
delta = parser.parse_delta(
|
||||
delta_text=token,
|
||||
delta_token_ids=[_token_id(token)],
|
||||
request=request,
|
||||
prompt_token_ids=[] if i == 0 else None,
|
||||
finished=(i == len(tokens) - 1),
|
||||
)
|
||||
if delta is None:
|
||||
continue
|
||||
if delta.reasoning:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
return "".join(reasoning_parts), "".join(content_parts)
|
||||
|
||||
|
||||
def test_nemotron_v3_streaming_promotes_reasoning_to_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
# Model never closes <think>: reasoning streams normally AND is duplicated
|
||||
# into content on the terminal delta.
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
|
||||
|
||||
assert reasoning == "4"
|
||||
assert content == "4"
|
||||
|
||||
|
||||
def test_nemotron_v3_streaming_no_promotion_with_real_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
chat_template_kwargs={"force_nonempty_content": True},
|
||||
)
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(
|
||||
parser, tokenizer, "<think>reason</think>real answer", request
|
||||
)
|
||||
|
||||
# Real content followed </think>, so nothing is duplicated.
|
||||
assert reasoning == "reason"
|
||||
assert content == "real answer"
|
||||
|
||||
|
||||
def test_nemotron_v3_streaming_no_promotion_without_opt_in(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
):
|
||||
# Without enable_thinking=False / force_nonempty_content the fallback must
|
||||
# stay disabled: the response stays reasoning-only, content empty.
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
|
||||
reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
|
||||
|
||||
assert reasoning == "4"
|
||||
assert content == ""
|
||||
|
||||
@@ -776,6 +776,28 @@ class DelegatingParser(Parser):
|
||||
last_tc.function.arguments or ""
|
||||
) + self._tool_parser.get_remaining_unstreamed_args()
|
||||
|
||||
def finalize_generation(
|
||||
self,
|
||||
delta_message: DeltaMessage | None,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
state: StreamState,
|
||||
) -> DeltaMessage | None:
|
||||
"""Finalize generation for cases where generation was incomplete.
|
||||
For example, if streaming terminated before reasoning ended
|
||||
"""
|
||||
fallback_fn = getattr(
|
||||
self._reasoning_parser, "get_streaming_fallback_content", None
|
||||
)
|
||||
if fallback_fn is not None and not state.reasoning_ended:
|
||||
promoted = fallback_fn(state.previous_text, request)
|
||||
if promoted:
|
||||
if delta_message is None:
|
||||
delta_message = DeltaMessage()
|
||||
delta_message.content = (delta_message.content or "") + promoted
|
||||
|
||||
self._append_unstreamed_tool_args(delta_message)
|
||||
return delta_message
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
@@ -883,6 +905,6 @@ class DelegatingParser(Parser):
|
||||
state.previous_token_ids = current_token_ids
|
||||
|
||||
if finished:
|
||||
self._append_unstreamed_tool_args(delta_message)
|
||||
delta_message = self.finalize_generation(delta_message, request, state)
|
||||
|
||||
return delta_message
|
||||
|
||||
@@ -14,20 +14,35 @@ class NemotronV3ReasoningParser(DeepSeekR1ReasoningParser):
|
||||
Reasoning parser for Nemotron V3 models.
|
||||
"""
|
||||
|
||||
def extract_reasoning(
|
||||
self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, final_content = super().extract_reasoning(model_output, request)
|
||||
def _should_force_content(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> bool:
|
||||
chat_template_kwargs = getattr(request, "chat_template_kwargs", None)
|
||||
|
||||
if (
|
||||
return bool(
|
||||
chat_template_kwargs
|
||||
and (
|
||||
chat_template_kwargs.get("enable_thinking") is False
|
||||
or chat_template_kwargs.get("force_nonempty_content") is True
|
||||
)
|
||||
and (final_content is None or not final_content.strip())
|
||||
)
|
||||
|
||||
def extract_reasoning(
|
||||
self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, final_content = super().extract_reasoning(model_output, request)
|
||||
|
||||
if self._should_force_content(request) and (
|
||||
final_content is None or not final_content.strip()
|
||||
):
|
||||
reasoning, final_content = final_content, reasoning
|
||||
|
||||
return reasoning, final_content
|
||||
|
||||
def get_streaming_fallback_content(
|
||||
self, text: str, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> str | None:
|
||||
"""Reasoning to duplicate into content on the terminal streaming delta."""
|
||||
if not self._should_force_content(request):
|
||||
return None
|
||||
reasoning, _ = super().extract_reasoning(text, request)
|
||||
return reasoning
|
||||
|
||||
Reference in New Issue
Block a user