[Bugfix] Fix reasoning dropped on streaming boundary deltas (#42691)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-05-23 16:18:30 -07:00
committed by GitHub
parent 10d264a2b9
commit b32fe416ea
2 changed files with 95 additions and 7 deletions
+72
View File
@@ -137,6 +137,78 @@ def test_parse_delta_both_parsers(tokenizer, request_obj):
assert json.loads(tool_args) == {"city": "Dallas"}
def stream_chunks(parser, tokenizer, chunks, request_obj):
"""Stream pre-split token-ID chunks through the parser."""
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for chunk in chunks:
delta_text = tokenizer.decode(chunk)
result = parser.parse_delta(
delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids
)
prompt_token_ids = None
results.append(result)
return results
def _boundary_chunks(tokenizer, parser):
"""Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
end_token_id = parser._reasoning_parser.end_token_id
end_idx = token_ids.index(end_token_id)
return [
token_ids[: end_idx - 1],
token_ids[end_idx - 1 : end_idx + 2],
token_ids[end_idx + 2 :],
]
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
"""Regression: reasoning must not be lost when a multi-token delta
spans the reasoning/tool-call boundary."""
parser = make_parser(tokenizer, reasoning=True, tool=True)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
"""When no tool parser is active, boundary-spanning chunks must still
preserve reasoning and pass post-</think> text as content."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
"""Regression: </think> must not leak into content when streaming
token-by-token with reasoning=True, tool=False."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert "</think>" not in content
assert "<think>" not in content
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
"""Regression test for vllm-project/vllm#40466.
+23 -7
View File
@@ -634,6 +634,13 @@ class DelegatingParser(Parser):
return False
return self._reasoning_parser.is_reasoning_end(input_ids)
def is_reasoning_end_streaming(
self, input_ids: list[int], delta_ids: list[int]
) -> bool:
if self._reasoning_parser is None:
return False
return self._reasoning_parser.is_reasoning_end_streaming(input_ids, delta_ids)
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
if self._reasoning_parser is None:
return input_ids
@@ -679,15 +686,16 @@ class DelegatingParser(Parser):
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
)
# Hand off remaining content to tool parser
if self._tool_parser and self.is_reasoning_end(delta_token_ids):
if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids):
state.reasoning_ended = True
current_token_ids = self.extract_content_ids(delta_token_ids)
if delta_message and delta_message.content:
current_text = delta_message.content
delta_message.content = None
else:
current_text = ""
current_text = (
delta_message.content
if delta_message and delta_message.content
else ""
)
delta_text = current_text
delta_token_ids = current_token_ids
# Tool call extraction
if self._in_tool_call_phase(state):
@@ -698,6 +706,9 @@ class DelegatingParser(Parser):
delta_text = current_text
delta_token_ids = current_token_ids
# A boundary delta may carry both reasoning and tool call,
# save it before the tool parser overwrites delta_message.
reasoning = delta_message.reasoning if delta_message else None
delta_message, state.function_name_returned = (
self._extract_tool_calls_streaming(
previous_text=state.previous_text,
@@ -712,6 +723,11 @@ class DelegatingParser(Parser):
function_name_returned=state.function_name_returned,
)
)
if reasoning:
if not delta_message:
delta_message = DeltaMessage()
delta_message.reasoning = reasoning
if (
delta_message
and delta_message.tool_calls