[Bugfix] Fix reasoning dropped on streaming boundary deltas (#42691)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-05-23 16:18:30 -07:00
committed by GitHub
parent 10d264a2b9
commit b32fe416ea
2 changed files with 95 additions and 7 deletions
+72
View File
@@ -137,6 +137,78 @@ def test_parse_delta_both_parsers(tokenizer, request_obj):
assert json.loads(tool_args) == {"city": "Dallas"}
def stream_chunks(parser, tokenizer, chunks, request_obj):
"""Stream pre-split token-ID chunks through the parser."""
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for chunk in chunks:
delta_text = tokenizer.decode(chunk)
result = parser.parse_delta(
delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids
)
prompt_token_ids = None
results.append(result)
return results
def _boundary_chunks(tokenizer, parser):
"""Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
end_token_id = parser._reasoning_parser.end_token_id
end_idx = token_ids.index(end_token_id)
return [
token_ids[: end_idx - 1],
token_ids[end_idx - 1 : end_idx + 2],
token_ids[end_idx + 2 :],
]
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
"""Regression: reasoning must not be lost when a multi-token delta
spans the reasoning/tool-call boundary."""
parser = make_parser(tokenizer, reasoning=True, tool=True)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
"""When no tool parser is active, boundary-spanning chunks must still
preserve reasoning and pass post-</think> text as content."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
"""Regression: </think> must not leak into content when streaming
token-by-token with reasoning=True, tool=False."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert "</think>" not in content
assert "<think>" not in content
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
"""Regression test for vllm-project/vllm#40466.