[Bugfix] Fix reasoning dropped on streaming boundary deltas (#42691)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -137,6 +137,78 @@ def test_parse_delta_both_parsers(tokenizer, request_obj):
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def stream_chunks(parser, tokenizer, chunks, request_obj):
|
||||
"""Stream pre-split token-ID chunks through the parser."""
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for chunk in chunks:
|
||||
delta_text = tokenizer.decode(chunk)
|
||||
result = parser.parse_delta(
|
||||
delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def _boundary_chunks(tokenizer, parser):
|
||||
"""Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""
|
||||
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
|
||||
end_token_id = parser._reasoning_parser.end_token_id
|
||||
end_idx = token_ids.index(end_token_id)
|
||||
return [
|
||||
token_ids[: end_idx - 1],
|
||||
token_ids[end_idx - 1 : end_idx + 2],
|
||||
token_ids[end_idx + 2 :],
|
||||
]
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
|
||||
"""Regression: reasoning must not be lost when a multi-token delta
|
||||
spans the reasoning/tool-call boundary."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
chunks = _boundary_chunks(tokenizer, parser)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "think about this" in reasoning
|
||||
assert content == ""
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
|
||||
"""When no tool parser is active, boundary-spanning chunks must still
|
||||
preserve reasoning and pass post-</think> text as content."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
chunks = _boundary_chunks(tokenizer, parser)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "think about this" in reasoning
|
||||
assert len(tool_calls) == 0
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
|
||||
"""Regression: </think> must not leak into content when streaming
|
||||
token-by-token with reasoning=True, tool=False."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert "</think>" not in content
|
||||
assert "<think>" not in content
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
|
||||
"""Regression test for vllm-project/vllm#40466.
|
||||
|
||||
|
||||
@@ -634,6 +634,13 @@ class DelegatingParser(Parser):
|
||||
return False
|
||||
return self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
if self._reasoning_parser is None:
|
||||
return False
|
||||
return self._reasoning_parser.is_reasoning_end_streaming(input_ids, delta_ids)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
if self._reasoning_parser is None:
|
||||
return input_ids
|
||||
@@ -679,15 +686,16 @@ class DelegatingParser(Parser):
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
)
|
||||
# Hand off remaining content to tool parser
|
||||
if self._tool_parser and self.is_reasoning_end(delta_token_ids):
|
||||
if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids):
|
||||
state.reasoning_ended = True
|
||||
current_token_ids = self.extract_content_ids(delta_token_ids)
|
||||
if delta_message and delta_message.content:
|
||||
current_text = delta_message.content
|
||||
delta_message.content = None
|
||||
else:
|
||||
current_text = ""
|
||||
current_text = (
|
||||
delta_message.content
|
||||
if delta_message and delta_message.content
|
||||
else ""
|
||||
)
|
||||
delta_text = current_text
|
||||
delta_token_ids = current_token_ids
|
||||
|
||||
# Tool call extraction
|
||||
if self._in_tool_call_phase(state):
|
||||
@@ -698,6 +706,9 @@ class DelegatingParser(Parser):
|
||||
delta_text = current_text
|
||||
delta_token_ids = current_token_ids
|
||||
|
||||
# A boundary delta may carry both reasoning and tool call,
|
||||
# save it before the tool parser overwrites delta_message.
|
||||
reasoning = delta_message.reasoning if delta_message else None
|
||||
delta_message, state.function_name_returned = (
|
||||
self._extract_tool_calls_streaming(
|
||||
previous_text=state.previous_text,
|
||||
@@ -712,6 +723,11 @@ class DelegatingParser(Parser):
|
||||
function_name_returned=state.function_name_returned,
|
||||
)
|
||||
)
|
||||
if reasoning:
|
||||
if not delta_message:
|
||||
delta_message = DeltaMessage()
|
||||
delta_message.reasoning = reasoning
|
||||
|
||||
if (
|
||||
delta_message
|
||||
and delta_message.tool_calls
|
||||
|
||||
Reference in New Issue
Block a user