[Bugfix] Fix unstreamed tool call args dropped in Responses API streaming (#44348)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -58,7 +58,11 @@ def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
|
||||
for tid in token_ids:
|
||||
delta_text = tokenizer.decode([tid])
|
||||
result = parser.parse_delta(
|
||||
delta_text, [tid], request, prompt_token_ids=prompt_token_ids
|
||||
delta_text,
|
||||
[tid],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
@@ -146,7 +150,11 @@ def stream_chunks(parser, tokenizer, chunks, request_obj):
|
||||
for chunk in chunks:
|
||||
delta_text = tokenizer.decode(chunk)
|
||||
result = parser.parse_delta(
|
||||
delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids
|
||||
delta_text,
|
||||
chunk,
|
||||
request_obj,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
|
||||
@@ -1408,6 +1408,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request,
|
||||
prompt_token_ids=ctx.last_output.prompt_token_ids,
|
||||
finished=output.finish_reason is not None,
|
||||
)
|
||||
else:
|
||||
delta_message = DeltaMessage(content=output.text)
|
||||
|
||||
@@ -344,7 +344,8 @@ class Parser:
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
finished: bool = False,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
"""Parse a single streaming delta, orchestrating reasoning then
|
||||
tool call extraction via internal stream state.
|
||||
@@ -809,7 +810,8 @@ class DelegatingParser(Parser):
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
finished: bool = False,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
state = self._stream_state
|
||||
|
||||
|
||||
Reference in New Issue
Block a user