[Bugfix] Fix shared-object aliasing in n>1 streaming with tool calls (#38158)
Signed-off-by: Yifan Zong <yzong@redhat.com> Signed-off-by: Yifan <yzong@redhat.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -548,7 +548,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# all_previous_token_ids will not be used twice in the same iteration.
|
||||
if tool_choice_auto or reasoning_parser:
|
||||
# These are only required in "auto" tool choice case
|
||||
all_previous_token_ids = [[]] * num_choices
|
||||
all_previous_token_ids = [[] for _ in range(num_choices)]
|
||||
# For reasoning parser and tool call all enabled
|
||||
added_content_delta_arr = [False] * num_choices
|
||||
reasoning_end_arr = [False] * num_choices
|
||||
@@ -566,7 +566,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
tool_parsers: list[ToolParser | None] = [
|
||||
self.tool_parser(tokenizer, request.tools)
|
||||
] * num_choices
|
||||
for _ in range(num_choices)
|
||||
]
|
||||
else:
|
||||
tool_parsers = [None] * num_choices
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user