[Bugfix][Frontend] Apply Harmony adjust_request in batched chat completions (#58958)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -259,15 +259,7 @@ class OnlineRenderer:
|
||||
)
|
||||
else:
|
||||
# For GPT-OSS.
|
||||
if self.parser is not None:
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
|
||||
self.adjust_harmony_request(request)
|
||||
should_include_tools = tool_dicts is not None
|
||||
conversation, engine_inputs = self._make_request_with_harmony(
|
||||
request, should_include_tools
|
||||
@@ -355,14 +347,7 @@ class OnlineRenderer:
|
||||
previous_response_outputs: list[ResponseOutputItem] | None,
|
||||
tool_server: "ToolServer | None",
|
||||
) -> ResponsesRenderResult | ErrorResponse:
|
||||
if self.parser is not None:
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
self.adjust_harmony_request(request)
|
||||
|
||||
if previous_messages is not None and any(
|
||||
not isinstance(message, OpenAIMessage) for message in previous_messages
|
||||
@@ -488,6 +473,24 @@ class OnlineRenderer:
|
||||
engine_input=engine_inputs[0],
|
||||
)
|
||||
|
||||
def adjust_harmony_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> None:
|
||||
"""Apply the Harmony parser's ``adjust_request`` to ``request`` in place.
|
||||
|
||||
Every Harmony render path must call this before sampling params are
|
||||
built from ``request``.
|
||||
"""
|
||||
if self.parser is None:
|
||||
return
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
Reference in New Issue
Block a user