[Bugfix][Frontend] Respect max_output_tokens in the Harmony tool-call loop (#58551)
Signed-off-by: errmakov <ide404@gmail.com> Co-authored-by: Du Bin <8174807+dubin555@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Du Bin
Claude Opus 5.5
Flora Feng
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
83db8c8390
commit
7871963fcc
@@ -615,12 +615,14 @@ class HarmonyContext(ConversationContext):
|
||||
available_tools: list[str],
|
||||
function_tool_names: frozenset[str],
|
||||
response_parser: Parser | None = None,
|
||||
request: ResponsesRequest | None = None,
|
||||
):
|
||||
from vllm.parser.harmony import HarmonyParser, Segment
|
||||
|
||||
assert isinstance(response_parser, HarmonyParser)
|
||||
|
||||
self._messages = messages
|
||||
self.request = request
|
||||
self.response_parser: HarmonyParser = response_parser
|
||||
self.finish_reason: str | None = None
|
||||
self.available_tools = available_tools
|
||||
|
||||
@@ -443,6 +443,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
available_tools,
|
||||
function_tool_names,
|
||||
response_parser=response_parser,
|
||||
request=request,
|
||||
)
|
||||
else:
|
||||
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
|
||||
@@ -678,8 +679,12 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
tok_params=tok_params,
|
||||
)
|
||||
|
||||
sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
|
||||
engine_input
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
context.request.max_output_tokens if context.request else None,
|
||||
self._extract_prompt_len(engine_input),
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
)
|
||||
elif isinstance(context, ParsableContext):
|
||||
(engine_input,) = await self._render_next_turn(
|
||||
|
||||
Reference in New Issue
Block a user