[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)
Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com> Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com> Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
YuvrajSinghBhadoria2
Cyrus Leung
Claude Opus 4.8
parent
89ac8d971b
commit
1a3a7a8f01
@@ -515,7 +515,11 @@ async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
|
||||
result = await renderer.render_responses(request)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
|
||||
# `max_tokens` is an output upper bound rather than a reservation off the
|
||||
# context window, so the prompt budget is the full max_model_len minus the
|
||||
# one token held back for output: 6 tokens truncate to 4, not to
|
||||
# max_model_len - max_output_tokens.
|
||||
assert result.engine_input["prompt_token_ids"] == [3, 4, 5, 6]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user