[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)

Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com>
Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com>
Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuvraj Singh Bhadoria
2026-10-10 01:15:22 +08:00
committed by GitHub
co-authored by YuvrajSinghBhadoria2 Cyrus Leung Claude Opus 4.8
parent 89ac8d971b
commit 1a3a7a8f01
7 changed files with 252 additions and 32 deletions
+118
View File
@@ -247,6 +247,8 @@ class TestRenderPrompt:
)
assert len(results) == 1
# max_output_tokens defaults to 0, so there is no output budget to
# reserve and -1 truncates to the full max_total_tokens.
assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens
def test_truncation_left(self):
@@ -505,6 +507,122 @@ class TestRenderPrompt:
assert renderer.tokenizer._captured_text_len <= 100
class TestMaxTokensNotReservation:
"""Regression tests for GitHub issue #42474.
``max_tokens`` must be treated as an upper bound on generation length, not
as a hard reservation that is subtracted from the available input space.
"""
def test_large_max_tokens_does_not_eat_prompt_budget(self):
"""Prompt fits in context even when max_tokens is large."""
renderer = _build_renderer(MockModelConfig())
# 50-token prompt with a 100-token context window and max_tokens=60.
# Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error.
# After the fix: max_input_tokens = 100 → 50 ≤ 100 → OK.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=60),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 50
def test_max_tokens_zero_still_rejects_oversized_prompt(self):
"""Genuine prompt overflow is still caught regardless of max_tokens."""
renderer = _build_renderer(MockModelConfig())
# 150-token prompt with 100-token context → should still fail.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 150)
)
with pytest.raises(VLLMValidationError, match="maximum context length"):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=0),
)
def test_negative_truncation_uses_max_total(self):
"""truncate_prompt_tokens=-1 reserves one output token."""
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 200)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
max_output_tokens=40,
truncate_prompt_tokens=-1,
),
)
# -1 maps to max_total_tokens (100) rather than the old
# max_total - max_output (60), minus the one token held back so
# truncation cannot leave a zero-token output budget.
assert len(results[0]["prompt_token_ids"]) == 99
def test_negative_pad_uses_max_total(self):
"""pad_prompt_tokens=-1 pads to max_total_tokens, not max_input."""
renderer = _build_renderer(MockModelConfig())
pad_id = renderer.tokenizer.pad_token_id
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 30)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
max_output_tokens=40,
pad_prompt_tokens=-1,
),
)
# -1 should map to max_total_tokens (100), not max_total - max_output (60).
assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70
def test_max_tokens_exceeds_context_is_still_clamped_by_api(self):
"""Renderer allows the prompt; API layer clamps max_tokens later."""
renderer = _build_renderer(MockModelConfig())
# Simulates the reported scenario: max_tokens > context - prompt_length.
# Renderer should accept the 40-token prompt; get_max_tokens() clamps
# sampling max_tokens to 60 at the entrypoint layer.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 40)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=80),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 40
def test_with_kwargs_preserves_output_budget(self):
"""with_kwargs without an explicit max_length keeps the output budget.
Under the old semantics `with_kwargs()` recomputed
`max_output_tokens = max_total - max_input`, which (after this fix)
would have silently reset a real output budget to zero.
"""
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
chained = tok_params.with_kwargs(truncation_side="left")
assert chained.max_output_tokens == 60
assert chained.max_input_tokens == 100
def test_with_kwargs_explicit_max_length_sets_budget(self):
"""An explicit max_length still re-derives the output budget."""
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
chained = tok_params.with_kwargs(max_length=80)
assert chained.max_output_tokens == 20
assert chained.max_input_tokens == 100
class TestRenderEmbedPrompt:
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
"""Helper to create base64-encoded tensor bytes."""