[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)

Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com>
Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com>
Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuvraj Singh Bhadoria
2026-10-10 01:15:22 +08:00
committed by GitHub
co-authored by YuvrajSinghBhadoria2 Cyrus Leung Claude Opus 4.8
parent 89ac8d971b
commit 1a3a7a8f01
7 changed files with 252 additions and 32 deletions
@@ -1684,7 +1684,14 @@ def test_serving_chat_starts_without_tokenizer():
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
directly, we must still apply input length + max_tokens validation.
directly, we must still apply input length validation and clamp the
generation budget to the context the prompt leaves over.
`max_tokens` is an upper bound on generation rather than a reservation off
the context window, so a 95-token prompt against `max_model_len=100` is
accepted even though the client asked for 10 output tokens. Genuine
overflow is covered by
test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected.
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
@@ -1696,9 +1703,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
tokenizer=mock_tokenizer,
)
# Force the Mistral chat template renderer to return token IDs.
# Choose a prompt length that is < max_model_len, but large enough that
# adding max_tokens should exceed the model context window.
# Force the Mistral chat template renderer to return token IDs. The prompt
# fits the context on its own; only the naive prompt+output sum overflows.
mock_renderer.render_messages_async = AsyncMock(
return_value=(
[],
@@ -1715,8 +1721,12 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
max_tokens=10,
)
with pytest.raises(VLLMValidationError):
await serving_chat.create_chat_completion(req)
# Must not raise: the prompt alone fits within max_model_len.
await serving_chat.create_chat_completion(req)
# get_max_tokens() clamps the sampling budget to the 5 tokens left over.
sampling_params = mock_engine.generate.call_args.args[1]
assert 1 <= sampling_params.max_tokens <= 5
@pytest.mark.asyncio
@@ -515,7 +515,11 @@ async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
result = await renderer.render_responses(request)
assert not isinstance(result, ErrorResponse)
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
# `max_tokens` is an output upper bound rather than a reservation off the
# context window, so the prompt budget is the full max_model_len minus the
# one token held back for output: 6 tokens truncate to 4, not to
# max_model_len - max_output_tokens.
assert result.engine_input["prompt_token_ids"] == [3, 4, 5, 6]
@pytest.mark.asyncio
@@ -4,12 +4,15 @@
underlying packing helpers, plus Cohere-specific serving helpers."""
import struct
from unittest.mock import Mock
import numpy as np
import pybase64 as base64
import pytest
from vllm.config import ModelConfig, PoolerConfig
from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingCompletionRequest,
build_typed_embeddings,
)
@@ -162,3 +165,59 @@ class TestBuildTypedEmbeddingsMultiple:
def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
assert result.float is not None
def _embedding_model_config(
*,
max_model_len: int = 128,
max_embed_len: int | None = None,
enable_chunked_processing: bool = False,
) -> Mock:
model_config = Mock(spec=ModelConfig)
model_config.max_model_len = max_model_len
model_config.encoder_config = None
model_config.pooler_config = PoolerConfig(
seq_pooling_type="CLS",
max_embed_len=max_embed_len,
enable_chunked_processing=enable_chunked_processing,
)
return model_config
class TestMaxEmbedLenCapsInput:
"""``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
expressed directly as ``max_total_tokens``."""
def test_non_chunked_max_embed_len_caps_input(self):
cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 64
assert tok_params.max_output_tokens == 0
def test_chunked_max_embed_len_caps_input(self):
cfg = _embedding_model_config(
max_model_len=128, max_embed_len=64, enable_chunked_processing=True
)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 64
assert tok_params.max_output_tokens == 0
def test_no_max_embed_len_falls_back_to_model_len(self):
cfg = _embedding_model_config(max_model_len=128)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 128
assert tok_params.max_output_tokens == 0
@@ -55,6 +55,8 @@ def test_max_truncation_size(vllm_model):
prompt_tokens = vllm_output[0].prompt_token_ids
# Pooling requests emit an embedding rather than generated tokens, so -1
# truncates to the full context length.
assert len(prompt_tokens) == max_model_len
+118
View File
@@ -247,6 +247,8 @@ class TestRenderPrompt:
)
assert len(results) == 1
# max_output_tokens defaults to 0, so there is no output budget to
# reserve and -1 truncates to the full max_total_tokens.
assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens
def test_truncation_left(self):
@@ -505,6 +507,122 @@ class TestRenderPrompt:
assert renderer.tokenizer._captured_text_len <= 100
class TestMaxTokensNotReservation:
"""Regression tests for GitHub issue #42474.
``max_tokens`` must be treated as an upper bound on generation length, not
as a hard reservation that is subtracted from the available input space.
"""
def test_large_max_tokens_does_not_eat_prompt_budget(self):
"""Prompt fits in context even when max_tokens is large."""
renderer = _build_renderer(MockModelConfig())
# 50-token prompt with a 100-token context window and max_tokens=60.
# Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error.
# After the fix: max_input_tokens = 100 → 50 ≤ 100 → OK.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=60),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 50
def test_max_tokens_zero_still_rejects_oversized_prompt(self):
"""Genuine prompt overflow is still caught regardless of max_tokens."""
renderer = _build_renderer(MockModelConfig())
# 150-token prompt with 100-token context → should still fail.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 150)
)
with pytest.raises(VLLMValidationError, match="maximum context length"):
renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=0),
)
def test_negative_truncation_uses_max_total(self):
"""truncate_prompt_tokens=-1 reserves one output token."""
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 200)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
max_output_tokens=40,
truncate_prompt_tokens=-1,
),
)
# -1 maps to max_total_tokens (100) rather than the old
# max_total - max_output (60), minus the one token held back so
# truncation cannot leave a zero-token output budget.
assert len(results[0]["prompt_token_ids"]) == 99
def test_negative_pad_uses_max_total(self):
"""pad_prompt_tokens=-1 pads to max_total_tokens, not max_input."""
renderer = _build_renderer(MockModelConfig())
pad_id = renderer.tokenizer.pad_token_id
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 30)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
max_output_tokens=40,
pad_prompt_tokens=-1,
),
)
# -1 should map to max_total_tokens (100), not max_total - max_output (60).
assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70
def test_max_tokens_exceeds_context_is_still_clamped_by_api(self):
"""Renderer allows the prompt; API layer clamps max_tokens later."""
renderer = _build_renderer(MockModelConfig())
# Simulates the reported scenario: max_tokens > context - prompt_length.
# Renderer should accept the 40-token prompt; get_max_tokens() clamps
# sampling max_tokens to 60 at the entrypoint layer.
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 40)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(max_total_tokens=100, max_output_tokens=80),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 40
def test_with_kwargs_preserves_output_budget(self):
"""with_kwargs without an explicit max_length keeps the output budget.
Under the old semantics `with_kwargs()` recomputed
`max_output_tokens = max_total - max_input`, which (after this fix)
would have silently reset a real output budget to zero.
"""
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
chained = tok_params.with_kwargs(truncation_side="left")
assert chained.max_output_tokens == 60
assert chained.max_input_tokens == 100
def test_with_kwargs_explicit_max_length_sets_budget(self):
"""An explicit max_length still re-derives the output budget."""
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
chained = tok_params.with_kwargs(max_length=80)
assert chained.max_output_tokens == 20
assert chained.max_input_tokens == 100
class TestRenderEmbedPrompt:
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
"""Helper to create base64-encoded tensor bytes."""
+3 -3
View File
@@ -183,15 +183,15 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
if pooler_config.enable_chunked_processing:
max_total_tokens = pooler_config.max_embed_len
else:
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
max_output_tokens = default_max_total_tokens - max_embed_len
max_total_tokens = (
pooler_config.max_embed_len or default_max_total_tokens
)
tok_params = self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
max_output_tokens_param="max_model_len - max_embed_len",
)
if (
+49 -22
View File
@@ -207,11 +207,38 @@ class TokenizeParams:
@property
def max_input_tokens(self) -> int | None:
"""Maximum allowed number of input tokens."""
"""Maximum allowed number of input tokens.
``max_output_tokens`` is *not* subtracted here: the value represents the
actual model context length, not a reservation of output space.
Downstream clamping of the sampling ``max_tokens`` is handled by
``get_max_tokens()`` in the entrypoint layer.
"""
if self.max_total_tokens is None:
return None
return self.max_total_tokens - self.max_output_tokens
return self.max_total_tokens
@property
def max_truncation_tokens(self) -> int | None:
"""Maximum prompt length that truncation may reduce a prompt to.
For generative requests this is `max_input_tokens` less one, holding a
token of context back for output. Truncating to the full
`max_total_tokens` would leave `get_max_tokens()` clamping the sampling
budget to 0, turning an over-long prompt into a silent zero-token
generation.
Pooling requests set `max_output_tokens = 0` and emit an embedding
rather than generated tokens, so they keep the full input budget.
"""
max_input_tokens = self.max_input_tokens
if max_input_tokens is None:
return None
if self.max_output_tokens == 0:
return max_input_tokens
return max(0, max_input_tokens - 1)
def __post_init__(self) -> None:
max_total_tokens = self.max_total_tokens
@@ -247,14 +274,15 @@ class TokenizeParams:
):
raise VLLMValidationError(
f"{self.truncate_prompt_tokens_param}={truncate_prompt_tokens} "
f"cannot be greater than {self.max_total_tokens_param} - "
f"{self.max_output_tokens_param} = {max_input_tokens}. "
f"cannot be greater than {self.max_total_tokens_param} "
f"= {max_input_tokens}. "
f"Please request a smaller truncation size.",
parameter=self.truncate_prompt_tokens_param,
value=truncate_prompt_tokens,
)
def with_kwargs(self, **tokenization_kwargs: Any):
has_max_length = "max_length" in tokenization_kwargs
max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens)
pad_prompt_tokens = tokenization_kwargs.pop(
"pad_prompt_tokens", self.pad_prompt_tokens
@@ -301,13 +329,17 @@ class TokenizeParams:
max_total_tokens = self.max_total_tokens
if has_max_length and max_total_tokens is not None and max_length is not None:
max_output_tokens = max_total_tokens - max_length
else:
# No explicit ``max_length`` override: preserve the original output
# budget instead of recomputing it from ``max_input_tokens`` (which
# is now the full context length).
max_output_tokens = self.max_output_tokens
return TokenizeParams(
max_total_tokens=max_total_tokens,
max_output_tokens=(
0
if max_total_tokens is None or max_length is None
else max_total_tokens - max_length
),
max_output_tokens=max_output_tokens,
pad_prompt_tokens=pad_prompt_tokens,
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
@@ -320,7 +352,7 @@ class TokenizeParams:
"""The arguments to pass to `tokenizer.encode`."""
max_length = self.truncate_prompt_tokens
if max_length is not None and max_length < 0:
max_length = self.max_input_tokens
max_length = self.max_truncation_tokens
elif max_length is None and self.max_input_tokens is not None:
# This prevents tokenization from taking up more resources than necessary
# while still failing `self._token_len_check` as expected by users
@@ -355,13 +387,11 @@ class TokenizeParams:
if len(text) > max_input_chars:
raise VLLMValidationError(
f"This model's maximum context length is "
f"{self.max_total_tokens} tokens. However, you requested "
f"{self.max_output_tokens} output tokens and your prompt "
f"{self.max_total_tokens} tokens. However, your prompt "
f"contains {len(text)} characters (more than "
f"{max_input_chars} characters, which is the upper bound "
f"for {max_input_tokens} input tokens). "
f"Please reduce the length of the input prompt or the "
f"number of requested output tokens.",
f"Please reduce the length of the input prompt.",
parameter="input_text",
value=len(text),
)
@@ -459,7 +489,7 @@ class TokenizeParams:
"""
max_length = self.truncate_prompt_tokens
if max_length is not None and max_length < 0:
max_length = self.max_input_tokens
max_length = self.max_truncation_tokens
if max_length is None or max_length >= length:
return None
@@ -494,15 +524,12 @@ class TokenizeParams:
# max_input_tokens + 1 (see get_encode_kwargs), so the
# actual prompt length could be larger.
qualifier = "at least " if token_count == max_input_tokens + 1 else ""
total = token_count + self.max_output_tokens
raise VLLMValidationError(
f"This model's maximum context length is "
f"{self.max_total_tokens} tokens. However, you requested "
f"{self.max_output_tokens} output tokens and your prompt "
f"contains {qualifier}{token_count} input tokens, "
f"for a total of {qualifier}{total} tokens. "
f"Please reduce the length of the input prompt or the "
f"number of requested output tokens.",
f"{self.max_total_tokens} tokens. However, your prompt "
f"contains {qualifier}{token_count} input tokens, which "
f"exceeds this limit even without any output tokens. "
f"Please reduce the length of the input prompt.",
parameter="input_tokens",
value=token_count,
)