[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)
Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com> Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com> Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
YuvrajSinghBhadoria2
Cyrus Leung
Claude Opus 4.8
parent
89ac8d971b
commit
1a3a7a8f01
@@ -1684,7 +1684,14 @@ def test_serving_chat_starts_without_tokenizer():
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
"""Regression test: when the Mistral tokenizer path returns token IDs
|
||||
directly, we must still apply input length + max_tokens validation.
|
||||
directly, we must still apply input length validation and clamp the
|
||||
generation budget to the context the prompt leaves over.
|
||||
|
||||
`max_tokens` is an upper bound on generation rather than a reservation off
|
||||
the context window, so a 95-token prompt against `max_model_len=100` is
|
||||
accepted even though the client asked for 10 output tokens. Genuine
|
||||
overflow is covered by
|
||||
test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected.
|
||||
"""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
@@ -1696,9 +1703,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
|
||||
tokenizer=mock_tokenizer,
|
||||
)
|
||||
# Force the Mistral chat template renderer to return token IDs.
|
||||
# Choose a prompt length that is < max_model_len, but large enough that
|
||||
# adding max_tokens should exceed the model context window.
|
||||
# Force the Mistral chat template renderer to return token IDs. The prompt
|
||||
# fits the context on its own; only the naive prompt+output sum overflows.
|
||||
mock_renderer.render_messages_async = AsyncMock(
|
||||
return_value=(
|
||||
[],
|
||||
@@ -1715,8 +1721,12 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
max_tokens=10,
|
||||
)
|
||||
|
||||
with pytest.raises(VLLMValidationError):
|
||||
await serving_chat.create_chat_completion(req)
|
||||
# Must not raise: the prompt alone fits within max_model_len.
|
||||
await serving_chat.create_chat_completion(req)
|
||||
|
||||
# get_max_tokens() clamps the sampling budget to the 5 tokens left over.
|
||||
sampling_params = mock_engine.generate.call_args.args[1]
|
||||
assert 1 <= sampling_params.max_tokens <= 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -515,7 +515,11 @@ async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
|
||||
result = await renderer.render_responses(request)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
|
||||
# `max_tokens` is an output upper bound rather than a reservation off the
|
||||
# context window, so the prompt budget is the full max_model_len minus the
|
||||
# one token held back for output: 6 tokens truncate to 4, not to
|
||||
# max_model_len - max_output_tokens.
|
||||
assert result.engine_input["prompt_token_ids"] == [3, 4, 5, 6]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -4,12 +4,15 @@
|
||||
underlying packing helpers, plus Cohere-specific serving helpers."""
|
||||
|
||||
import struct
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingCompletionRequest,
|
||||
build_typed_embeddings,
|
||||
)
|
||||
|
||||
@@ -162,3 +165,59 @@ class TestBuildTypedEmbeddingsMultiple:
|
||||
def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
|
||||
assert result.float is not None
|
||||
|
||||
|
||||
def _embedding_model_config(
|
||||
*,
|
||||
max_model_len: int = 128,
|
||||
max_embed_len: int | None = None,
|
||||
enable_chunked_processing: bool = False,
|
||||
) -> Mock:
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.max_model_len = max_model_len
|
||||
model_config.encoder_config = None
|
||||
model_config.pooler_config = PoolerConfig(
|
||||
seq_pooling_type="CLS",
|
||||
max_embed_len=max_embed_len,
|
||||
enable_chunked_processing=enable_chunked_processing,
|
||||
)
|
||||
return model_config
|
||||
|
||||
|
||||
class TestMaxEmbedLenCapsInput:
|
||||
"""``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
|
||||
setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
|
||||
``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
|
||||
expressed directly as ``max_total_tokens``."""
|
||||
|
||||
def test_non_chunked_max_embed_len_caps_input(self):
|
||||
cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 64
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
def test_chunked_max_embed_len_caps_input(self):
|
||||
cfg = _embedding_model_config(
|
||||
max_model_len=128, max_embed_len=64, enable_chunked_processing=True
|
||||
)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 64
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
def test_no_max_embed_len_falls_back_to_model_len(self):
|
||||
cfg = _embedding_model_config(max_model_len=128)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 128
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
@@ -55,6 +55,8 @@ def test_max_truncation_size(vllm_model):
|
||||
|
||||
prompt_tokens = vllm_output[0].prompt_token_ids
|
||||
|
||||
# Pooling requests emit an embedding rather than generated tokens, so -1
|
||||
# truncates to the full context length.
|
||||
assert len(prompt_tokens) == max_model_len
|
||||
|
||||
|
||||
|
||||
@@ -247,6 +247,8 @@ class TestRenderPrompt:
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
# max_output_tokens defaults to 0, so there is no output budget to
|
||||
# reserve and -1 truncates to the full max_total_tokens.
|
||||
assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens
|
||||
|
||||
def test_truncation_left(self):
|
||||
@@ -505,6 +507,122 @@ class TestRenderPrompt:
|
||||
assert renderer.tokenizer._captured_text_len <= 100
|
||||
|
||||
|
||||
class TestMaxTokensNotReservation:
|
||||
"""Regression tests for GitHub issue #42474.
|
||||
|
||||
``max_tokens`` must be treated as an upper bound on generation length, not
|
||||
as a hard reservation that is subtracted from the available input space.
|
||||
"""
|
||||
|
||||
def test_large_max_tokens_does_not_eat_prompt_budget(self):
|
||||
"""Prompt fits in context even when max_tokens is large."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# 50-token prompt with a 100-token context window and max_tokens=60.
|
||||
# Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error.
|
||||
# After the fix: max_input_tokens = 100 → 50 ≤ 100 → OK.
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, max_output_tokens=60),
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 50
|
||||
|
||||
def test_max_tokens_zero_still_rejects_oversized_prompt(self):
|
||||
"""Genuine prompt overflow is still caught regardless of max_tokens."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# 150-token prompt with 100-token context → should still fail.
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 150)
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="maximum context length"):
|
||||
renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, max_output_tokens=0),
|
||||
)
|
||||
|
||||
def test_negative_truncation_uses_max_total(self):
|
||||
"""truncate_prompt_tokens=-1 reserves one output token."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 200)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
max_output_tokens=40,
|
||||
truncate_prompt_tokens=-1,
|
||||
),
|
||||
)
|
||||
# -1 maps to max_total_tokens (100) rather than the old
|
||||
# max_total - max_output (60), minus the one token held back so
|
||||
# truncation cannot leave a zero-token output budget.
|
||||
assert len(results[0]["prompt_token_ids"]) == 99
|
||||
|
||||
def test_negative_pad_uses_max_total(self):
|
||||
"""pad_prompt_tokens=-1 pads to max_total_tokens, not max_input."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
pad_id = renderer.tokenizer.pad_token_id
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 30)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
max_output_tokens=40,
|
||||
pad_prompt_tokens=-1,
|
||||
),
|
||||
)
|
||||
# -1 should map to max_total_tokens (100), not max_total - max_output (60).
|
||||
assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70
|
||||
|
||||
def test_max_tokens_exceeds_context_is_still_clamped_by_api(self):
|
||||
"""Renderer allows the prompt; API layer clamps max_tokens later."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
# Simulates the reported scenario: max_tokens > context - prompt_length.
|
||||
# Renderer should accept the 40-token prompt; get_max_tokens() clamps
|
||||
# sampling max_tokens to 60 at the entrypoint layer.
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 40)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(max_total_tokens=100, max_output_tokens=80),
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 40
|
||||
|
||||
def test_with_kwargs_preserves_output_budget(self):
|
||||
"""with_kwargs without an explicit max_length keeps the output budget.
|
||||
|
||||
Under the old semantics `with_kwargs()` recomputed
|
||||
`max_output_tokens = max_total - max_input`, which (after this fix)
|
||||
would have silently reset a real output budget to zero.
|
||||
"""
|
||||
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
|
||||
chained = tok_params.with_kwargs(truncation_side="left")
|
||||
|
||||
assert chained.max_output_tokens == 60
|
||||
assert chained.max_input_tokens == 100
|
||||
|
||||
def test_with_kwargs_explicit_max_length_sets_budget(self):
|
||||
"""An explicit max_length still re-derives the output budget."""
|
||||
tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
|
||||
chained = tok_params.with_kwargs(max_length=80)
|
||||
|
||||
assert chained.max_output_tokens == 20
|
||||
assert chained.max_input_tokens == 100
|
||||
|
||||
|
||||
class TestRenderEmbedPrompt:
|
||||
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
|
||||
"""Helper to create base64-encoded tensor bytes."""
|
||||
|
||||
@@ -183,15 +183,15 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
if pooler_config.enable_chunked_processing:
|
||||
max_total_tokens = pooler_config.max_embed_len
|
||||
else:
|
||||
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
|
||||
max_output_tokens = default_max_total_tokens - max_embed_len
|
||||
max_total_tokens = (
|
||||
pooler_config.max_embed_len or default_max_total_tokens
|
||||
)
|
||||
|
||||
tok_params = self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
+49
-22
@@ -207,11 +207,38 @@ class TokenizeParams:
|
||||
|
||||
@property
|
||||
def max_input_tokens(self) -> int | None:
|
||||
"""Maximum allowed number of input tokens."""
|
||||
"""Maximum allowed number of input tokens.
|
||||
|
||||
``max_output_tokens`` is *not* subtracted here: the value represents the
|
||||
actual model context length, not a reservation of output space.
|
||||
Downstream clamping of the sampling ``max_tokens`` is handled by
|
||||
``get_max_tokens()`` in the entrypoint layer.
|
||||
"""
|
||||
if self.max_total_tokens is None:
|
||||
return None
|
||||
|
||||
return self.max_total_tokens - self.max_output_tokens
|
||||
return self.max_total_tokens
|
||||
|
||||
@property
|
||||
def max_truncation_tokens(self) -> int | None:
|
||||
"""Maximum prompt length that truncation may reduce a prompt to.
|
||||
|
||||
For generative requests this is `max_input_tokens` less one, holding a
|
||||
token of context back for output. Truncating to the full
|
||||
`max_total_tokens` would leave `get_max_tokens()` clamping the sampling
|
||||
budget to 0, turning an over-long prompt into a silent zero-token
|
||||
generation.
|
||||
|
||||
Pooling requests set `max_output_tokens = 0` and emit an embedding
|
||||
rather than generated tokens, so they keep the full input budget.
|
||||
"""
|
||||
max_input_tokens = self.max_input_tokens
|
||||
if max_input_tokens is None:
|
||||
return None
|
||||
if self.max_output_tokens == 0:
|
||||
return max_input_tokens
|
||||
|
||||
return max(0, max_input_tokens - 1)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
max_total_tokens = self.max_total_tokens
|
||||
@@ -247,14 +274,15 @@ class TokenizeParams:
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
f"{self.truncate_prompt_tokens_param}={truncate_prompt_tokens} "
|
||||
f"cannot be greater than {self.max_total_tokens_param} - "
|
||||
f"{self.max_output_tokens_param} = {max_input_tokens}. "
|
||||
f"cannot be greater than {self.max_total_tokens_param} "
|
||||
f"= {max_input_tokens}. "
|
||||
f"Please request a smaller truncation size.",
|
||||
parameter=self.truncate_prompt_tokens_param,
|
||||
value=truncate_prompt_tokens,
|
||||
)
|
||||
|
||||
def with_kwargs(self, **tokenization_kwargs: Any):
|
||||
has_max_length = "max_length" in tokenization_kwargs
|
||||
max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens)
|
||||
pad_prompt_tokens = tokenization_kwargs.pop(
|
||||
"pad_prompt_tokens", self.pad_prompt_tokens
|
||||
@@ -301,13 +329,17 @@ class TokenizeParams:
|
||||
|
||||
max_total_tokens = self.max_total_tokens
|
||||
|
||||
if has_max_length and max_total_tokens is not None and max_length is not None:
|
||||
max_output_tokens = max_total_tokens - max_length
|
||||
else:
|
||||
# No explicit ``max_length`` override: preserve the original output
|
||||
# budget instead of recomputing it from ``max_input_tokens`` (which
|
||||
# is now the full context length).
|
||||
max_output_tokens = self.max_output_tokens
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=(
|
||||
0
|
||||
if max_total_tokens is None or max_length is None
|
||||
else max_total_tokens - max_length
|
||||
),
|
||||
max_output_tokens=max_output_tokens,
|
||||
pad_prompt_tokens=pad_prompt_tokens,
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
@@ -320,7 +352,7 @@ class TokenizeParams:
|
||||
"""The arguments to pass to `tokenizer.encode`."""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = self.max_input_tokens
|
||||
max_length = self.max_truncation_tokens
|
||||
elif max_length is None and self.max_input_tokens is not None:
|
||||
# This prevents tokenization from taking up more resources than necessary
|
||||
# while still failing `self._token_len_check` as expected by users
|
||||
@@ -355,13 +387,11 @@ class TokenizeParams:
|
||||
if len(text) > max_input_chars:
|
||||
raise VLLMValidationError(
|
||||
f"This model's maximum context length is "
|
||||
f"{self.max_total_tokens} tokens. However, you requested "
|
||||
f"{self.max_output_tokens} output tokens and your prompt "
|
||||
f"{self.max_total_tokens} tokens. However, your prompt "
|
||||
f"contains {len(text)} characters (more than "
|
||||
f"{max_input_chars} characters, which is the upper bound "
|
||||
f"for {max_input_tokens} input tokens). "
|
||||
f"Please reduce the length of the input prompt or the "
|
||||
f"number of requested output tokens.",
|
||||
f"Please reduce the length of the input prompt.",
|
||||
parameter="input_text",
|
||||
value=len(text),
|
||||
)
|
||||
@@ -459,7 +489,7 @@ class TokenizeParams:
|
||||
"""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = self.max_input_tokens
|
||||
max_length = self.max_truncation_tokens
|
||||
|
||||
if max_length is None or max_length >= length:
|
||||
return None
|
||||
@@ -494,15 +524,12 @@ class TokenizeParams:
|
||||
# max_input_tokens + 1 (see get_encode_kwargs), so the
|
||||
# actual prompt length could be larger.
|
||||
qualifier = "at least " if token_count == max_input_tokens + 1 else ""
|
||||
total = token_count + self.max_output_tokens
|
||||
raise VLLMValidationError(
|
||||
f"This model's maximum context length is "
|
||||
f"{self.max_total_tokens} tokens. However, you requested "
|
||||
f"{self.max_output_tokens} output tokens and your prompt "
|
||||
f"contains {qualifier}{token_count} input tokens, "
|
||||
f"for a total of {qualifier}{total} tokens. "
|
||||
f"Please reduce the length of the input prompt or the "
|
||||
f"number of requested output tokens.",
|
||||
f"{self.max_total_tokens} tokens. However, your prompt "
|
||||
f"contains {qualifier}{token_count} input tokens, which "
|
||||
f"exceeds this limit even without any output tokens. "
|
||||
f"Please reduce the length of the input prompt.",
|
||||
parameter="input_tokens",
|
||||
value=token_count,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user