[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)

Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com>
Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com>
Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuvraj Singh Bhadoria
2026-10-10 01:15:22 +08:00
committed by GitHub
co-authored by YuvrajSinghBhadoria2 Cyrus Leung Claude Opus 4.8
parent 89ac8d971b
commit 1a3a7a8f01
7 changed files with 252 additions and 32 deletions
@@ -1684,7 +1684,14 @@ def test_serving_chat_starts_without_tokenizer():
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
directly, we must still apply input length + max_tokens validation.
directly, we must still apply input length validation and clamp the
generation budget to the context the prompt leaves over.
`max_tokens` is an upper bound on generation rather than a reservation off
the context window, so a 95-token prompt against `max_model_len=100` is
accepted even though the client asked for 10 output tokens. Genuine
overflow is covered by
test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected.
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
@@ -1696,9 +1703,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
tokenizer=mock_tokenizer,
)
# Force the Mistral chat template renderer to return token IDs.
# Choose a prompt length that is < max_model_len, but large enough that
# adding max_tokens should exceed the model context window.
# Force the Mistral chat template renderer to return token IDs. The prompt
# fits the context on its own; only the naive prompt+output sum overflows.
mock_renderer.render_messages_async = AsyncMock(
return_value=(
[],
@@ -1715,8 +1721,12 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
max_tokens=10,
)
with pytest.raises(VLLMValidationError):
await serving_chat.create_chat_completion(req)
# Must not raise: the prompt alone fits within max_model_len.
await serving_chat.create_chat_completion(req)
# get_max_tokens() clamps the sampling budget to the 5 tokens left over.
sampling_params = mock_engine.generate.call_args.args[1]
assert 1 <= sampling_params.max_tokens <= 5
@pytest.mark.asyncio
@@ -515,7 +515,11 @@ async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
result = await renderer.render_responses(request)
assert not isinstance(result, ErrorResponse)
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
# `max_tokens` is an output upper bound rather than a reservation off the
# context window, so the prompt budget is the full max_model_len minus the
# one token held back for output: 6 tokens truncate to 4, not to
# max_model_len - max_output_tokens.
assert result.engine_input["prompt_token_ids"] == [3, 4, 5, 6]
@pytest.mark.asyncio
@@ -4,12 +4,15 @@
underlying packing helpers, plus Cohere-specific serving helpers."""
import struct
from unittest.mock import Mock
import numpy as np
import pybase64 as base64
import pytest
from vllm.config import ModelConfig, PoolerConfig
from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingCompletionRequest,
build_typed_embeddings,
)
@@ -162,3 +165,59 @@ class TestBuildTypedEmbeddingsMultiple:
def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
assert result.float is not None
def _embedding_model_config(
*,
max_model_len: int = 128,
max_embed_len: int | None = None,
enable_chunked_processing: bool = False,
) -> Mock:
model_config = Mock(spec=ModelConfig)
model_config.max_model_len = max_model_len
model_config.encoder_config = None
model_config.pooler_config = PoolerConfig(
seq_pooling_type="CLS",
max_embed_len=max_embed_len,
enable_chunked_processing=enable_chunked_processing,
)
return model_config
class TestMaxEmbedLenCapsInput:
"""``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
expressed directly as ``max_total_tokens``."""
def test_non_chunked_max_embed_len_caps_input(self):
cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 64
assert tok_params.max_output_tokens == 0
def test_chunked_max_embed_len_caps_input(self):
cfg = _embedding_model_config(
max_model_len=128, max_embed_len=64, enable_chunked_processing=True
)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 64
assert tok_params.max_output_tokens == 0
def test_no_max_embed_len_falls_back_to_model_len(self):
cfg = _embedding_model_config(max_model_len=128)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 128
assert tok_params.max_output_tokens == 0