diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index bfaa2d4ca0..d22f581f67 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -1684,7 +1684,14 @@ def test_serving_chat_starts_without_tokenizer(): @pytest.mark.asyncio async def test_serving_chat_mistral_token_ids_prompt_is_validated(): """Regression test: when the Mistral tokenizer path returns token IDs - directly, we must still apply input length + max_tokens validation. + directly, we must still apply input length validation and clamp the + generation budget to the context the prompt leaves over. + + `max_tokens` is an upper bound on generation rather than a reservation off + the context window, so a 95-token prompt against `max_model_len=100` is + accepted even though the client asked for 10 output tokens. Genuine + overflow is covered by + test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected. """ mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False @@ -1696,9 +1703,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated(): MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()), tokenizer=mock_tokenizer, ) - # Force the Mistral chat template renderer to return token IDs. - # Choose a prompt length that is < max_model_len, but large enough that - # adding max_tokens should exceed the model context window. + # Force the Mistral chat template renderer to return token IDs. The prompt + # fits the context on its own; only the naive prompt+output sum overflows. mock_renderer.render_messages_async = AsyncMock( return_value=( [], @@ -1715,8 +1721,12 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated(): max_tokens=10, ) - with pytest.raises(VLLMValidationError): - await serving_chat.create_chat_completion(req) + # Must not raise: the prompt alone fits within max_model_len. + await serving_chat.create_chat_completion(req) + + # get_max_tokens() clamps the sampling budget to the 5 tokens left over. + sampling_params = mock_engine.generate.call_args.args[1] + assert 1 <= sampling_params.max_tokens <= 5 @pytest.mark.asyncio diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index acab4cb6e2..1622d0e8f8 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -515,7 +515,11 @@ async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt( result = await renderer.render_responses(request) assert not isinstance(result, ErrorResponse) - assert result.engine_input["prompt_token_ids"] == [4, 5, 6] + # `max_tokens` is an output upper bound rather than a reservation off the + # context window, so the prompt budget is the full max_model_len minus the + # one token held back for output: 6 tokens truncate to 4, not to + # max_model_len - max_output_tokens. + assert result.engine_input["prompt_token_ids"] == [3, 4, 5, 6] @pytest.mark.asyncio diff --git a/tests/entrypoints/pooling/embed/test_protocol.py b/tests/entrypoints/pooling/embed/test_protocol.py index 91c853f910..ba797b4ff1 100644 --- a/tests/entrypoints/pooling/embed/test_protocol.py +++ b/tests/entrypoints/pooling/embed/test_protocol.py @@ -4,12 +4,15 @@ underlying packing helpers, plus Cohere-specific serving helpers.""" import struct +from unittest.mock import Mock import numpy as np import pybase64 as base64 import pytest +from vllm.config import ModelConfig, PoolerConfig from vllm.entrypoints.pooling.embed.protocol import ( + EmbeddingCompletionRequest, build_typed_embeddings, ) @@ -162,3 +165,59 @@ class TestBuildTypedEmbeddingsMultiple: def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]): result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"]) assert result.float is not None + + +def _embedding_model_config( + *, + max_model_len: int = 128, + max_embed_len: int | None = None, + enable_chunked_processing: bool = False, +) -> Mock: + model_config = Mock(spec=ModelConfig) + model_config.max_model_len = max_model_len + model_config.encoder_config = None + model_config.pooler_config = PoolerConfig( + seq_pooling_type="CLS", + max_embed_len=max_embed_len, + enable_chunked_processing=enable_chunked_processing, + ) + return model_config + + +class TestMaxEmbedLenCapsInput: + """``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by + setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on + ``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now + expressed directly as ``max_total_tokens``.""" + + def test_non_chunked_max_embed_len_caps_input(self): + cfg = _embedding_model_config(max_model_len=128, max_embed_len=64) + + tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params( + cfg + ) + + assert tok_params.max_input_tokens == 64 + assert tok_params.max_output_tokens == 0 + + def test_chunked_max_embed_len_caps_input(self): + cfg = _embedding_model_config( + max_model_len=128, max_embed_len=64, enable_chunked_processing=True + ) + + tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params( + cfg + ) + + assert tok_params.max_input_tokens == 64 + assert tok_params.max_output_tokens == 0 + + def test_no_max_embed_len_falls_back_to_model_len(self): + cfg = _embedding_model_config(max_model_len=128) + + tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params( + cfg + ) + + assert tok_params.max_input_tokens == 128 + assert tok_params.max_output_tokens == 0 diff --git a/tests/models/language/pooling/test_truncation_control.py b/tests/models/language/pooling/test_truncation_control.py index c448519555..003213410c 100644 --- a/tests/models/language/pooling/test_truncation_control.py +++ b/tests/models/language/pooling/test_truncation_control.py @@ -55,6 +55,8 @@ def test_max_truncation_size(vllm_model): prompt_tokens = vllm_output[0].prompt_token_ids + # Pooling requests emit an embedding rather than generated tokens, so -1 + # truncates to the full context length. assert len(prompt_tokens) == max_model_len diff --git a/tests/renderers/test_completions.py b/tests/renderers/test_completions.py index 0dce7ef684..4fac063807 100644 --- a/tests/renderers/test_completions.py +++ b/tests/renderers/test_completions.py @@ -247,6 +247,8 @@ class TestRenderPrompt: ) assert len(results) == 1 + # max_output_tokens defaults to 0, so there is no output budget to + # reserve and -1 truncates to the full max_total_tokens. assert len(results[0]["prompt_token_ids"]) == 100 # max_total_tokens def test_truncation_left(self): @@ -505,6 +507,122 @@ class TestRenderPrompt: assert renderer.tokenizer._captured_text_len <= 100 +class TestMaxTokensNotReservation: + """Regression tests for GitHub issue #42474. + + ``max_tokens`` must be treated as an upper bound on generation length, not + as a hard reservation that is subtracted from the available input space. + """ + + def test_large_max_tokens_does_not_eat_prompt_budget(self): + """Prompt fits in context even when max_tokens is large.""" + renderer = _build_renderer(MockModelConfig()) + + # 50-token prompt with a 100-token context window and max_tokens=60. + # Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error. + # After the fix: max_input_tokens = 100 → 50 ≤ 100 → OK. + prompts = renderer.render_prompts( + _preprocess_prompt(renderer.model_config, "x" * 50) + ) + results = renderer.tokenize_prompts( + prompts, + TokenizeParams(max_total_tokens=100, max_output_tokens=60), + ) + assert len(results) == 1 + assert len(results[0]["prompt_token_ids"]) == 50 + + def test_max_tokens_zero_still_rejects_oversized_prompt(self): + """Genuine prompt overflow is still caught regardless of max_tokens.""" + renderer = _build_renderer(MockModelConfig()) + + # 150-token prompt with 100-token context → should still fail. + prompts = renderer.render_prompts( + _preprocess_prompt(renderer.model_config, "x" * 150) + ) + with pytest.raises(VLLMValidationError, match="maximum context length"): + renderer.tokenize_prompts( + prompts, + TokenizeParams(max_total_tokens=100, max_output_tokens=0), + ) + + def test_negative_truncation_uses_max_total(self): + """truncate_prompt_tokens=-1 reserves one output token.""" + renderer = _build_renderer(MockModelConfig()) + + prompts = renderer.render_prompts( + _preprocess_prompt(renderer.model_config, "x" * 200) + ) + results = renderer.tokenize_prompts( + prompts, + TokenizeParams( + max_total_tokens=100, + max_output_tokens=40, + truncate_prompt_tokens=-1, + ), + ) + # -1 maps to max_total_tokens (100) rather than the old + # max_total - max_output (60), minus the one token held back so + # truncation cannot leave a zero-token output budget. + assert len(results[0]["prompt_token_ids"]) == 99 + + def test_negative_pad_uses_max_total(self): + """pad_prompt_tokens=-1 pads to max_total_tokens, not max_input.""" + renderer = _build_renderer(MockModelConfig()) + pad_id = renderer.tokenizer.pad_token_id + + prompts = renderer.render_prompts( + _preprocess_prompt(renderer.model_config, "x" * 30) + ) + results = renderer.tokenize_prompts( + prompts, + TokenizeParams( + max_total_tokens=100, + max_output_tokens=40, + pad_prompt_tokens=-1, + ), + ) + # -1 should map to max_total_tokens (100), not max_total - max_output (60). + assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70 + + def test_max_tokens_exceeds_context_is_still_clamped_by_api(self): + """Renderer allows the prompt; API layer clamps max_tokens later.""" + renderer = _build_renderer(MockModelConfig()) + + # Simulates the reported scenario: max_tokens > context - prompt_length. + # Renderer should accept the 40-token prompt; get_max_tokens() clamps + # sampling max_tokens to 60 at the entrypoint layer. + prompts = renderer.render_prompts( + _preprocess_prompt(renderer.model_config, "x" * 40) + ) + results = renderer.tokenize_prompts( + prompts, + TokenizeParams(max_total_tokens=100, max_output_tokens=80), + ) + assert len(results) == 1 + assert len(results[0]["prompt_token_ids"]) == 40 + + def test_with_kwargs_preserves_output_budget(self): + """with_kwargs without an explicit max_length keeps the output budget. + + Under the old semantics `with_kwargs()` recomputed + `max_output_tokens = max_total - max_input`, which (after this fix) + would have silently reset a real output budget to zero. + """ + tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60) + chained = tok_params.with_kwargs(truncation_side="left") + + assert chained.max_output_tokens == 60 + assert chained.max_input_tokens == 100 + + def test_with_kwargs_explicit_max_length_sets_budget(self): + """An explicit max_length still re-derives the output budget.""" + tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60) + chained = tok_params.with_kwargs(max_length=80) + + assert chained.max_output_tokens == 20 + assert chained.max_input_tokens == 100 + + class TestRenderEmbedPrompt: def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes: """Helper to create base64-encoded tensor bytes.""" diff --git a/vllm/entrypoints/pooling/base/protocol.py b/vllm/entrypoints/pooling/base/protocol.py index c1366734f7..0c646b969a 100644 --- a/vllm/entrypoints/pooling/base/protocol.py +++ b/vllm/entrypoints/pooling/base/protocol.py @@ -183,15 +183,15 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin): if pooler_config.enable_chunked_processing: max_total_tokens = pooler_config.max_embed_len else: - max_embed_len = pooler_config.max_embed_len or default_max_total_tokens - max_output_tokens = default_max_total_tokens - max_embed_len + max_total_tokens = ( + pooler_config.max_embed_len or default_max_total_tokens + ) tok_params = self._build_pooling_tok_params( model_config, add_special_tokens=self.add_special_tokens, max_total_tokens=max_total_tokens, max_output_tokens=max_output_tokens, - max_output_tokens_param="max_model_len - max_embed_len", ) if ( diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 9e62dfbb2d..456a4d0ac5 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -207,11 +207,38 @@ class TokenizeParams: @property def max_input_tokens(self) -> int | None: - """Maximum allowed number of input tokens.""" + """Maximum allowed number of input tokens. + + ``max_output_tokens`` is *not* subtracted here: the value represents the + actual model context length, not a reservation of output space. + Downstream clamping of the sampling ``max_tokens`` is handled by + ``get_max_tokens()`` in the entrypoint layer. + """ if self.max_total_tokens is None: return None - return self.max_total_tokens - self.max_output_tokens + return self.max_total_tokens + + @property + def max_truncation_tokens(self) -> int | None: + """Maximum prompt length that truncation may reduce a prompt to. + + For generative requests this is `max_input_tokens` less one, holding a + token of context back for output. Truncating to the full + `max_total_tokens` would leave `get_max_tokens()` clamping the sampling + budget to 0, turning an over-long prompt into a silent zero-token + generation. + + Pooling requests set `max_output_tokens = 0` and emit an embedding + rather than generated tokens, so they keep the full input budget. + """ + max_input_tokens = self.max_input_tokens + if max_input_tokens is None: + return None + if self.max_output_tokens == 0: + return max_input_tokens + + return max(0, max_input_tokens - 1) def __post_init__(self) -> None: max_total_tokens = self.max_total_tokens @@ -247,14 +274,15 @@ class TokenizeParams: ): raise VLLMValidationError( f"{self.truncate_prompt_tokens_param}={truncate_prompt_tokens} " - f"cannot be greater than {self.max_total_tokens_param} - " - f"{self.max_output_tokens_param} = {max_input_tokens}. " + f"cannot be greater than {self.max_total_tokens_param} " + f"= {max_input_tokens}. " f"Please request a smaller truncation size.", parameter=self.truncate_prompt_tokens_param, value=truncate_prompt_tokens, ) def with_kwargs(self, **tokenization_kwargs: Any): + has_max_length = "max_length" in tokenization_kwargs max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens) pad_prompt_tokens = tokenization_kwargs.pop( "pad_prompt_tokens", self.pad_prompt_tokens @@ -301,13 +329,17 @@ class TokenizeParams: max_total_tokens = self.max_total_tokens + if has_max_length and max_total_tokens is not None and max_length is not None: + max_output_tokens = max_total_tokens - max_length + else: + # No explicit ``max_length`` override: preserve the original output + # budget instead of recomputing it from ``max_input_tokens`` (which + # is now the full context length). + max_output_tokens = self.max_output_tokens + return TokenizeParams( max_total_tokens=max_total_tokens, - max_output_tokens=( - 0 - if max_total_tokens is None or max_length is None - else max_total_tokens - max_length - ), + max_output_tokens=max_output_tokens, pad_prompt_tokens=pad_prompt_tokens, truncate_prompt_tokens=truncate_prompt_tokens, truncation_side=truncation_side, @@ -320,7 +352,7 @@ class TokenizeParams: """The arguments to pass to `tokenizer.encode`.""" max_length = self.truncate_prompt_tokens if max_length is not None and max_length < 0: - max_length = self.max_input_tokens + max_length = self.max_truncation_tokens elif max_length is None and self.max_input_tokens is not None: # This prevents tokenization from taking up more resources than necessary # while still failing `self._token_len_check` as expected by users @@ -355,13 +387,11 @@ class TokenizeParams: if len(text) > max_input_chars: raise VLLMValidationError( f"This model's maximum context length is " - f"{self.max_total_tokens} tokens. However, you requested " - f"{self.max_output_tokens} output tokens and your prompt " + f"{self.max_total_tokens} tokens. However, your prompt " f"contains {len(text)} characters (more than " f"{max_input_chars} characters, which is the upper bound " f"for {max_input_tokens} input tokens). " - f"Please reduce the length of the input prompt or the " - f"number of requested output tokens.", + f"Please reduce the length of the input prompt.", parameter="input_text", value=len(text), ) @@ -459,7 +489,7 @@ class TokenizeParams: """ max_length = self.truncate_prompt_tokens if max_length is not None and max_length < 0: - max_length = self.max_input_tokens + max_length = self.max_truncation_tokens if max_length is None or max_length >= length: return None @@ -494,15 +524,12 @@ class TokenizeParams: # max_input_tokens + 1 (see get_encode_kwargs), so the # actual prompt length could be larger. qualifier = "at least " if token_count == max_input_tokens + 1 else "" - total = token_count + self.max_output_tokens raise VLLMValidationError( f"This model's maximum context length is " - f"{self.max_total_tokens} tokens. However, you requested " - f"{self.max_output_tokens} output tokens and your prompt " - f"contains {qualifier}{token_count} input tokens, " - f"for a total of {qualifier}{total} tokens. " - f"Please reduce the length of the input prompt or the " - f"number of requested output tokens.", + f"{self.max_total_tokens} tokens. However, your prompt " + f"contains {qualifier}{token_count} input tokens, which " + f"exceeds this limit even without any output tokens. " + f"Please reduce the length of the input prompt.", parameter="input_tokens", value=token_count, )