[Bugfix] Treat generation_config max_tokens as default not ceiling (#34063)
Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
@@ -526,6 +526,7 @@ class MockModelConfig:
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
override_generation_config: dict[str, Any] = field(default_factory=dict)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init: bool = False
|
||||
is_encoder_decoder: bool = False
|
||||
@@ -651,12 +652,10 @@ async def test_serving_chat_should_set_correct_max_tokens():
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 10
|
||||
|
||||
# Setting server's max_tokens in the generation_config.json
|
||||
# lower than context_window - prompt_tokens
|
||||
# Model author's generation_config.json sets max_tokens (auto, no override)
|
||||
# — should act as fallback only, not ceiling
|
||||
mock_model_config = MockModelConfig()
|
||||
mock_model_config.diff_sampling_param = {
|
||||
"max_tokens": 10 # Setting server-side max_tokens limit
|
||||
}
|
||||
mock_model_config.diff_sampling_param = {"max_tokens": 10}
|
||||
|
||||
# Reinitialize the engine with new settings
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
@@ -680,7 +679,50 @@ async def test_serving_chat_should_set_correct_max_tokens():
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 10
|
||||
|
||||
# Test Case 2: Request's max_tokens set higher than server accepts
|
||||
# Test Case 2: Request's max_tokens set higher than generation_config
|
||||
# default so request-provided max_tokens takes precedence
|
||||
req.max_tokens = 15
|
||||
|
||||
with suppress(Exception):
|
||||
await serving_chat.create_chat_completion(req)
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 15
|
||||
|
||||
# Test Case 3: Request's max_tokens set lower than server accepts
|
||||
req.max_tokens = 5
|
||||
|
||||
with suppress(Exception):
|
||||
await serving_chat.create_chat_completion(req)
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 5
|
||||
|
||||
# User explicitly sets max_tokens via --override-generation-config
|
||||
# — should act as a ceiling
|
||||
mock_model_config = MockModelConfig()
|
||||
mock_model_config.diff_sampling_param = {"max_tokens": 10}
|
||||
mock_model_config.override_generation_config = {"max_new_tokens": 10}
|
||||
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = mock_model_config
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.io_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
# Test Case 3.1: No max_tokens — uses override as default
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "what is 1+1?"}],
|
||||
)
|
||||
|
||||
with suppress(Exception):
|
||||
await serving_chat.create_chat_completion(req)
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 10
|
||||
|
||||
# Test Case 3.2: Request max_tokens higher — capped by user ceiling from override
|
||||
req.max_tokens = 15
|
||||
|
||||
with suppress(Exception):
|
||||
@@ -688,7 +730,7 @@ async def test_serving_chat_should_set_correct_max_tokens():
|
||||
|
||||
assert mock_engine.generate.call_args.args[1].max_tokens == 10
|
||||
|
||||
# Test Case 3: Request's max_tokens set lower than server accepts
|
||||
# Test Case 3.3: Request max_tokens lower — respected
|
||||
req.max_tokens = 5
|
||||
|
||||
with suppress(Exception):
|
||||
@@ -699,9 +741,7 @@ async def test_serving_chat_should_set_correct_max_tokens():
|
||||
# Setting server's max_tokens in the generation_config.json
|
||||
# higher than context_window - prompt_tokens
|
||||
mock_model_config = MockModelConfig()
|
||||
mock_model_config.diff_sampling_param = {
|
||||
"max_tokens": 200 # Setting server-side max_tokens limit
|
||||
}
|
||||
mock_model_config.diff_sampling_param = {"max_tokens": 200}
|
||||
|
||||
# Reinitialize the engine with new settings
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.entrypoints.utils import sanitize_message
|
||||
|
||||
from vllm.entrypoints.utils import get_max_tokens, sanitize_message
|
||||
|
||||
|
||||
def test_sanitize_message():
|
||||
@@ -8,3 +9,74 @@ def test_sanitize_message():
|
||||
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
|
||||
== "<_io.BytesIO object>"
|
||||
)
|
||||
|
||||
|
||||
class TestGetMaxTokens:
|
||||
"""Tests for get_max_tokens() to ensure generation_config's max_tokens
|
||||
acts as a default when from model author, and as a ceiling when
|
||||
explicitly set by the user."""
|
||||
|
||||
def test_default_sampling_params_used_when_no_request_max_tokens(self):
|
||||
"""When user doesn't specify max_tokens, generation_config default
|
||||
should apply."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=None,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 2048
|
||||
|
||||
def test_request_max_tokens_not_capped_by_default_sampling_params(self):
|
||||
"""When user specifies max_tokens in request, model author's
|
||||
generation_config max_tokens must NOT cap it (fixes #34005)."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=5000,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 5000
|
||||
|
||||
def test_override_max_tokens_caps_request(self):
|
||||
"""When user explicitly sets max_tokens, it acts as a ceiling."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=5000,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
override_max_tokens=2048,
|
||||
)
|
||||
assert result == 2048
|
||||
|
||||
def test_override_max_tokens_used_as_default(self):
|
||||
"""When no request max_tokens, override still applies as default."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=None,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
override_max_tokens=2048,
|
||||
)
|
||||
assert result == 2048
|
||||
|
||||
def test_max_model_len_still_caps_output(self):
|
||||
"""max_model_len - input_length is always the hard ceiling."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=3000,
|
||||
max_tokens=5000,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 2900 # 3000 - 100
|
||||
|
||||
def test_request_max_tokens_smaller_than_default(self):
|
||||
"""When user explicitly requests fewer tokens than gen_config default,
|
||||
that should be respected."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=512,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 512
|
||||
|
||||
Reference in New Issue
Block a user