[Bugfix] Account for truncate_prompt_tokens when computing max_tokens (#41800)
Signed-off-by: Viktor Pus <viktorpus@tenstorrent.com>
This commit is contained in:
@@ -807,6 +807,57 @@ async def test_serving_chat_should_set_correct_max_tokens():
|
|||||||
assert mock_engine.generate.call_args.args[1].max_tokens == 5
|
assert mock_engine.generate.call_args.args[1].max_tokens == 5
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_serving_chat_truncate_prompt_tokens_max_token_accounting():
|
||||||
|
"""When truncate_prompt_tokens is set, max_tokens must be calculated using
|
||||||
|
the truncated prompt length, not the original prompt length.
|
||||||
|
|
||||||
|
Regression: without the fix, get_max_tokens received the untruncated prompt
|
||||||
|
length, causing the output budget to be underestimated.
|
||||||
|
"""
|
||||||
|
mock_engine = MagicMock(spec=AsyncLLM)
|
||||||
|
mock_engine.errored = False
|
||||||
|
mock_engine.model_config = MockModelConfig()
|
||||||
|
mock_engine.input_processor = MagicMock()
|
||||||
|
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||||
|
|
||||||
|
serving_chat = _build_serving_chat(mock_engine)
|
||||||
|
|
||||||
|
# "what is 1+1?" tokenizes to 7 tokens with the test chat template
|
||||||
|
# (max_model_len=100 -> max_tokens = 93 without truncation, confirmed by
|
||||||
|
# test_serving_chat_should_set_correct_max_tokens above).
|
||||||
|
messages = [{"role": "user", "content": "what is 1+1?"}]
|
||||||
|
|
||||||
|
# Baseline: no truncation -> max_tokens = 100 - 7 = 93.
|
||||||
|
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||||
|
with suppress(Exception):
|
||||||
|
await serving_chat.create_chat_completion(req)
|
||||||
|
assert mock_engine.generate.call_args.args[1].max_tokens == 93
|
||||||
|
|
||||||
|
# With truncate_prompt_tokens=5 (less than 7): the effective prompt length
|
||||||
|
# is 5, so max_tokens should be 100 - 5 = 95, not 93.
|
||||||
|
req = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=messages,
|
||||||
|
truncate_prompt_tokens=5,
|
||||||
|
)
|
||||||
|
with suppress(Exception):
|
||||||
|
await serving_chat.create_chat_completion(req)
|
||||||
|
assert mock_engine.generate.call_args.args[1].max_tokens == 95
|
||||||
|
|
||||||
|
# With truncate_prompt_tokens=-1 (meaning use full max_model_len as the
|
||||||
|
# truncation limit, i.e., no practical truncation vs the window): effective
|
||||||
|
# length = min(7, 100) = 7 -> max_tokens = 93 again.
|
||||||
|
req = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=messages,
|
||||||
|
truncate_prompt_tokens=-1,
|
||||||
|
)
|
||||||
|
with suppress(Exception):
|
||||||
|
await serving_chat.create_chat_completion(req)
|
||||||
|
assert mock_engine.generate.call_args.args[1].max_tokens == 93
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||||
"""Regression test: when the Mistral tokenizer path returns token IDs
|
"""Regression test: when the Mistral tokenizer path returns token IDs
|
||||||
|
|||||||
@@ -289,6 +289,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
self._extract_prompt_len(engine_input),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
|
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
sampling_params: SamplingParams | BeamSearchParams
|
sampling_params: SamplingParams | BeamSearchParams
|
||||||
|
|||||||
@@ -151,6 +151,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
self._extract_prompt_len(engine_input),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
|
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
sampling_params: SamplingParams | BeamSearchParams
|
sampling_params: SamplingParams | BeamSearchParams
|
||||||
|
|||||||
@@ -416,6 +416,9 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
self._extract_prompt_len(engine_input),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
|
truncate_prompt_tokens=(
|
||||||
|
-1 if request.truncation != "disabled" else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
sampling_params = request.to_sampling_params(
|
sampling_params = request.to_sampling_params(
|
||||||
@@ -700,6 +703,9 @@ class OpenAIServingResponses(OpenAIServing):
|
|||||||
self._extract_prompt_len(engine_input),
|
self._extract_prompt_len(engine_input),
|
||||||
self.default_sampling_params, # type: ignore
|
self.default_sampling_params, # type: ignore
|
||||||
self.override_max_tokens, # type: ignore
|
self.override_max_tokens, # type: ignore
|
||||||
|
truncate_prompt_tokens=(
|
||||||
|
-1 if context.request.truncation != "disabled" else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# OPTIMIZATION
|
# OPTIMIZATION
|
||||||
|
|||||||
@@ -164,6 +164,7 @@ class OpenAIServingRender:
|
|||||||
input_length,
|
input_length,
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
|
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||||
)
|
)
|
||||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||||
|
|
||||||
@@ -298,6 +299,7 @@ class OpenAIServingRender:
|
|||||||
input_length,
|
input_length,
|
||||||
self.default_sampling_params,
|
self.default_sampling_params,
|
||||||
self.override_max_tokens,
|
self.override_max_tokens,
|
||||||
|
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||||
)
|
)
|
||||||
params = request.to_sampling_params(
|
params = request.to_sampling_params(
|
||||||
max_tokens, self.default_sampling_params
|
max_tokens, self.default_sampling_params
|
||||||
|
|||||||
@@ -177,7 +177,14 @@ def get_max_tokens(
|
|||||||
input_length: int,
|
input_length: int,
|
||||||
default_sampling_params: dict,
|
default_sampling_params: dict,
|
||||||
override_max_tokens: int | None = None,
|
override_max_tokens: int | None = None,
|
||||||
|
truncate_prompt_tokens: int | None = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
|
if truncate_prompt_tokens is not None:
|
||||||
|
limit = truncate_prompt_tokens
|
||||||
|
input_length = min(
|
||||||
|
input_length,
|
||||||
|
max_model_len if limit == -1 else limit,
|
||||||
|
)
|
||||||
if max_model_len < input_length:
|
if max_model_len < input_length:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Input length ({input_length}) exceeds model's maximum "
|
f"Input length ({input_length}) exceeds model's maximum "
|
||||||
|
|||||||
Reference in New Issue
Block a user