[Bugfix] Account for truncate_prompt_tokens when computing max_tokens (#41800)

Signed-off-by: Viktor Pus <viktorpus@tenstorrent.com>
This commit is contained in:
Viktor Pus
2026-05-06 16:10:17 +00:00
committed by GitHub
parent ee38750a75
commit d5b31c954d
6 changed files with 68 additions and 0 deletions
+7
View File
@@ -177,7 +177,14 @@ def get_max_tokens(
input_length: int,
default_sampling_params: dict,
override_max_tokens: int | None = None,
truncate_prompt_tokens: int | None = None,
) -> int:
if truncate_prompt_tokens is not None:
limit = truncate_prompt_tokens
input_length = min(
input_length,
max_model_len if limit == -1 else limit,
)
if max_model_len < input_length:
raise ValueError(
f"Input length ({input_length}) exceeds model's maximum "