[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)

Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com>
Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com>
Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuvraj Singh Bhadoria
2026-10-10 01:15:22 +08:00
committed by GitHub
co-authored by YuvrajSinghBhadoria2 Cyrus Leung Claude Opus 4.8
parent 89ac8d971b
commit 1a3a7a8f01
7 changed files with 252 additions and 32 deletions
+3 -3
View File
@@ -183,15 +183,15 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
if pooler_config.enable_chunked_processing:
max_total_tokens = pooler_config.max_embed_len
else:
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
max_output_tokens = default_max_total_tokens - max_embed_len
max_total_tokens = (
pooler_config.max_embed_len or default_max_total_tokens
)
tok_params = self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
max_output_tokens_param="max_model_len - max_embed_len",
)
if (