[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)
Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com> Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com> Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
YuvrajSinghBhadoria2
Cyrus Leung
Claude Opus 4.8
parent
89ac8d971b
commit
1a3a7a8f01
@@ -183,15 +183,15 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
if pooler_config.enable_chunked_processing:
|
||||
max_total_tokens = pooler_config.max_embed_len
|
||||
else:
|
||||
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
|
||||
max_output_tokens = default_max_total_tokens - max_embed_len
|
||||
max_total_tokens = (
|
||||
pooler_config.max_embed_len or default_max_total_tokens
|
||||
)
|
||||
|
||||
tok_params = self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user