[Pooling] Cap max-length padding for chunked embeddings (#56505)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-15 20:31:51 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 85c1f58d50
commit 1fd119def5
+17 -1
View File
@@ -171,6 +171,8 @@ class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
padding: Literal["max_length", "do_not_pad"] | None
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
default_max_total_tokens = model_config.max_model_len
max_total_tokens: int | None = default_max_total_tokens
@@ -184,7 +186,7 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
max_output_tokens = default_max_total_tokens - max_embed_len
return self._build_pooling_tok_params(
tok_params = self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=max_total_tokens,
@@ -192,6 +194,20 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
max_output_tokens_param="max_model_len - max_embed_len",
)
if (
self.padding == "max_length"
and pooler_config is not None
and pooler_config.enable_chunked_processing
):
max_padding_len = pooler_config.max_embed_len
if max_padding_len is None:
max_padding_len = default_max_total_tokens
return tok_params.with_kwargs(
pad_prompt_tokens=min(default_max_total_tokens, max_padding_len)
)
return tok_params
class CompletionRequestMixin(OpenAIBaseModel):
# --8<-- [start:completion-params]