[Pooling] Cap max-length padding for chunked embeddings (#56505)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
85c1f58d50
commit
1fd119def5
@@ -171,6 +171,8 @@ class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
|
||||
|
||||
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
padding: Literal["max_length", "do_not_pad"] | None
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
default_max_total_tokens = model_config.max_model_len
|
||||
max_total_tokens: int | None = default_max_total_tokens
|
||||
@@ -184,7 +186,7 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
|
||||
max_output_tokens = default_max_total_tokens - max_embed_len
|
||||
|
||||
return self._build_pooling_tok_params(
|
||||
tok_params = self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
@@ -192,6 +194,20 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
if (
|
||||
self.padding == "max_length"
|
||||
and pooler_config is not None
|
||||
and pooler_config.enable_chunked_processing
|
||||
):
|
||||
max_padding_len = pooler_config.max_embed_len
|
||||
if max_padding_len is None:
|
||||
max_padding_len = default_max_total_tokens
|
||||
return tok_params.with_kwargs(
|
||||
pad_prompt_tokens=min(default_max_total_tokens, max_padding_len)
|
||||
)
|
||||
|
||||
return tok_params
|
||||
|
||||
|
||||
class CompletionRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:completion-params]
|
||||
|
||||
Reference in New Issue
Block a user