[Model] Pooling models default to using chunked prefill & prefix caching if supported. (#20930)

Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
wang.yuqi
2025-08-11 09:41:37 -07:00
committed by GitHub
parent 16fb668b61
commit 84cf78acee
31 changed files with 452 additions and 261 deletions
+14
View File
@@ -641,6 +641,20 @@ def supports_cross_encoding(
return is_pooling_model(model) and _supports_cross_encoding(model)
def default_pooling_type(pooling_type: str) -> object:
"""Set default_pooling_type decorator. """
def func(model: object):
model.default_pooling_type = pooling_type
return model
return func
def get_default_pooling_type(model: Union[type[object], object]) -> str:
return getattr(model, "default_pooling_type", "LAST")
class SupportsQuant:
"""The interface required for all models that support quantization."""