[Model] Pooling models default to using chunked prefill & prefix caching if supported. (#20930)

Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
wang.yuqi
2025-08-11 09:41:37 -07:00
committed by GitHub
parent 16fb668b61
commit 84cf78acee
31 changed files with 452 additions and 261 deletions
+11 -27
View File
@@ -44,15 +44,14 @@ class ResolvedPoolingConfig:
task: PoolingTask
@classmethod
def from_config_with_defaults(
def from_config(
cls,
task: PoolingTask,
pooler_config: PoolerConfig,
pooling_type: PoolingType,
) -> "ResolvedPoolingConfig":
assert pooler_config.pooling_type is not None
return cls(task=task,
pooling_type=PoolingType[pooler_config.pooling_type]
if pooler_config.pooling_type is not None else pooling_type)
pooling_type=PoolingType[pooler_config.pooling_type])
@dataclass(frozen=True)
@@ -68,32 +67,20 @@ class Pooler(nn.Module, ABC):
"""The interface required for all poolers used in pooling models in vLLM."""
@staticmethod
def for_encode(
pooler_config: PoolerConfig,
*,
default_pooling_type: PoolingType = PoolingType.ALL,
):
resolved_config = ResolvedPoolingConfig.from_config_with_defaults(
task="encode",
pooler_config=pooler_config,
pooling_type=default_pooling_type,
)
if resolved_config.pooling_type == PoolingType.STEP:
def for_encode(pooler_config: PoolerConfig):
if pooler_config.pooling_type == "STEP":
return StepPooler()
resolved_config = ResolvedPoolingConfig(task="encode",
pooling_type=PoolingType.ALL)
return SimplePooler.from_config(resolved_config)
@staticmethod
def for_embed(
pooler_config: PoolerConfig,
*,
default_pooling_type: PoolingType = PoolingType.LAST,
):
resolved_config = ResolvedPoolingConfig.from_config_with_defaults(
def for_embed(pooler_config: PoolerConfig):
resolved_config = ResolvedPoolingConfig.from_config(
task="embed",
pooler_config=pooler_config,
pooling_type=default_pooling_type,
)
return SimplePooler.from_config(resolved_config)
@@ -102,13 +89,10 @@ class Pooler(nn.Module, ABC):
def for_classify(
pooler_config: PoolerConfig,
classifier: Optional[ClassifierFn],
*,
default_pooling_type: PoolingType = PoolingType.LAST,
):
resolved_config = ResolvedPoolingConfig.from_config_with_defaults(
resolved_config = ResolvedPoolingConfig.from_config(
task="classify",
pooler_config=pooler_config,
pooling_type=default_pooling_type,
)
pooling = PoolingMethod.from_pooling_type(resolved_config.pooling_type)