[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
+47
-2
@@ -36,6 +36,28 @@ class LateInteractionParams(
|
||||
query_uses: int | None = None
|
||||
|
||||
|
||||
class LateChunkingParams(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
array_like=True,
|
||||
): # type: ignore[call-arg]
|
||||
"""Fixed-length mean pooling before the token embedding head.
|
||||
|
||||
Requests recompute all prompt tokens, bypassing prefix cache reads.
|
||||
Special tokens count toward chunk boundaries and are included in chunk means,
|
||||
including the BOS token normally filtered from BGE-M3 token embeddings.
|
||||
"""
|
||||
|
||||
chunk_size: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.verify()
|
||||
|
||||
def verify(self) -> None:
|
||||
if type(self.chunk_size) is not int or self.chunk_size <= 0:
|
||||
raise VLLMValidationError("chunk_size must be a positive integer")
|
||||
|
||||
|
||||
class PoolingParams(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
@@ -48,6 +70,8 @@ class PoolingParams(
|
||||
`None` uses the pooler's default, which is `True` in most cases.
|
||||
dimensions: Reduce the dimensions of embeddings
|
||||
if model support matryoshka representation.
|
||||
late_chunking_params: Mean-pool contextual token states into fixed-size
|
||||
chunks for the token embedding task.
|
||||
|
||||
"""
|
||||
|
||||
@@ -72,16 +96,22 @@ class PoolingParams(
|
||||
extra_kwargs: dict[str, Any] | None = None
|
||||
output_kind: RequestOutputKind = RequestOutputKind.FINAL_ONLY
|
||||
|
||||
## for token embedding models
|
||||
# --8<-- [start:token-embed-pooling-params]
|
||||
late_chunking_params: LateChunkingParams | None = None
|
||||
"""Mean-pool contextual tokens into fixed-size chunks for `token_embed`."""
|
||||
# --8<-- [end:token-embed-pooling-params]
|
||||
|
||||
@property
|
||||
def all_parameters(self) -> list[str]:
|
||||
return ["dimensions", "use_activation"]
|
||||
return ["dimensions", "use_activation", "late_chunking_params"]
|
||||
|
||||
@property
|
||||
def valid_parameters(self):
|
||||
return {
|
||||
"embed": ["dimensions", "use_activation"],
|
||||
"classify": ["use_activation"],
|
||||
"token_embed": ["dimensions", "use_activation"],
|
||||
"token_embed": ["dimensions", "use_activation", "late_chunking_params"],
|
||||
"token_classify": ["use_activation"],
|
||||
}
|
||||
|
||||
@@ -90,6 +120,15 @@ class PoolingParams(
|
||||
return deepcopy(self)
|
||||
|
||||
def verify(self, model_config: ModelConfig) -> None:
|
||||
if self.late_chunking_params is not None:
|
||||
self.late_chunking_params.verify()
|
||||
if self.task != "token_embed":
|
||||
raise VLLMValidationError("Late chunking requires token_embed")
|
||||
if self.late_interaction_params is not None:
|
||||
raise VLLMValidationError(
|
||||
"Late chunking cannot be combined with late-interaction scoring"
|
||||
)
|
||||
|
||||
# plugin task uses io_processor.parse_data to verify inputs,
|
||||
# skipping PoolingParams verify
|
||||
if self.task == "plugin":
|
||||
@@ -107,6 +146,11 @@ class PoolingParams(
|
||||
self._merge_default_parameters(model_config)
|
||||
self._set_default_parameters(model_config)
|
||||
self._verify_valid_parameters()
|
||||
if self.late_chunking_params is not None:
|
||||
if self.step_tag_id is not None:
|
||||
raise VLLMValidationError("Late chunking requires all token rows")
|
||||
# KV cache hits omit hidden states needed for complete chunk means.
|
||||
self.skip_reading_prefix_cache = True
|
||||
|
||||
def _merge_default_parameters(self, model_config: ModelConfig) -> None:
|
||||
pooler_config = model_config.pooler_config
|
||||
@@ -228,6 +272,7 @@ class PoolingParams(
|
||||
f"requires_token_ids={self.requires_token_ids}, "
|
||||
f"skip_reading_prefix_cache={self.skip_reading_prefix_cache}, "
|
||||
f"late_interaction_params={self.late_interaction_params}, "
|
||||
f"late_chunking_params={self.late_chunking_params}, "
|
||||
f"extra_kwargs={self.extra_kwargs})"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user