[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
@@ -198,6 +198,11 @@ class BOSEOSFilter(Pooler):
|
||||
for i, (prompt_len, token_ids) in enumerate(
|
||||
zip(pooling_metadata.prompt_lens, prompt_token_ids)
|
||||
):
|
||||
# Late chunking returns one row per chunk, with special tokens
|
||||
# already included in the means. Skip BOS/EOS row removal for these
|
||||
# requests; ordinary token outputs still use the filtering below.
|
||||
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
|
||||
continue
|
||||
pooled_data = pooled_outputs[i]
|
||||
assert (
|
||||
isinstance(pooled_data, torch.Tensor)
|
||||
|
||||
@@ -17,6 +17,23 @@ from .methods import TokenPoolingMethodOutputItem
|
||||
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
|
||||
|
||||
|
||||
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
|
||||
"""Mean-pool complete contextual states, accumulating in FP32."""
|
||||
chunk_size = min(chunk_size, len(data))
|
||||
num_full, remainder = divmod(len(data), chunk_size)
|
||||
full_end = num_full * chunk_size
|
||||
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
|
||||
if not remainder:
|
||||
return full_chunks.mean(dim=1, dtype=torch.float32)
|
||||
|
||||
return torch.cat(
|
||||
[
|
||||
full_chunks.mean(dim=1, dtype=torch.float32),
|
||||
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TokenPoolerHead(nn.Module, ABC):
|
||||
@abstractmethod
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
@@ -80,6 +97,12 @@ class TokenEmbeddingPoolerHead(TokenPoolerHead):
|
||||
if pooled_data is None:
|
||||
return None
|
||||
|
||||
if (late_chunking := pooling_param.late_chunking_params) is not None:
|
||||
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
|
||||
pooled_data = (
|
||||
means.to(pooled_data.dtype) if self.head_dtype is None else means
|
||||
)
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
# pooled_data shape: [n_tokens, hidden_size]
|
||||
|
||||
Reference in New Issue
Block a user