[Feature][Pooling] Add offline fixed-length late chunking (#60164)

Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
Linze Shi
2026-10-13 11:16:21 +00:00
committed by GitHub
parent 426f3bd36b
commit e6e84f350c
15 changed files with 1147 additions and 7 deletions
@@ -12,6 +12,7 @@ from openai.types.chat import (
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
from vllm import PoolingParams
from vllm.exceptions import VLLMValidationError
from vllm.inputs import tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -26,18 +27,26 @@ from vllm.utils.collection_utils import chunk_list
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..late_chunking import (
attach_late_chunking_metadata,
build_late_chunking_metadata,
prepare_late_chunking_input,
)
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
ChunkedEmbeddingMetadata,
EncodeChatRenderParams,
LateChunkingSlot,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
)
from .protocol import (
CohereEmbedContent,
@@ -657,10 +666,75 @@ class EmbedIOProcessor(PoolingIOProcessor):
class TokenEmbedIOProcessor(PoolingIOProcessor):
name = "token_embed"
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
factory, num_requests = super().get_request_factory_offline(ctx)
assert isinstance(ctx, OfflineEncodeInputsContext)
if ctx.pooling_params is None or not any(
p.late_chunking_params is not None
for p in self._params_to_seq(ctx.pooling_params, num_requests)
):
return factory, num_requests
if (ctx.tokenization_kwargs or {}).get("padding") not in (
None,
False,
"do_not_pad",
):
raise VLLMValidationError("Late chunking does not support input padding")
def request_factory() -> RequestGenerator:
for request in factory():
slot = None
if request["params"].late_chunking_params is not None:
slot = LateChunkingSlot()
request["late_chunking"] = slot
ctx.late_chunking.append(slot)
yield request
return request_factory, num_requests
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
late_chunking = render_params["params"].late_chunking_params
if late_chunking is None:
return super().render(render_params)
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
render_params = render_params.copy()
render_params["tok_params"] = tok_params
result = super().render(render_params)
prompt = result["prompts"]
if prompt["type"] != "token":
raise VLLMValidationError("Late chunking requires tokenized text")
render_params["late_chunking"].metadata = build_late_chunking_metadata(
text,
len(prompt["prompt_token_ids"]),
prompt.pop("prompt_token_offsets", None),
late_chunking.chunk_size,
)
return result
def post_process_offline(self, ctx: OfflineOutputsContext):
if ctx.late_chunking:
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
if slot is not None:
assert slot.metadata is not None
attach_late_chunking_metadata(output, slot.metadata)
return ctx.outputs
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
if render_params["params"].late_chunking_params is not None:
raise VLLMValidationError(
"JinaForRanking returns document/query vectors and does not support "
"late chunking"
)
return super().render(render_params)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import replace
from typing import cast
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
from vllm.renderers import TokenizeParams
from .typing import AnyRenderParam, EncodeCMPLRenderParams
def prepare_late_chunking_input(
config: VllmConfig, render_params: AnyRenderParam
) -> tuple[str, TokenizeParams]:
"""Validate full-document text input before requesting tokenizer offsets."""
params = render_params["params"]
params.verify(config.model_config)
if "prompts" not in render_params:
raise VLLMValidationError("Late chunking requires plain-text input")
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
if (
not isinstance(prompt, dict)
or set(prompt) - {"prompt", "cache_salt"}
or not isinstance(text := prompt.get("prompt"), str)
or not text
):
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
tok_params = render_params["tok_params"]
if (
tok_params.pad_prompt_tokens is not None
or tok_params.truncate_prompt_tokens is not None
or tok_params.do_lower_case
):
raise VLLMValidationError(
"Late chunking does not support input padding, truncation or "
"renderer text normalization"
)
return text, replace(tok_params, return_token_offsets=True)
def build_late_chunking_metadata(
text: str,
num_tokens: int,
offsets: Sequence[tuple[int, int]] | None,
chunk_size: int,
) -> LateChunkingMetadata:
"""Map chunks to original text, keeping special-token-only chunks."""
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
raise VLLMValidationError(
"Late chunking requires a fast tokenizer with offsets aligned to "
"all input tokens"
)
previous_start = previous_end = 0
for start, end in offsets:
if not 0 <= start <= end <= len(text):
raise VLLMValidationError("Token offsets are not aligned to the input text")
if start < end:
if start < previous_start or end < previous_end:
raise VLLMValidationError("Token offsets are not in source-text order")
previous_start, previous_end = start, end
chunks = []
for start in range(0, num_tokens, chunk_size):
end = min(start + chunk_size, num_tokens)
source = [(a, b) for a, b in offsets[start:end] if a < b]
char_range = (
(min(a for a, _ in source), max(b for _, b in source)) if source else None
)
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
return LateChunkingMetadata(
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
)
def attach_late_chunking_metadata(
output: PoolingRequestOutput, metadata: LateChunkingMetadata
) -> None:
"""Attach ranges only to successful, complete chunk-vector results."""
if not output.finished or output.error is not None:
return
data = output.outputs.data
if (
data.ndim != 2
or data.shape[0] != len(metadata.chunks)
or len(output.prompt_token_ids) != metadata.input_tokens
):
raise ValueError("Late-chunking output does not match its token/source ranges")
output.late_chunking = metadata
+1 -1
View File
@@ -129,7 +129,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs)
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
)
return outputs
+14 -1
View File
@@ -3,7 +3,7 @@
import time
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
@@ -11,6 +11,7 @@ from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import LateChunkingMetadata
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.renderers.inputs import DictPrompt
@@ -80,6 +81,13 @@ class ChunkedEmbeddingMetadata:
chunk_index: int
@dataclass
class LateChunkingSlot:
"""Per-document source ranges shared by frontend rendering and output contexts."""
metadata: LateChunkingMetadata | None = None
@dataclass(kw_only=True)
class PoolingServeContext(Generic[PoolingRequestT]):
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -124,6 +132,9 @@ class OfflineInputsContext:
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: int | Sequence[int] | None
late_chunking: list[LateChunkingSlot | None] = field(
default_factory=list, kw_only=True
)
@dataclass
@@ -158,6 +169,7 @@ class OfflineOutputsContext:
## for bi-encoder & late-interaction
n_queries: int | None = None
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
class RenderParams(TypedDict):
@@ -168,6 +180,7 @@ class RenderParams(TypedDict):
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
late_chunking: NotRequired[LateChunkingSlot]
class EncodeCMPLRenderParams(RenderParams):
@@ -198,6 +198,11 @@ class BOSEOSFilter(Pooler):
for i, (prompt_len, token_ids) in enumerate(
zip(pooling_metadata.prompt_lens, prompt_token_ids)
):
# Late chunking returns one row per chunk, with special tokens
# already included in the means. Skip BOS/EOS row removal for these
# requests; ordinary token outputs still use the filtering below.
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
continue
pooled_data = pooled_outputs[i]
assert (
isinstance(pooled_data, torch.Tensor)
@@ -17,6 +17,23 @@ from .methods import TokenPoolingMethodOutputItem
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
"""Mean-pool complete contextual states, accumulating in FP32."""
chunk_size = min(chunk_size, len(data))
num_full, remainder = divmod(len(data), chunk_size)
full_end = num_full * chunk_size
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
if not remainder:
return full_chunks.mean(dim=1, dtype=torch.float32)
return torch.cat(
[
full_chunks.mean(dim=1, dtype=torch.float32),
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
]
)
class TokenPoolerHead(nn.Module, ABC):
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
@@ -80,6 +97,12 @@ class TokenEmbeddingPoolerHead(TokenPoolerHead):
if pooled_data is None:
return None
if (late_chunking := pooling_param.late_chunking_params) is not None:
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
pooled_data = (
means.to(pooled_data.dtype) if self.head_dtype is None else means
)
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# pooled_data shape: [n_tokens, hidden_size]
+17
View File
@@ -277,6 +277,20 @@ STREAM_FINISHED = RequestOutput(
_O = TypeVar("_O", default=PoolingOutput)
@dataclass(frozen=True)
class LateChunk:
token_range: tuple[int, int]
char_range: tuple[int, int] | None
@dataclass(frozen=True)
class LateChunkingMetadata:
chunk_size: int
input_tokens: int
chunks: list[LateChunk]
strategy: str = "fixed_tokens"
class PoolingRequestOutput(Generic[_O]):
"""The output data of a pooling request to the LLM.
@@ -287,6 +301,7 @@ class PoolingRequestOutput(Generic[_O]):
num_cached_tokens: The number of tokens with prefix cache hit.
finished (bool): A flag indicating whether the pooling is completed.
error: Structured request-level error information, if the request failed.
late_chunking: Token and source ranges for fixed-length late chunking.
"""
@@ -299,8 +314,10 @@ class PoolingRequestOutput(Generic[_O]):
finished: bool,
*,
error: RequestError | None = None,
late_chunking: LateChunkingMetadata | None = None,
):
self.request_id = request_id
self.late_chunking = late_chunking
self.prompt_token_ids = prompt_token_ids
self.num_cached_tokens = num_cached_tokens
self.finished = finished
+47 -2
View File
@@ -36,6 +36,28 @@ class LateInteractionParams(
query_uses: int | None = None
class LateChunkingParams(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
array_like=True,
): # type: ignore[call-arg]
"""Fixed-length mean pooling before the token embedding head.
Requests recompute all prompt tokens, bypassing prefix cache reads.
Special tokens count toward chunk boundaries and are included in chunk means,
including the BOS token normally filtered from BGE-M3 token embeddings.
"""
chunk_size: int
def __post_init__(self) -> None:
self.verify()
def verify(self) -> None:
if type(self.chunk_size) is not int or self.chunk_size <= 0:
raise VLLMValidationError("chunk_size must be a positive integer")
class PoolingParams(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
@@ -48,6 +70,8 @@ class PoolingParams(
`None` uses the pooler's default, which is `True` in most cases.
dimensions: Reduce the dimensions of embeddings
if model support matryoshka representation.
late_chunking_params: Mean-pool contextual token states into fixed-size
chunks for the token embedding task.
"""
@@ -72,16 +96,22 @@ class PoolingParams(
extra_kwargs: dict[str, Any] | None = None
output_kind: RequestOutputKind = RequestOutputKind.FINAL_ONLY
## for token embedding models
# --8<-- [start:token-embed-pooling-params]
late_chunking_params: LateChunkingParams | None = None
"""Mean-pool contextual tokens into fixed-size chunks for `token_embed`."""
# --8<-- [end:token-embed-pooling-params]
@property
def all_parameters(self) -> list[str]:
return ["dimensions", "use_activation"]
return ["dimensions", "use_activation", "late_chunking_params"]
@property
def valid_parameters(self):
return {
"embed": ["dimensions", "use_activation"],
"classify": ["use_activation"],
"token_embed": ["dimensions", "use_activation"],
"token_embed": ["dimensions", "use_activation", "late_chunking_params"],
"token_classify": ["use_activation"],
}
@@ -90,6 +120,15 @@ class PoolingParams(
return deepcopy(self)
def verify(self, model_config: ModelConfig) -> None:
if self.late_chunking_params is not None:
self.late_chunking_params.verify()
if self.task != "token_embed":
raise VLLMValidationError("Late chunking requires token_embed")
if self.late_interaction_params is not None:
raise VLLMValidationError(
"Late chunking cannot be combined with late-interaction scoring"
)
# plugin task uses io_processor.parse_data to verify inputs,
# skipping PoolingParams verify
if self.task == "plugin":
@@ -107,6 +146,11 @@ class PoolingParams(
self._merge_default_parameters(model_config)
self._set_default_parameters(model_config)
self._verify_valid_parameters()
if self.late_chunking_params is not None:
if self.step_tag_id is not None:
raise VLLMValidationError("Late chunking requires all token rows")
# KV cache hits omit hidden states needed for complete chunk means.
self.skip_reading_prefix_cache = True
def _merge_default_parameters(self, model_config: ModelConfig) -> None:
pooler_config = model_config.pooler_config
@@ -228,6 +272,7 @@ class PoolingParams(
f"requires_token_ids={self.requires_token_ids}, "
f"skip_reading_prefix_cache={self.skip_reading_prefix_cache}, "
f"late_interaction_params={self.late_interaction_params}, "
f"late_chunking_params={self.late_chunking_params}, "
f"extra_kwargs={self.extra_kwargs})"
)