[Feature][Pooling] Add offline fixed-length late chunking (#60164)

Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
Linze Shi
2026-10-13 11:16:21 +00:00
committed by GitHub
parent 426f3bd36b
commit e6e84f350c
15 changed files with 1147 additions and 7 deletions
@@ -12,6 +12,7 @@ from openai.types.chat import (
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
from vllm import PoolingParams
from vllm.exceptions import VLLMValidationError
from vllm.inputs import tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -26,18 +27,26 @@ from vllm.utils.collection_utils import chunk_list
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..late_chunking import (
attach_late_chunking_metadata,
build_late_chunking_metadata,
prepare_late_chunking_input,
)
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
ChunkedEmbeddingMetadata,
EncodeChatRenderParams,
LateChunkingSlot,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
)
from .protocol import (
CohereEmbedContent,
@@ -657,10 +666,75 @@ class EmbedIOProcessor(PoolingIOProcessor):
class TokenEmbedIOProcessor(PoolingIOProcessor):
name = "token_embed"
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
factory, num_requests = super().get_request_factory_offline(ctx)
assert isinstance(ctx, OfflineEncodeInputsContext)
if ctx.pooling_params is None or not any(
p.late_chunking_params is not None
for p in self._params_to_seq(ctx.pooling_params, num_requests)
):
return factory, num_requests
if (ctx.tokenization_kwargs or {}).get("padding") not in (
None,
False,
"do_not_pad",
):
raise VLLMValidationError("Late chunking does not support input padding")
def request_factory() -> RequestGenerator:
for request in factory():
slot = None
if request["params"].late_chunking_params is not None:
slot = LateChunkingSlot()
request["late_chunking"] = slot
ctx.late_chunking.append(slot)
yield request
return request_factory, num_requests
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
late_chunking = render_params["params"].late_chunking_params
if late_chunking is None:
return super().render(render_params)
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
render_params = render_params.copy()
render_params["tok_params"] = tok_params
result = super().render(render_params)
prompt = result["prompts"]
if prompt["type"] != "token":
raise VLLMValidationError("Late chunking requires tokenized text")
render_params["late_chunking"].metadata = build_late_chunking_metadata(
text,
len(prompt["prompt_token_ids"]),
prompt.pop("prompt_token_offsets", None),
late_chunking.chunk_size,
)
return result
def post_process_offline(self, ctx: OfflineOutputsContext):
if ctx.late_chunking:
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
if slot is not None:
assert slot.metadata is not None
attach_late_chunking_metadata(output, slot.metadata)
return ctx.outputs
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
if render_params["params"].late_chunking_params is not None:
raise VLLMValidationError(
"JinaForRanking returns document/query vectors and does not support "
"late chunking"
)
return super().render(render_params)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import replace
from typing import cast
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
from vllm.renderers import TokenizeParams
from .typing import AnyRenderParam, EncodeCMPLRenderParams
def prepare_late_chunking_input(
config: VllmConfig, render_params: AnyRenderParam
) -> tuple[str, TokenizeParams]:
"""Validate full-document text input before requesting tokenizer offsets."""
params = render_params["params"]
params.verify(config.model_config)
if "prompts" not in render_params:
raise VLLMValidationError("Late chunking requires plain-text input")
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
if (
not isinstance(prompt, dict)
or set(prompt) - {"prompt", "cache_salt"}
or not isinstance(text := prompt.get("prompt"), str)
or not text
):
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
tok_params = render_params["tok_params"]
if (
tok_params.pad_prompt_tokens is not None
or tok_params.truncate_prompt_tokens is not None
or tok_params.do_lower_case
):
raise VLLMValidationError(
"Late chunking does not support input padding, truncation or "
"renderer text normalization"
)
return text, replace(tok_params, return_token_offsets=True)
def build_late_chunking_metadata(
text: str,
num_tokens: int,
offsets: Sequence[tuple[int, int]] | None,
chunk_size: int,
) -> LateChunkingMetadata:
"""Map chunks to original text, keeping special-token-only chunks."""
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
raise VLLMValidationError(
"Late chunking requires a fast tokenizer with offsets aligned to "
"all input tokens"
)
previous_start = previous_end = 0
for start, end in offsets:
if not 0 <= start <= end <= len(text):
raise VLLMValidationError("Token offsets are not aligned to the input text")
if start < end:
if start < previous_start or end < previous_end:
raise VLLMValidationError("Token offsets are not in source-text order")
previous_start, previous_end = start, end
chunks = []
for start in range(0, num_tokens, chunk_size):
end = min(start + chunk_size, num_tokens)
source = [(a, b) for a, b in offsets[start:end] if a < b]
char_range = (
(min(a for a, _ in source), max(b for _, b in source)) if source else None
)
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
return LateChunkingMetadata(
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
)
def attach_late_chunking_metadata(
output: PoolingRequestOutput, metadata: LateChunkingMetadata
) -> None:
"""Attach ranges only to successful, complete chunk-vector results."""
if not output.finished or output.error is not None:
return
data = output.outputs.data
if (
data.ndim != 2
or data.shape[0] != len(metadata.chunks)
or len(output.prompt_token_ids) != metadata.input_tokens
):
raise ValueError("Late-chunking output does not match its token/source ranges")
output.late_chunking = metadata
+1 -1
View File
@@ -129,7 +129,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs)
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
)
return outputs
+14 -1
View File
@@ -3,7 +3,7 @@
import time
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
@@ -11,6 +11,7 @@ from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import LateChunkingMetadata
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.renderers.inputs import DictPrompt
@@ -80,6 +81,13 @@ class ChunkedEmbeddingMetadata:
chunk_index: int
@dataclass
class LateChunkingSlot:
"""Per-document source ranges shared by frontend rendering and output contexts."""
metadata: LateChunkingMetadata | None = None
@dataclass(kw_only=True)
class PoolingServeContext(Generic[PoolingRequestT]):
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -124,6 +132,9 @@ class OfflineInputsContext:
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: int | Sequence[int] | None
late_chunking: list[LateChunkingSlot | None] = field(
default_factory=list, kw_only=True
)
@dataclass
@@ -158,6 +169,7 @@ class OfflineOutputsContext:
## for bi-encoder & late-interaction
n_queries: int | None = None
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
class RenderParams(TypedDict):
@@ -168,6 +180,7 @@ class RenderParams(TypedDict):
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
late_chunking: NotRequired[LateChunkingSlot]
class EncodeCMPLRenderParams(RenderParams):