[Feature][Pooling] Add offline fixed-length late chunking (#60164)

Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
Linze Shi
2026-10-13 11:16:21 +00:00
committed by GitHub
parent 426f3bd36b
commit e6e84f350c
15 changed files with 1147 additions and 7 deletions
@@ -12,6 +12,7 @@ from openai.types.chat import (
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
from vllm import PoolingParams
from vllm.exceptions import VLLMValidationError
from vllm.inputs import tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -26,18 +27,26 @@ from vllm.utils.collection_utils import chunk_list
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..late_chunking import (
attach_late_chunking_metadata,
build_late_chunking_metadata,
prepare_late_chunking_input,
)
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
ChunkedEmbeddingMetadata,
EncodeChatRenderParams,
LateChunkingSlot,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
)
from .protocol import (
CohereEmbedContent,
@@ -657,10 +666,75 @@ class EmbedIOProcessor(PoolingIOProcessor):
class TokenEmbedIOProcessor(PoolingIOProcessor):
name = "token_embed"
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
factory, num_requests = super().get_request_factory_offline(ctx)
assert isinstance(ctx, OfflineEncodeInputsContext)
if ctx.pooling_params is None or not any(
p.late_chunking_params is not None
for p in self._params_to_seq(ctx.pooling_params, num_requests)
):
return factory, num_requests
if (ctx.tokenization_kwargs or {}).get("padding") not in (
None,
False,
"do_not_pad",
):
raise VLLMValidationError("Late chunking does not support input padding")
def request_factory() -> RequestGenerator:
for request in factory():
slot = None
if request["params"].late_chunking_params is not None:
slot = LateChunkingSlot()
request["late_chunking"] = slot
ctx.late_chunking.append(slot)
yield request
return request_factory, num_requests
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
late_chunking = render_params["params"].late_chunking_params
if late_chunking is None:
return super().render(render_params)
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
render_params = render_params.copy()
render_params["tok_params"] = tok_params
result = super().render(render_params)
prompt = result["prompts"]
if prompt["type"] != "token":
raise VLLMValidationError("Late chunking requires tokenized text")
render_params["late_chunking"].metadata = build_late_chunking_metadata(
text,
len(prompt["prompt_token_ids"]),
prompt.pop("prompt_token_offsets", None),
late_chunking.chunk_size,
)
return result
def post_process_offline(self, ctx: OfflineOutputsContext):
if ctx.late_chunking:
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
if slot is not None:
assert slot.metadata is not None
attach_late_chunking_metadata(output, slot.metadata)
return ctx.outputs
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
if render_params["params"].late_chunking_params is not None:
raise VLLMValidationError(
"JinaForRanking returns document/query vectors and does not support "
"late chunking"
)
return super().render(render_params)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]: