[Feature][Pooling] Add offline fixed-length late chunking (#60164)

Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
Linze Shi
2026-10-13 11:16:21 +00:00
committed by GitHub
parent 426f3bd36b
commit e6e84f350c
15 changed files with 1147 additions and 7 deletions
+14 -1
View File
@@ -3,7 +3,7 @@
import time
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
@@ -11,6 +11,7 @@ from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import LateChunkingMetadata
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.renderers.inputs import DictPrompt
@@ -80,6 +81,13 @@ class ChunkedEmbeddingMetadata:
chunk_index: int
@dataclass
class LateChunkingSlot:
"""Per-document source ranges shared by frontend rendering and output contexts."""
metadata: LateChunkingMetadata | None = None
@dataclass(kw_only=True)
class PoolingServeContext(Generic[PoolingRequestT]):
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -124,6 +132,9 @@ class OfflineInputsContext:
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: int | Sequence[int] | None
late_chunking: list[LateChunkingSlot | None] = field(
default_factory=list, kw_only=True
)
@dataclass
@@ -158,6 +169,7 @@ class OfflineOutputsContext:
## for bi-encoder & late-interaction
n_queries: int | None = None
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
class RenderParams(TypedDict):
@@ -168,6 +180,7 @@ class RenderParams(TypedDict):
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
late_chunking: NotRequired[LateChunkingSlot]
class EncodeCMPLRenderParams(RenderParams):