[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
|
||||
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
@@ -11,6 +11,7 @@ from pydantic import ConfigDict
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.inputs import DataPrompt, EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import LateChunkingMetadata
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.renderers.inputs import DictPrompt
|
||||
@@ -80,6 +81,13 @@ class ChunkedEmbeddingMetadata:
|
||||
chunk_index: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class LateChunkingSlot:
|
||||
"""Per-document source ranges shared by frontend rendering and output contexts."""
|
||||
|
||||
metadata: LateChunkingMetadata | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
@@ -124,6 +132,9 @@ class OfflineInputsContext:
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: int | Sequence[int] | None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(
|
||||
default_factory=list, kw_only=True
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -158,6 +169,7 @@ class OfflineOutputsContext:
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
|
||||
|
||||
|
||||
class RenderParams(TypedDict):
|
||||
@@ -168,6 +180,7 @@ class RenderParams(TypedDict):
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
late_chunking: NotRequired[LateChunkingSlot]
|
||||
|
||||
|
||||
class EncodeCMPLRenderParams(RenderParams):
|
||||
|
||||
Reference in New Issue
Block a user