[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from openai.types.chat import (
|
||||
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
@@ -26,18 +27,26 @@ from vllm.utils.collection_utils import chunk_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..late_chunking import (
|
||||
attach_late_chunking_metadata,
|
||||
build_late_chunking_metadata,
|
||||
prepare_late_chunking_input,
|
||||
)
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
ChunkedEmbeddingMetadata,
|
||||
EncodeChatRenderParams,
|
||||
LateChunkingSlot,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
)
|
||||
from .protocol import (
|
||||
CohereEmbedContent,
|
||||
@@ -657,10 +666,75 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "token_embed"
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
factory, num_requests = super().get_request_factory_offline(ctx)
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if ctx.pooling_params is None or not any(
|
||||
p.late_chunking_params is not None
|
||||
for p in self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
):
|
||||
return factory, num_requests
|
||||
|
||||
if (ctx.tokenization_kwargs or {}).get("padding") not in (
|
||||
None,
|
||||
False,
|
||||
"do_not_pad",
|
||||
):
|
||||
raise VLLMValidationError("Late chunking does not support input padding")
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for request in factory():
|
||||
slot = None
|
||||
if request["params"].late_chunking_params is not None:
|
||||
slot = LateChunkingSlot()
|
||||
request["late_chunking"] = slot
|
||||
ctx.late_chunking.append(slot)
|
||||
yield request
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
late_chunking = render_params["params"].late_chunking_params
|
||||
if late_chunking is None:
|
||||
return super().render(render_params)
|
||||
|
||||
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
|
||||
render_params = render_params.copy()
|
||||
render_params["tok_params"] = tok_params
|
||||
result = super().render(render_params)
|
||||
prompt = result["prompts"]
|
||||
if prompt["type"] != "token":
|
||||
raise VLLMValidationError("Late chunking requires tokenized text")
|
||||
render_params["late_chunking"].metadata = build_late_chunking_metadata(
|
||||
text,
|
||||
len(prompt["prompt_token_ids"]),
|
||||
prompt.pop("prompt_token_offsets", None),
|
||||
late_chunking.chunk_size,
|
||||
)
|
||||
return result
|
||||
|
||||
def post_process_offline(self, ctx: OfflineOutputsContext):
|
||||
if ctx.late_chunking:
|
||||
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
|
||||
if slot is not None:
|
||||
assert slot.metadata is not None
|
||||
attach_late_chunking_metadata(output, slot.metadata)
|
||||
return ctx.outputs
|
||||
|
||||
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
if render_params["params"].late_chunking_params is not None:
|
||||
raise VLLMValidationError(
|
||||
"JinaForRanking returns document/query vectors and does not support "
|
||||
"late chunking"
|
||||
)
|
||||
return super().render(render_params)
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
from .typing import AnyRenderParam, EncodeCMPLRenderParams
|
||||
|
||||
|
||||
def prepare_late_chunking_input(
|
||||
config: VllmConfig, render_params: AnyRenderParam
|
||||
) -> tuple[str, TokenizeParams]:
|
||||
"""Validate full-document text input before requesting tokenizer offsets."""
|
||||
params = render_params["params"]
|
||||
params.verify(config.model_config)
|
||||
if "prompts" not in render_params:
|
||||
raise VLLMValidationError("Late chunking requires plain-text input")
|
||||
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
|
||||
if (
|
||||
not isinstance(prompt, dict)
|
||||
or set(prompt) - {"prompt", "cache_salt"}
|
||||
or not isinstance(text := prompt.get("prompt"), str)
|
||||
or not text
|
||||
):
|
||||
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
|
||||
tok_params = render_params["tok_params"]
|
||||
if (
|
||||
tok_params.pad_prompt_tokens is not None
|
||||
or tok_params.truncate_prompt_tokens is not None
|
||||
or tok_params.do_lower_case
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Late chunking does not support input padding, truncation or "
|
||||
"renderer text normalization"
|
||||
)
|
||||
return text, replace(tok_params, return_token_offsets=True)
|
||||
|
||||
|
||||
def build_late_chunking_metadata(
|
||||
text: str,
|
||||
num_tokens: int,
|
||||
offsets: Sequence[tuple[int, int]] | None,
|
||||
chunk_size: int,
|
||||
) -> LateChunkingMetadata:
|
||||
"""Map chunks to original text, keeping special-token-only chunks."""
|
||||
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
|
||||
raise VLLMValidationError(
|
||||
"Late chunking requires a fast tokenizer with offsets aligned to "
|
||||
"all input tokens"
|
||||
)
|
||||
previous_start = previous_end = 0
|
||||
for start, end in offsets:
|
||||
if not 0 <= start <= end <= len(text):
|
||||
raise VLLMValidationError("Token offsets are not aligned to the input text")
|
||||
if start < end:
|
||||
if start < previous_start or end < previous_end:
|
||||
raise VLLMValidationError("Token offsets are not in source-text order")
|
||||
previous_start, previous_end = start, end
|
||||
|
||||
chunks = []
|
||||
for start in range(0, num_tokens, chunk_size):
|
||||
end = min(start + chunk_size, num_tokens)
|
||||
source = [(a, b) for a, b in offsets[start:end] if a < b]
|
||||
char_range = (
|
||||
(min(a for a, _ in source), max(b for _, b in source)) if source else None
|
||||
)
|
||||
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
|
||||
return LateChunkingMetadata(
|
||||
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
|
||||
)
|
||||
|
||||
|
||||
def attach_late_chunking_metadata(
|
||||
output: PoolingRequestOutput, metadata: LateChunkingMetadata
|
||||
) -> None:
|
||||
"""Attach ranges only to successful, complete chunk-vector results."""
|
||||
if not output.finished or output.error is not None:
|
||||
return
|
||||
data = output.outputs.data
|
||||
if (
|
||||
data.ndim != 2
|
||||
or data.shape[0] != len(metadata.chunks)
|
||||
or len(output.prompt_token_ids) != metadata.input_tokens
|
||||
):
|
||||
raise ValueError("Late-chunking output does not match its token/source ranges")
|
||||
output.late_chunking = metadata
|
||||
@@ -129,7 +129,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs)
|
||||
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
|
||||
)
|
||||
return outputs
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
|
||||
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
@@ -11,6 +11,7 @@ from pydantic import ConfigDict
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.inputs import DataPrompt, EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import LateChunkingMetadata
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.renderers.inputs import DictPrompt
|
||||
@@ -80,6 +81,13 @@ class ChunkedEmbeddingMetadata:
|
||||
chunk_index: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class LateChunkingSlot:
|
||||
"""Per-document source ranges shared by frontend rendering and output contexts."""
|
||||
|
||||
metadata: LateChunkingMetadata | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
@@ -124,6 +132,9 @@ class OfflineInputsContext:
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: int | Sequence[int] | None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(
|
||||
default_factory=list, kw_only=True
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -158,6 +169,7 @@ class OfflineOutputsContext:
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
|
||||
|
||||
|
||||
class RenderParams(TypedDict):
|
||||
@@ -168,6 +180,7 @@ class RenderParams(TypedDict):
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
late_chunking: NotRequired[LateChunkingSlot]
|
||||
|
||||
|
||||
class EncodeCMPLRenderParams(RenderParams):
|
||||
|
||||
@@ -198,6 +198,11 @@ class BOSEOSFilter(Pooler):
|
||||
for i, (prompt_len, token_ids) in enumerate(
|
||||
zip(pooling_metadata.prompt_lens, prompt_token_ids)
|
||||
):
|
||||
# Late chunking returns one row per chunk, with special tokens
|
||||
# already included in the means. Skip BOS/EOS row removal for these
|
||||
# requests; ordinary token outputs still use the filtering below.
|
||||
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
|
||||
continue
|
||||
pooled_data = pooled_outputs[i]
|
||||
assert (
|
||||
isinstance(pooled_data, torch.Tensor)
|
||||
|
||||
@@ -17,6 +17,23 @@ from .methods import TokenPoolingMethodOutputItem
|
||||
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
|
||||
|
||||
|
||||
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
|
||||
"""Mean-pool complete contextual states, accumulating in FP32."""
|
||||
chunk_size = min(chunk_size, len(data))
|
||||
num_full, remainder = divmod(len(data), chunk_size)
|
||||
full_end = num_full * chunk_size
|
||||
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
|
||||
if not remainder:
|
||||
return full_chunks.mean(dim=1, dtype=torch.float32)
|
||||
|
||||
return torch.cat(
|
||||
[
|
||||
full_chunks.mean(dim=1, dtype=torch.float32),
|
||||
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TokenPoolerHead(nn.Module, ABC):
|
||||
@abstractmethod
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
@@ -80,6 +97,12 @@ class TokenEmbeddingPoolerHead(TokenPoolerHead):
|
||||
if pooled_data is None:
|
||||
return None
|
||||
|
||||
if (late_chunking := pooling_param.late_chunking_params) is not None:
|
||||
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
|
||||
pooled_data = (
|
||||
means.to(pooled_data.dtype) if self.head_dtype is None else means
|
||||
)
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
# pooled_data shape: [n_tokens, hidden_size]
|
||||
|
||||
@@ -277,6 +277,20 @@ STREAM_FINISHED = RequestOutput(
|
||||
_O = TypeVar("_O", default=PoolingOutput)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LateChunk:
|
||||
token_range: tuple[int, int]
|
||||
char_range: tuple[int, int] | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LateChunkingMetadata:
|
||||
chunk_size: int
|
||||
input_tokens: int
|
||||
chunks: list[LateChunk]
|
||||
strategy: str = "fixed_tokens"
|
||||
|
||||
|
||||
class PoolingRequestOutput(Generic[_O]):
|
||||
"""The output data of a pooling request to the LLM.
|
||||
|
||||
@@ -287,6 +301,7 @@ class PoolingRequestOutput(Generic[_O]):
|
||||
num_cached_tokens: The number of tokens with prefix cache hit.
|
||||
finished (bool): A flag indicating whether the pooling is completed.
|
||||
error: Structured request-level error information, if the request failed.
|
||||
late_chunking: Token and source ranges for fixed-length late chunking.
|
||||
|
||||
"""
|
||||
|
||||
@@ -299,8 +314,10 @@ class PoolingRequestOutput(Generic[_O]):
|
||||
finished: bool,
|
||||
*,
|
||||
error: RequestError | None = None,
|
||||
late_chunking: LateChunkingMetadata | None = None,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.late_chunking = late_chunking
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.num_cached_tokens = num_cached_tokens
|
||||
self.finished = finished
|
||||
|
||||
+47
-2
@@ -36,6 +36,28 @@ class LateInteractionParams(
|
||||
query_uses: int | None = None
|
||||
|
||||
|
||||
class LateChunkingParams(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
array_like=True,
|
||||
): # type: ignore[call-arg]
|
||||
"""Fixed-length mean pooling before the token embedding head.
|
||||
|
||||
Requests recompute all prompt tokens, bypassing prefix cache reads.
|
||||
Special tokens count toward chunk boundaries and are included in chunk means,
|
||||
including the BOS token normally filtered from BGE-M3 token embeddings.
|
||||
"""
|
||||
|
||||
chunk_size: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.verify()
|
||||
|
||||
def verify(self) -> None:
|
||||
if type(self.chunk_size) is not int or self.chunk_size <= 0:
|
||||
raise VLLMValidationError("chunk_size must be a positive integer")
|
||||
|
||||
|
||||
class PoolingParams(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
@@ -48,6 +70,8 @@ class PoolingParams(
|
||||
`None` uses the pooler's default, which is `True` in most cases.
|
||||
dimensions: Reduce the dimensions of embeddings
|
||||
if model support matryoshka representation.
|
||||
late_chunking_params: Mean-pool contextual token states into fixed-size
|
||||
chunks for the token embedding task.
|
||||
|
||||
"""
|
||||
|
||||
@@ -72,16 +96,22 @@ class PoolingParams(
|
||||
extra_kwargs: dict[str, Any] | None = None
|
||||
output_kind: RequestOutputKind = RequestOutputKind.FINAL_ONLY
|
||||
|
||||
## for token embedding models
|
||||
# --8<-- [start:token-embed-pooling-params]
|
||||
late_chunking_params: LateChunkingParams | None = None
|
||||
"""Mean-pool contextual tokens into fixed-size chunks for `token_embed`."""
|
||||
# --8<-- [end:token-embed-pooling-params]
|
||||
|
||||
@property
|
||||
def all_parameters(self) -> list[str]:
|
||||
return ["dimensions", "use_activation"]
|
||||
return ["dimensions", "use_activation", "late_chunking_params"]
|
||||
|
||||
@property
|
||||
def valid_parameters(self):
|
||||
return {
|
||||
"embed": ["dimensions", "use_activation"],
|
||||
"classify": ["use_activation"],
|
||||
"token_embed": ["dimensions", "use_activation"],
|
||||
"token_embed": ["dimensions", "use_activation", "late_chunking_params"],
|
||||
"token_classify": ["use_activation"],
|
||||
}
|
||||
|
||||
@@ -90,6 +120,15 @@ class PoolingParams(
|
||||
return deepcopy(self)
|
||||
|
||||
def verify(self, model_config: ModelConfig) -> None:
|
||||
if self.late_chunking_params is not None:
|
||||
self.late_chunking_params.verify()
|
||||
if self.task != "token_embed":
|
||||
raise VLLMValidationError("Late chunking requires token_embed")
|
||||
if self.late_interaction_params is not None:
|
||||
raise VLLMValidationError(
|
||||
"Late chunking cannot be combined with late-interaction scoring"
|
||||
)
|
||||
|
||||
# plugin task uses io_processor.parse_data to verify inputs,
|
||||
# skipping PoolingParams verify
|
||||
if self.task == "plugin":
|
||||
@@ -107,6 +146,11 @@ class PoolingParams(
|
||||
self._merge_default_parameters(model_config)
|
||||
self._set_default_parameters(model_config)
|
||||
self._verify_valid_parameters()
|
||||
if self.late_chunking_params is not None:
|
||||
if self.step_tag_id is not None:
|
||||
raise VLLMValidationError("Late chunking requires all token rows")
|
||||
# KV cache hits omit hidden states needed for complete chunk means.
|
||||
self.skip_reading_prefix_cache = True
|
||||
|
||||
def _merge_default_parameters(self, model_config: ModelConfig) -> None:
|
||||
pooler_config = model_config.pooler_config
|
||||
@@ -228,6 +272,7 @@ class PoolingParams(
|
||||
f"requires_token_ids={self.requires_token_ids}, "
|
||||
f"skip_reading_prefix_cache={self.skip_reading_prefix_cache}, "
|
||||
f"late_interaction_params={self.late_interaction_params}, "
|
||||
f"late_chunking_params={self.late_chunking_params}, "
|
||||
f"extra_kwargs={self.extra_kwargs})"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user