[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
from .typing import AnyRenderParam, EncodeCMPLRenderParams
|
||||
|
||||
|
||||
def prepare_late_chunking_input(
|
||||
config: VllmConfig, render_params: AnyRenderParam
|
||||
) -> tuple[str, TokenizeParams]:
|
||||
"""Validate full-document text input before requesting tokenizer offsets."""
|
||||
params = render_params["params"]
|
||||
params.verify(config.model_config)
|
||||
if "prompts" not in render_params:
|
||||
raise VLLMValidationError("Late chunking requires plain-text input")
|
||||
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
|
||||
if (
|
||||
not isinstance(prompt, dict)
|
||||
or set(prompt) - {"prompt", "cache_salt"}
|
||||
or not isinstance(text := prompt.get("prompt"), str)
|
||||
or not text
|
||||
):
|
||||
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
|
||||
tok_params = render_params["tok_params"]
|
||||
if (
|
||||
tok_params.pad_prompt_tokens is not None
|
||||
or tok_params.truncate_prompt_tokens is not None
|
||||
or tok_params.do_lower_case
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Late chunking does not support input padding, truncation or "
|
||||
"renderer text normalization"
|
||||
)
|
||||
return text, replace(tok_params, return_token_offsets=True)
|
||||
|
||||
|
||||
def build_late_chunking_metadata(
|
||||
text: str,
|
||||
num_tokens: int,
|
||||
offsets: Sequence[tuple[int, int]] | None,
|
||||
chunk_size: int,
|
||||
) -> LateChunkingMetadata:
|
||||
"""Map chunks to original text, keeping special-token-only chunks."""
|
||||
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
|
||||
raise VLLMValidationError(
|
||||
"Late chunking requires a fast tokenizer with offsets aligned to "
|
||||
"all input tokens"
|
||||
)
|
||||
previous_start = previous_end = 0
|
||||
for start, end in offsets:
|
||||
if not 0 <= start <= end <= len(text):
|
||||
raise VLLMValidationError("Token offsets are not aligned to the input text")
|
||||
if start < end:
|
||||
if start < previous_start or end < previous_end:
|
||||
raise VLLMValidationError("Token offsets are not in source-text order")
|
||||
previous_start, previous_end = start, end
|
||||
|
||||
chunks = []
|
||||
for start in range(0, num_tokens, chunk_size):
|
||||
end = min(start + chunk_size, num_tokens)
|
||||
source = [(a, b) for a, b in offsets[start:end] if a < b]
|
||||
char_range = (
|
||||
(min(a for a, _ in source), max(b for _, b in source)) if source else None
|
||||
)
|
||||
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
|
||||
return LateChunkingMetadata(
|
||||
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
|
||||
)
|
||||
|
||||
|
||||
def attach_late_chunking_metadata(
|
||||
output: PoolingRequestOutput, metadata: LateChunkingMetadata
|
||||
) -> None:
|
||||
"""Attach ranges only to successful, complete chunk-vector results."""
|
||||
if not output.finished or output.error is not None:
|
||||
return
|
||||
data = output.outputs.data
|
||||
if (
|
||||
data.ndim != 2
|
||||
or data.shape[0] != len(metadata.chunks)
|
||||
or len(output.prompt_token_ids) != metadata.input_tokens
|
||||
):
|
||||
raise ValueError("Late-chunking output does not match its token/source ranges")
|
||||
output.late_chunking = metadata
|
||||
Reference in New Issue
Block a user