[Feature][Pooling] Add offline fixed-length late chunking (#60164)
Signed-off-by: Linze-Shi <linzeshi0@gmail.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from openai.types.chat import (
|
||||
from openai.types.chat.chat_completion_content_part_image_param import ImageURL
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
@@ -26,18 +27,26 @@ from vllm.utils.collection_utils import chunk_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..late_chunking import (
|
||||
attach_late_chunking_metadata,
|
||||
build_late_chunking_metadata,
|
||||
prepare_late_chunking_input,
|
||||
)
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
ChunkedEmbeddingMetadata,
|
||||
EncodeChatRenderParams,
|
||||
LateChunkingSlot,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
)
|
||||
from .protocol import (
|
||||
CohereEmbedContent,
|
||||
@@ -657,10 +666,75 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "token_embed"
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
factory, num_requests = super().get_request_factory_offline(ctx)
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if ctx.pooling_params is None or not any(
|
||||
p.late_chunking_params is not None
|
||||
for p in self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
):
|
||||
return factory, num_requests
|
||||
|
||||
if (ctx.tokenization_kwargs or {}).get("padding") not in (
|
||||
None,
|
||||
False,
|
||||
"do_not_pad",
|
||||
):
|
||||
raise VLLMValidationError("Late chunking does not support input padding")
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for request in factory():
|
||||
slot = None
|
||||
if request["params"].late_chunking_params is not None:
|
||||
slot = LateChunkingSlot()
|
||||
request["late_chunking"] = slot
|
||||
ctx.late_chunking.append(slot)
|
||||
yield request
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
late_chunking = render_params["params"].late_chunking_params
|
||||
if late_chunking is None:
|
||||
return super().render(render_params)
|
||||
|
||||
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
|
||||
render_params = render_params.copy()
|
||||
render_params["tok_params"] = tok_params
|
||||
result = super().render(render_params)
|
||||
prompt = result["prompts"]
|
||||
if prompt["type"] != "token":
|
||||
raise VLLMValidationError("Late chunking requires tokenized text")
|
||||
render_params["late_chunking"].metadata = build_late_chunking_metadata(
|
||||
text,
|
||||
len(prompt["prompt_token_ids"]),
|
||||
prompt.pop("prompt_token_offsets", None),
|
||||
late_chunking.chunk_size,
|
||||
)
|
||||
return result
|
||||
|
||||
def post_process_offline(self, ctx: OfflineOutputsContext):
|
||||
if ctx.late_chunking:
|
||||
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
|
||||
if slot is not None:
|
||||
assert slot.metadata is not None
|
||||
attach_late_chunking_metadata(output, slot.metadata)
|
||||
return ctx.outputs
|
||||
|
||||
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
if render_params["params"].late_chunking_params is not None:
|
||||
raise VLLMValidationError(
|
||||
"JinaForRanking returns document/query vectors and does not support "
|
||||
"late chunking"
|
||||
)
|
||||
return super().render(render_params)
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
|
||||
Reference in New Issue
Block a user