Fixed chunked embedding aggregation with request-id metadata (#46782)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import torch
|
||||
@@ -27,6 +28,7 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
ChunkedEmbeddingMetadata,
|
||||
OfflineInputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
@@ -46,6 +48,12 @@ from .protocol import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ChunkedPromptAggregator:
|
||||
weighted_sum: torch.Tensor | None = None
|
||||
total_weight: int = 0
|
||||
|
||||
|
||||
class EmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "embed"
|
||||
|
||||
@@ -113,6 +121,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
chunked_engine_inputs: list[EngineInput] = []
|
||||
prompt_request_ids: list[str] = []
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
|
||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||
token_ids = engine_input.get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
@@ -132,9 +141,16 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
prompt_request_ids.append(
|
||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||
)
|
||||
chunked_embedding_metadata.append(
|
||||
ChunkedEmbeddingMetadata(
|
||||
prompt_index=prompt_idx,
|
||||
chunk_index=chunk_idx,
|
||||
)
|
||||
)
|
||||
|
||||
ctx.engine_inputs = chunked_engine_inputs
|
||||
ctx.prompt_request_ids = prompt_request_ids
|
||||
ctx.chunked_embedding_metadata = chunked_embedding_metadata
|
||||
|
||||
return None
|
||||
|
||||
@@ -142,66 +158,48 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
# Online aggregation for chunked requests to
|
||||
# minimize memory usage
|
||||
# Track aggregation state for each prompt
|
||||
prompt_aggregators: dict[int, dict[str, Any]] = {}
|
||||
short_prompts_results: dict[int, PoolingRequestOutput] = {}
|
||||
for result_idx, result in enumerate(ctx.final_res_batch):
|
||||
if "-chunk-" not in result.request_id:
|
||||
# Non-chunked result - extract prompt_idx from request_id
|
||||
parts = result.request_id.split("-")
|
||||
try:
|
||||
# Last part should be prompt index
|
||||
prompt_idx = int(parts[-1])
|
||||
except (ValueError, IndexError):
|
||||
prompt_idx = result_idx # Fallback to result_idx
|
||||
if ctx.chunked_embedding_metadata is None:
|
||||
raise ValueError("Chunked embedding metadata not available")
|
||||
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
|
||||
raise ValueError(
|
||||
"Chunked embedding metadata count does not match result count"
|
||||
)
|
||||
|
||||
short_prompts_results[prompt_idx] = result
|
||||
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
|
||||
for result, chunk_metadata in zip(
|
||||
ctx.final_res_batch, ctx.chunked_embedding_metadata
|
||||
):
|
||||
prompt_idx = chunk_metadata.prompt_index
|
||||
aggregator = prompt_aggregators.setdefault(
|
||||
prompt_idx, _ChunkedPromptAggregator()
|
||||
)
|
||||
|
||||
# MEAN pooling with online weighted averaging
|
||||
# Ensure result is PoolingRequestOutput
|
||||
# for embedding processing
|
||||
if not isinstance(result, PoolingRequestOutput):
|
||||
raise ValueError(
|
||||
f"Expected PoolingRequestOutput for "
|
||||
f"chunked embedding, got "
|
||||
f"{type(result).__name__}"
|
||||
)
|
||||
if result.prompt_token_ids is None:
|
||||
raise ValueError(
|
||||
"prompt_token_ids cannot be None for chunked processing"
|
||||
)
|
||||
|
||||
weight = len(result.prompt_token_ids)
|
||||
embedding_data = result.outputs.data
|
||||
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
|
||||
|
||||
if aggregator.weighted_sum is None:
|
||||
# First chunk
|
||||
aggregator.weighted_sum = weighted_embedding
|
||||
else:
|
||||
# Extract prompt_idx from chunked request_id
|
||||
parts = result.request_id.split("-")
|
||||
try:
|
||||
prompt_idx = int(parts[parts.index("prompt") + 1])
|
||||
except (ValueError, IndexError):
|
||||
# Fallback: extract from result_idx if parsing fails
|
||||
prompt_idx = result_idx
|
||||
# Accumulate
|
||||
aggregator.weighted_sum += weighted_embedding
|
||||
|
||||
# Initialize aggregator for this prompt if needed
|
||||
if prompt_idx not in prompt_aggregators:
|
||||
prompt_aggregators[prompt_idx] = {
|
||||
"weighted_sum": None,
|
||||
"total_weight": 0,
|
||||
"chunk_count": 0,
|
||||
"request_id": result.request_id.split("-chunk-")[0],
|
||||
}
|
||||
|
||||
aggregator = prompt_aggregators[prompt_idx]
|
||||
|
||||
# MEAN pooling with online weighted averaging
|
||||
# Ensure result is PoolingRequestOutput
|
||||
# for embedding processing
|
||||
if not isinstance(result, PoolingRequestOutput):
|
||||
raise ValueError(
|
||||
f"Expected PoolingRequestOutput for "
|
||||
f"chunked embedding, got "
|
||||
f"{type(result).__name__}"
|
||||
)
|
||||
if result.prompt_token_ids is None:
|
||||
raise ValueError(
|
||||
"prompt_token_ids cannot be None for chunked processing"
|
||||
)
|
||||
|
||||
weight = len(result.prompt_token_ids)
|
||||
embedding_data = result.outputs.data
|
||||
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
|
||||
|
||||
if aggregator["weighted_sum"] is None:
|
||||
# First chunk
|
||||
aggregator["weighted_sum"] = weighted_embedding
|
||||
else:
|
||||
# Accumulate
|
||||
aggregator["weighted_sum"] += weighted_embedding
|
||||
|
||||
aggregator["total_weight"] += weight
|
||||
aggregator["chunk_count"] += 1
|
||||
aggregator.total_weight += weight
|
||||
|
||||
if ctx.original_engine_inputs is None:
|
||||
raise ValueError("Original engine inputs not available")
|
||||
@@ -216,8 +214,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
# Finalize MEAN aggregation for this chunked prompt
|
||||
aggregator = prompt_aggregators[prompt_idx]
|
||||
|
||||
weighted_sum = aggregator["weighted_sum"]
|
||||
total_weight = aggregator["total_weight"]
|
||||
weighted_sum = aggregator.weighted_sum
|
||||
total_weight = aggregator.total_weight
|
||||
|
||||
if (
|
||||
weighted_sum is not None
|
||||
@@ -243,7 +241,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
|
||||
original_token_ids = cast(list[int], token_ids)
|
||||
pooling_request_output = PoolingRequestOutput(
|
||||
request_id=aggregator["request_id"],
|
||||
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
|
||||
prompt_token_ids=original_token_ids,
|
||||
outputs=pooling_output_data,
|
||||
num_cached_tokens=0,
|
||||
@@ -255,8 +253,6 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
raise ValueError(
|
||||
f"Failed to aggregate chunks for prompt {prompt_idx}"
|
||||
)
|
||||
elif prompt_idx in short_prompts_results:
|
||||
final_res_batch.append(short_prompts_results[prompt_idx])
|
||||
else:
|
||||
raise ValueError(f"Result not found for prompt {prompt_idx}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user