Fixed chunked embedding aggregation with request-id metadata (#46782)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-06-27 20:57:47 +00:00
committed by GitHub
parent ea2ead1db3
commit 8bf064f8d3
3 changed files with 157 additions and 62 deletions
+58 -62
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, Literal, cast
import torch
@@ -27,6 +28,7 @@ from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
ChunkedEmbeddingMetadata,
OfflineInputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
@@ -46,6 +48,12 @@ from .protocol import (
logger = init_logger(__name__)
@dataclass
class _ChunkedPromptAggregator:
weighted_sum: torch.Tensor | None = None
total_weight: int = 0
class EmbedIOProcessor(PoolingIOProcessor):
name = "embed"
@@ -113,6 +121,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
max_model_len = self.model_config.max_model_len
chunked_engine_inputs: list[EngineInput] = []
prompt_request_ids: list[str] = []
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
token_ids = engine_input.get("prompt_token_ids", None)
if token_ids is None:
@@ -132,9 +141,16 @@ class EmbedIOProcessor(PoolingIOProcessor):
prompt_request_ids.append(
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
)
chunked_embedding_metadata.append(
ChunkedEmbeddingMetadata(
prompt_index=prompt_idx,
chunk_index=chunk_idx,
)
)
ctx.engine_inputs = chunked_engine_inputs
ctx.prompt_request_ids = prompt_request_ids
ctx.chunked_embedding_metadata = chunked_embedding_metadata
return None
@@ -142,66 +158,48 @@ class EmbedIOProcessor(PoolingIOProcessor):
# Online aggregation for chunked requests to
# minimize memory usage
# Track aggregation state for each prompt
prompt_aggregators: dict[int, dict[str, Any]] = {}
short_prompts_results: dict[int, PoolingRequestOutput] = {}
for result_idx, result in enumerate(ctx.final_res_batch):
if "-chunk-" not in result.request_id:
# Non-chunked result - extract prompt_idx from request_id
parts = result.request_id.split("-")
try:
# Last part should be prompt index
prompt_idx = int(parts[-1])
except (ValueError, IndexError):
prompt_idx = result_idx # Fallback to result_idx
if ctx.chunked_embedding_metadata is None:
raise ValueError("Chunked embedding metadata not available")
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
raise ValueError(
"Chunked embedding metadata count does not match result count"
)
short_prompts_results[prompt_idx] = result
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
for result, chunk_metadata in zip(
ctx.final_res_batch, ctx.chunked_embedding_metadata
):
prompt_idx = chunk_metadata.prompt_index
aggregator = prompt_aggregators.setdefault(
prompt_idx, _ChunkedPromptAggregator()
)
# MEAN pooling with online weighted averaging
# Ensure result is PoolingRequestOutput
# for embedding processing
if not isinstance(result, PoolingRequestOutput):
raise ValueError(
f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}"
)
if result.prompt_token_ids is None:
raise ValueError(
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
embedding_data = result.outputs.data
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
if aggregator.weighted_sum is None:
# First chunk
aggregator.weighted_sum = weighted_embedding
else:
# Extract prompt_idx from chunked request_id
parts = result.request_id.split("-")
try:
prompt_idx = int(parts[parts.index("prompt") + 1])
except (ValueError, IndexError):
# Fallback: extract from result_idx if parsing fails
prompt_idx = result_idx
# Accumulate
aggregator.weighted_sum += weighted_embedding
# Initialize aggregator for this prompt if needed
if prompt_idx not in prompt_aggregators:
prompt_aggregators[prompt_idx] = {
"weighted_sum": None,
"total_weight": 0,
"chunk_count": 0,
"request_id": result.request_id.split("-chunk-")[0],
}
aggregator = prompt_aggregators[prompt_idx]
# MEAN pooling with online weighted averaging
# Ensure result is PoolingRequestOutput
# for embedding processing
if not isinstance(result, PoolingRequestOutput):
raise ValueError(
f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}"
)
if result.prompt_token_ids is None:
raise ValueError(
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
embedding_data = result.outputs.data
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
if aggregator["weighted_sum"] is None:
# First chunk
aggregator["weighted_sum"] = weighted_embedding
else:
# Accumulate
aggregator["weighted_sum"] += weighted_embedding
aggregator["total_weight"] += weight
aggregator["chunk_count"] += 1
aggregator.total_weight += weight
if ctx.original_engine_inputs is None:
raise ValueError("Original engine inputs not available")
@@ -216,8 +214,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
# Finalize MEAN aggregation for this chunked prompt
aggregator = prompt_aggregators[prompt_idx]
weighted_sum = aggregator["weighted_sum"]
total_weight = aggregator["total_weight"]
weighted_sum = aggregator.weighted_sum
total_weight = aggregator.total_weight
if (
weighted_sum is not None
@@ -243,7 +241,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
original_token_ids = cast(list[int], token_ids)
pooling_request_output = PoolingRequestOutput(
request_id=aggregator["request_id"],
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
num_cached_tokens=0,
@@ -255,8 +253,6 @@ class EmbedIOProcessor(PoolingIOProcessor):
raise ValueError(
f"Failed to aggregate chunks for prompt {prompt_idx}"
)
elif prompt_idx in short_prompts_results:
final_res_batch.append(short_prompts_results[prompt_idx])
else:
raise ValueError(f"Result not found for prompt {prompt_idx}")