Fixed chunked embedding aggregation with request-id metadata (#46782)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
|||||||
"""Unit tests for EmbedIOProcessor."""
|
"""Unit tests for EmbedIOProcessor."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
import torch
|
||||||
from pydantic import TypeAdapter, ValidationError
|
from pydantic import TypeAdapter, ValidationError
|
||||||
|
|
||||||
from vllm import PoolingParams
|
from vllm import PoolingParams
|
||||||
@@ -19,6 +20,7 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
|||||||
EmbeddingRequest,
|
EmbeddingRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||||
|
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||||
|
|
||||||
|
|
||||||
class TestEmbeddingRequestParsing:
|
class TestEmbeddingRequestParsing:
|
||||||
@@ -398,6 +400,96 @@ class TestValidateInputType:
|
|||||||
handler._validate_input_type("z")
|
handler._validate_input_type("z")
|
||||||
|
|
||||||
|
|
||||||
|
class TestChunkedEmbeddingProcessing:
|
||||||
|
"""Unit tests for chunked embedding aggregation."""
|
||||||
|
|
||||||
|
class _FakeModelConfig:
|
||||||
|
max_model_len = 3
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _make_handler(cls):
|
||||||
|
handler = object.__new__(EmbedIOProcessor)
|
||||||
|
handler.model_config = cls._FakeModelConfig()
|
||||||
|
return handler
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
|
||||||
|
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||||
|
{
|
||||||
|
"model": "test",
|
||||||
|
"input": [[0, 1, 2, 3, 4], [10, 11]],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert isinstance(request, EmbeddingCompletionRequest)
|
||||||
|
return PoolingServeContext(
|
||||||
|
request=request,
|
||||||
|
pooling_params=PoolingParams(),
|
||||||
|
model_name="test",
|
||||||
|
request_id="embd-client-prompt-999-chunk-888",
|
||||||
|
engine_inputs=[
|
||||||
|
{"prompt_token_ids": [0, 1, 2, 3, 4]},
|
||||||
|
{"prompt_token_ids": [10, 11]},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _make_output(
|
||||||
|
request_id: str,
|
||||||
|
prompt_token_ids: list[int],
|
||||||
|
embedding: list[float],
|
||||||
|
) -> PoolingRequestOutput:
|
||||||
|
return PoolingRequestOutput(
|
||||||
|
request_id=request_id,
|
||||||
|
outputs=PoolingOutput(data=torch.tensor(embedding)),
|
||||||
|
prompt_token_ids=prompt_token_ids,
|
||||||
|
num_cached_tokens=0,
|
||||||
|
finished=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_aggregation_uses_metadata_not_request_id_parsing(self):
|
||||||
|
handler = self._make_handler()
|
||||||
|
ctx = self._make_context()
|
||||||
|
|
||||||
|
handler._pre_process_chunked(ctx)
|
||||||
|
|
||||||
|
assert ctx.prompt_request_ids == [
|
||||||
|
"embd-client-prompt-999-chunk-888-prompt-0-chunk-0",
|
||||||
|
"embd-client-prompt-999-chunk-888-prompt-0-chunk-1",
|
||||||
|
"embd-client-prompt-999-chunk-888-prompt-1-chunk-0",
|
||||||
|
]
|
||||||
|
assert ctx.chunked_embedding_metadata is not None
|
||||||
|
assert [
|
||||||
|
(item.prompt_index, item.chunk_index)
|
||||||
|
for item in ctx.chunked_embedding_metadata
|
||||||
|
] == [(0, 0), (0, 1), (1, 0)]
|
||||||
|
|
||||||
|
ctx.final_res_batch = [
|
||||||
|
self._make_output(ctx.prompt_request_ids[0], [0, 1, 2], [1.0, 1.0]),
|
||||||
|
self._make_output(ctx.prompt_request_ids[1], [3, 4], [4.0, 7.0]),
|
||||||
|
self._make_output(ctx.prompt_request_ids[2], [10, 11], [9.0, 9.0]),
|
||||||
|
]
|
||||||
|
|
||||||
|
handler._post_process_chunked(ctx)
|
||||||
|
|
||||||
|
assert len(ctx.final_res_batch) == 2
|
||||||
|
assert ctx.final_res_batch[0].request_id == (
|
||||||
|
"embd-client-prompt-999-chunk-888-prompt-0"
|
||||||
|
)
|
||||||
|
assert ctx.final_res_batch[0].prompt_token_ids == [0, 1, 2, 3, 4]
|
||||||
|
assert torch.allclose(
|
||||||
|
ctx.final_res_batch[0].outputs.data,
|
||||||
|
torch.tensor([2.2, 3.4]),
|
||||||
|
)
|
||||||
|
assert ctx.final_res_batch[1].request_id == (
|
||||||
|
"embd-client-prompt-999-chunk-888-prompt-1"
|
||||||
|
)
|
||||||
|
assert ctx.final_res_batch[1].prompt_token_ids == [10, 11]
|
||||||
|
assert torch.allclose(
|
||||||
|
ctx.final_res_batch[1].outputs.data,
|
||||||
|
torch.tensor([9.0, 9.0]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestPreProcessCohereOnline:
|
class TestPreProcessCohereOnline:
|
||||||
"""Unit tests for EmbedIOProcessor._pre_process_cohere_online."""
|
"""Unit tests for EmbedIOProcessor._pre_process_cohere_online."""
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
from typing import Any, Literal, cast
|
from typing import Any, Literal, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -27,6 +28,7 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
|||||||
from ..base.io_processor import PoolingIOProcessor
|
from ..base.io_processor import PoolingIOProcessor
|
||||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||||
from ..typing import (
|
from ..typing import (
|
||||||
|
ChunkedEmbeddingMetadata,
|
||||||
OfflineInputsContext,
|
OfflineInputsContext,
|
||||||
PoolingChatLikeRequest,
|
PoolingChatLikeRequest,
|
||||||
PoolingCompletionLikeRequest,
|
PoolingCompletionLikeRequest,
|
||||||
@@ -46,6 +48,12 @@ from .protocol import (
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _ChunkedPromptAggregator:
|
||||||
|
weighted_sum: torch.Tensor | None = None
|
||||||
|
total_weight: int = 0
|
||||||
|
|
||||||
|
|
||||||
class EmbedIOProcessor(PoolingIOProcessor):
|
class EmbedIOProcessor(PoolingIOProcessor):
|
||||||
name = "embed"
|
name = "embed"
|
||||||
|
|
||||||
@@ -113,6 +121,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
max_model_len = self.model_config.max_model_len
|
max_model_len = self.model_config.max_model_len
|
||||||
chunked_engine_inputs: list[EngineInput] = []
|
chunked_engine_inputs: list[EngineInput] = []
|
||||||
prompt_request_ids: list[str] = []
|
prompt_request_ids: list[str] = []
|
||||||
|
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
|
||||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||||
token_ids = engine_input.get("prompt_token_ids", None)
|
token_ids = engine_input.get("prompt_token_ids", None)
|
||||||
if token_ids is None:
|
if token_ids is None:
|
||||||
@@ -132,9 +141,16 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
prompt_request_ids.append(
|
prompt_request_ids.append(
|
||||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||||
)
|
)
|
||||||
|
chunked_embedding_metadata.append(
|
||||||
|
ChunkedEmbeddingMetadata(
|
||||||
|
prompt_index=prompt_idx,
|
||||||
|
chunk_index=chunk_idx,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
ctx.engine_inputs = chunked_engine_inputs
|
ctx.engine_inputs = chunked_engine_inputs
|
||||||
ctx.prompt_request_ids = prompt_request_ids
|
ctx.prompt_request_ids = prompt_request_ids
|
||||||
|
ctx.chunked_embedding_metadata = chunked_embedding_metadata
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -142,66 +158,48 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
# Online aggregation for chunked requests to
|
# Online aggregation for chunked requests to
|
||||||
# minimize memory usage
|
# minimize memory usage
|
||||||
# Track aggregation state for each prompt
|
# Track aggregation state for each prompt
|
||||||
prompt_aggregators: dict[int, dict[str, Any]] = {}
|
if ctx.chunked_embedding_metadata is None:
|
||||||
short_prompts_results: dict[int, PoolingRequestOutput] = {}
|
raise ValueError("Chunked embedding metadata not available")
|
||||||
for result_idx, result in enumerate(ctx.final_res_batch):
|
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
|
||||||
if "-chunk-" not in result.request_id:
|
raise ValueError(
|
||||||
# Non-chunked result - extract prompt_idx from request_id
|
"Chunked embedding metadata count does not match result count"
|
||||||
parts = result.request_id.split("-")
|
)
|
||||||
try:
|
|
||||||
# Last part should be prompt index
|
|
||||||
prompt_idx = int(parts[-1])
|
|
||||||
except (ValueError, IndexError):
|
|
||||||
prompt_idx = result_idx # Fallback to result_idx
|
|
||||||
|
|
||||||
short_prompts_results[prompt_idx] = result
|
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
|
||||||
|
for result, chunk_metadata in zip(
|
||||||
|
ctx.final_res_batch, ctx.chunked_embedding_metadata
|
||||||
|
):
|
||||||
|
prompt_idx = chunk_metadata.prompt_index
|
||||||
|
aggregator = prompt_aggregators.setdefault(
|
||||||
|
prompt_idx, _ChunkedPromptAggregator()
|
||||||
|
)
|
||||||
|
|
||||||
|
# MEAN pooling with online weighted averaging
|
||||||
|
# Ensure result is PoolingRequestOutput
|
||||||
|
# for embedding processing
|
||||||
|
if not isinstance(result, PoolingRequestOutput):
|
||||||
|
raise ValueError(
|
||||||
|
f"Expected PoolingRequestOutput for "
|
||||||
|
f"chunked embedding, got "
|
||||||
|
f"{type(result).__name__}"
|
||||||
|
)
|
||||||
|
if result.prompt_token_ids is None:
|
||||||
|
raise ValueError(
|
||||||
|
"prompt_token_ids cannot be None for chunked processing"
|
||||||
|
)
|
||||||
|
|
||||||
|
weight = len(result.prompt_token_ids)
|
||||||
|
embedding_data = result.outputs.data
|
||||||
|
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
|
||||||
|
|
||||||
|
if aggregator.weighted_sum is None:
|
||||||
|
# First chunk
|
||||||
|
aggregator.weighted_sum = weighted_embedding
|
||||||
else:
|
else:
|
||||||
# Extract prompt_idx from chunked request_id
|
# Accumulate
|
||||||
parts = result.request_id.split("-")
|
aggregator.weighted_sum += weighted_embedding
|
||||||
try:
|
|
||||||
prompt_idx = int(parts[parts.index("prompt") + 1])
|
|
||||||
except (ValueError, IndexError):
|
|
||||||
# Fallback: extract from result_idx if parsing fails
|
|
||||||
prompt_idx = result_idx
|
|
||||||
|
|
||||||
# Initialize aggregator for this prompt if needed
|
aggregator.total_weight += weight
|
||||||
if prompt_idx not in prompt_aggregators:
|
|
||||||
prompt_aggregators[prompt_idx] = {
|
|
||||||
"weighted_sum": None,
|
|
||||||
"total_weight": 0,
|
|
||||||
"chunk_count": 0,
|
|
||||||
"request_id": result.request_id.split("-chunk-")[0],
|
|
||||||
}
|
|
||||||
|
|
||||||
aggregator = prompt_aggregators[prompt_idx]
|
|
||||||
|
|
||||||
# MEAN pooling with online weighted averaging
|
|
||||||
# Ensure result is PoolingRequestOutput
|
|
||||||
# for embedding processing
|
|
||||||
if not isinstance(result, PoolingRequestOutput):
|
|
||||||
raise ValueError(
|
|
||||||
f"Expected PoolingRequestOutput for "
|
|
||||||
f"chunked embedding, got "
|
|
||||||
f"{type(result).__name__}"
|
|
||||||
)
|
|
||||||
if result.prompt_token_ids is None:
|
|
||||||
raise ValueError(
|
|
||||||
"prompt_token_ids cannot be None for chunked processing"
|
|
||||||
)
|
|
||||||
|
|
||||||
weight = len(result.prompt_token_ids)
|
|
||||||
embedding_data = result.outputs.data
|
|
||||||
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
|
|
||||||
|
|
||||||
if aggregator["weighted_sum"] is None:
|
|
||||||
# First chunk
|
|
||||||
aggregator["weighted_sum"] = weighted_embedding
|
|
||||||
else:
|
|
||||||
# Accumulate
|
|
||||||
aggregator["weighted_sum"] += weighted_embedding
|
|
||||||
|
|
||||||
aggregator["total_weight"] += weight
|
|
||||||
aggregator["chunk_count"] += 1
|
|
||||||
|
|
||||||
if ctx.original_engine_inputs is None:
|
if ctx.original_engine_inputs is None:
|
||||||
raise ValueError("Original engine inputs not available")
|
raise ValueError("Original engine inputs not available")
|
||||||
@@ -216,8 +214,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
# Finalize MEAN aggregation for this chunked prompt
|
# Finalize MEAN aggregation for this chunked prompt
|
||||||
aggregator = prompt_aggregators[prompt_idx]
|
aggregator = prompt_aggregators[prompt_idx]
|
||||||
|
|
||||||
weighted_sum = aggregator["weighted_sum"]
|
weighted_sum = aggregator.weighted_sum
|
||||||
total_weight = aggregator["total_weight"]
|
total_weight = aggregator.total_weight
|
||||||
|
|
||||||
if (
|
if (
|
||||||
weighted_sum is not None
|
weighted_sum is not None
|
||||||
@@ -243,7 +241,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
|
|
||||||
original_token_ids = cast(list[int], token_ids)
|
original_token_ids = cast(list[int], token_ids)
|
||||||
pooling_request_output = PoolingRequestOutput(
|
pooling_request_output = PoolingRequestOutput(
|
||||||
request_id=aggregator["request_id"],
|
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
|
||||||
prompt_token_ids=original_token_ids,
|
prompt_token_ids=original_token_ids,
|
||||||
outputs=pooling_output_data,
|
outputs=pooling_output_data,
|
||||||
num_cached_tokens=0,
|
num_cached_tokens=0,
|
||||||
@@ -255,8 +253,6 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Failed to aggregate chunks for prompt {prompt_idx}"
|
f"Failed to aggregate chunks for prompt {prompt_idx}"
|
||||||
)
|
)
|
||||||
elif prompt_idx in short_prompts_results:
|
|
||||||
final_res_batch.append(short_prompts_results[prompt_idx])
|
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Result not found for prompt {prompt_idx}")
|
raise ValueError(f"Result not found for prompt {prompt_idx}")
|
||||||
|
|
||||||
|
|||||||
@@ -70,6 +70,12 @@ AnyPoolingResponse: TypeAlias = (
|
|||||||
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
|
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(kw_only=True)
|
||||||
|
class ChunkedEmbeddingMetadata:
|
||||||
|
prompt_index: int
|
||||||
|
chunk_index: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass(kw_only=True)
|
@dataclass(kw_only=True)
|
||||||
class PoolingServeContext(Generic[PoolingRequestT]):
|
class PoolingServeContext(Generic[PoolingRequestT]):
|
||||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||||
@@ -91,6 +97,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
|||||||
|
|
||||||
## for Long Text Embedding with Chunked Processing
|
## for Long Text Embedding with Chunked Processing
|
||||||
original_engine_inputs: Sequence[EngineInput] | None = None
|
original_engine_inputs: Sequence[EngineInput] | None = None
|
||||||
|
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
|
||||||
|
|
||||||
## for bi-encoder & late-interaction
|
## for bi-encoder & late-interaction
|
||||||
n_queries: int | None = None
|
n_queries: int | None = None
|
||||||
|
|||||||
Reference in New Issue
Block a user