From 887cf91e30545373f90eb967c981b456c9cbbf8d Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Mon, 21 Sep 2026 09:46:39 -0500 Subject: [PATCH] [Pooling] Fix normalization of chunked long-text embeddings (#57498) Signed-off-by: Taneem Ibrahim --- .../entrypoints/pooling/embed/io_processor.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/pooling/embed/io_processor.py b/vllm/entrypoints/pooling/embed/io_processor.py index 57728c8837..3c5b75cee6 100644 --- a/vllm/entrypoints/pooling/embed/io_processor.py +++ b/vllm/entrypoints/pooling/embed/io_processor.py @@ -57,6 +57,8 @@ logger = init_logger(__name__) class _ChunkedPromptAggregator: weighted_sum: torch.Tensor | None = None total_weight: int = 0 + num_chunks: int = 0 + all_chunks_l2_normalized: bool = True class EmbedIOProcessor(PoolingIOProcessor): @@ -204,7 +206,17 @@ class EmbedIOProcessor(PoolingIOProcessor): weight = len(result.prompt_token_ids) embedding_data = result.outputs.data - weighted_embedding = embedding_data.to(dtype=torch.float32) * weight + float_embedding = embedding_data.to(dtype=torch.float32) + if ctx.pooling_params.use_activation: + # Custom activations need not L2-normalize; allow BF16 rounding. + embedding_norm = torch.linalg.vector_norm(float_embedding) + aggregator.all_chunks_l2_normalized &= torch.allclose( + embedding_norm, + torch.ones_like(embedding_norm), + rtol=1e-2, + atol=1e-5, + ) + weighted_embedding = float_embedding * weight if aggregator.weighted_sum is None: # First chunk @@ -214,6 +226,7 @@ class EmbedIOProcessor(PoolingIOProcessor): aggregator.weighted_sum += weighted_embedding aggregator.total_weight += weight + aggregator.num_chunks += 1 if ctx.original_engine_inputs is None: raise ValueError("Original engine inputs not available") @@ -239,6 +252,14 @@ class EmbedIOProcessor(PoolingIOProcessor): ): # Compute final mean embedding final_embedding = weighted_sum / total_weight + if ( + ctx.pooling_params.use_activation + and aggregator.num_chunks > 1 + and aggregator.all_chunks_l2_normalized + ): + final_embedding = torch.nn.functional.normalize( + final_embedding, dim=-1 + ) # Create a PoolingRequestOutput # for the aggregated result