[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-24 08:33:32 -04:00
committed by GitHub
parent 41ae917c12
commit 79bb395eea
10 changed files with 106 additions and 66 deletions
+1 -1
View File
@@ -18,7 +18,7 @@ ActivationFn = Callable[[_T], _T]
@dataclass(frozen=True)
class PoolingParamsUpdate:
requires_token_ids: bool = False
"""Set this flag to enable prompt token IDs for your pooler."""
"""Set this flag to enable CPU prompt token IDs for your pooler."""
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
return PoolingParamsUpdate(
@@ -69,7 +69,7 @@ class AllPool(TokenPoolingMethod):
return hidden_states_lst
pooling_states = pooling_metadata.pooling_states
finished_mask = pooling_cursor.is_finished().tolist()
finished_mask = pooling_cursor.get_finished_mask()
# If chunked_prefill is enabled
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
+24 -19
View File
@@ -12,11 +12,12 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.sequence import IntermediateTensors
from vllm.tasks import PoolingTask
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
from ..layers.pooler import DispatchPooler
@@ -104,33 +105,37 @@ class JinaForRankingPool(StepPool):
def get_supported_tasks(self) -> set[PoolingTask]:
return {"token_embed"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
pooled_data_lst = super().forward(hidden_states, pooling_metadata)
prompt_token_ids = pooling_metadata.get_prompt_token_ids()
prompt_token_ids_cpu = pooling_metadata.get_prompt_token_ids_cpu()
embeds_list = list[torch.Tensor | None]()
# `torch.where` resolves the match count on the host.
with gpu_sync_allowed():
for data, token_ids in zip(pooled_data_lst, prompt_token_ids):
# for unfinished chunked prefill
if data is None:
embeds_list.append(None)
else:
doc_match = torch.eq(token_ids, self.doc_token_id)
docs_indexes = torch.where(doc_match)[0]
query_indexes = torch.where(
torch.eq(token_ids, self.query_token_id)
)[0]
for data, token_ids_cpu in zip(pooled_data_lst, prompt_token_ids_cpu):
# for unfinished chunked prefill
if data is None:
embeds_list.append(None)
else:
doc_match = torch.eq(token_ids_cpu, self.doc_token_id)
docs_indexes_cpu = torch.where(doc_match)[0]
query_indexes_cpu = torch.where(
torch.eq(token_ids_cpu, self.query_token_id)
)[0]
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
indexes = torch.cat([docs_indexes, query_indexes])
embeds = self.projector(data[indexes])
embeds_list.append(embeds)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
indexes_cpu = torch.cat([docs_indexes_cpu, query_indexes_cpu])
indexes = async_tensor_h2d(
indexes_cpu, device=data.device, dtype=torch.long
)
embeds = self.projector(data[indexes])
embeds_list.append(embeds)
return embeds_list