[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-24 08:33:32 -04:00
committed by GitHub
parent 41ae917c12
commit 79bb395eea
10 changed files with 106 additions and 66 deletions
+4 -6
View File
@@ -476,9 +476,7 @@ class InputBatch:
self.pooling_params[req_id] = pooling_params
self.pooling_states[req_id] = pooling_states
self.logits_processing_needs_token_ids[req_index] = (
pooling_params.requires_token_ids
)
self.logits_processing_needs_token_ids[req_index] = False
else:
raise NotImplementedError("Unrecognized request type")
@@ -890,8 +888,8 @@ class InputBatch:
not self.no_penalties
or self.logits_processing_needs_token_ids[:num_reqs].any()
)
# The prompt tokens are used only for applying penalties or
# step pooling during the sampling/pooling process.
# The device prompt tokens are used only for applying penalties or
# pooling methods that explicitly request GPU token IDs.
# Hence copy these tensors only when there are requests which
# need penalties/step_pooler to be applied.
prompt_token_ids_cpu = (
@@ -979,7 +977,7 @@ class InputBatch:
return PoolingMetadata(
prompt_lens=self.num_prompt_tokens_cpu_tensor[: self.num_reqs].clone(),
prompt_token_ids=self.sampling_metadata.prompt_token_ids,
prompt_token_ids=None,
prompt_token_ids_cpu=prompt_token_ids_cpu,
pooling_params=pooling_params,
pooling_states=pooling_states,