[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-24 08:33:32 -04:00
committed by GitHub
parent 41ae917c12
commit 79bb395eea
10 changed files with 106 additions and 66 deletions
+1 -4
View File
@@ -3550,10 +3550,7 @@ class GPUModelRunner(
hidden_states=hidden_states, pooling_metadata=pooling_metadata
)
finished_mask = [
seq_len == prompt_len
for seq_len, prompt_len in zip(seq_lens_cpu, pooling_metadata.prompt_lens)
]
finished_mask = pooling_metadata.get_pooling_cursor().get_finished_mask()
raw_pooler_output = self.late_interaction_runner.postprocess_pooler_output(
raw_pooler_output=raw_pooler_output,
pooling_params=pooling_metadata.pooling_params,