[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-24 08:33:32 -04:00
committed by GitHub
parent 41ae917c12
commit 79bb395eea
10 changed files with 106 additions and 66 deletions
+23 -5
View File
@@ -17,22 +17,33 @@ class PoolingCursor:
prompt_lens_cpu: torch.Tensor
seq_lens_cpu: torch.Tensor
num_scheduled_tokens_cpu: torch.Tensor
partial_prefill: bool
finished_mask: list[bool]
def __getitem__(self, indices: slice) -> "PoolingCursor":
prompt_lens_cpu = self.prompt_lens_cpu[indices]
seq_lens_cpu = self.seq_lens_cpu[indices]
num_scheduled_tokens_cpu = self.num_scheduled_tokens_cpu[indices]
return PoolingCursor(
first_token_indices_gpu=self.first_token_indices_gpu[indices],
last_token_indices_gpu=self.last_token_indices_gpu[indices],
prompt_lens_cpu=self.prompt_lens_cpu[indices],
seq_lens_cpu=self.seq_lens_cpu[indices],
num_scheduled_tokens_cpu=self.num_scheduled_tokens_cpu[indices],
prompt_lens_cpu=prompt_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
)
def is_partial_prefill(self) -> bool:
return not torch.all(self.prompt_lens_cpu == self.num_scheduled_tokens_cpu)
return self.partial_prefill
def is_finished(self) -> torch.Tensor:
return self.prompt_lens_cpu == self.seq_lens_cpu
def get_finished_mask(self) -> list[bool]:
return self.finished_mask
class PoolingStates:
def __init__(self) -> None:
@@ -130,9 +141,14 @@ class PoolingMetadata:
)
num_scheduled_tokens_cpu = torch.from_numpy(num_scheduled_tokens_np)
prompt_lens_np = prompt_lens.numpy()
seq_lens_np = seq_lens_cpu.numpy()
if query_start_loc_gpu is None:
cumsum = torch.zeros(
n_seq + 1, dtype=torch.int64, pin_memory=PIN_MEMORY, device="cpu"
n_seq + 1,
dtype=torch.int64,
pin_memory=PIN_MEMORY,
device="cpu",
)
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
cumsum = cumsum.to(device, non_blocking=True)
@@ -155,4 +171,6 @@ class PoolingMetadata:
prompt_lens_cpu=prompt_lens,
seq_lens_cpu=seq_lens_cpu,
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
partial_prefill=not np.array_equal(prompt_lens_np, num_scheduled_tokens_np),
finished_mask=np.equal(prompt_lens_np, seq_lens_np).tolist(),
)