[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -103,28 +103,26 @@ class PoolingRunner:
|
||||
pooling_params = [self.pooling_params[i] for i in req_indices]
|
||||
pooling_states = [self.pooling_states[i] for i in req_indices]
|
||||
prompt_lens = torch.from_numpy(
|
||||
req_states.prompt_len.np[input_batch.idx_mapping_np].copy()
|
||||
req_states.prompt_len.np[input_batch.idx_mapping_np]
|
||||
)
|
||||
|
||||
prompt_token_ids_cpu = None
|
||||
prompt_token_ids = None
|
||||
if any(params.requires_token_ids for params in pooling_params):
|
||||
max_prompt_len = int(prompt_lens.max())
|
||||
prompt_token_ids_cpu = torch.zeros(
|
||||
(input_batch.num_reqs, max_prompt_len),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
pin_memory=PIN_MEMORY and device.type != "cpu",
|
||||
)
|
||||
for i, (req_index, params) in enumerate(zip(req_indices, pooling_params)):
|
||||
if not params.requires_token_ids:
|
||||
continue
|
||||
token_ids = self.prompt_token_ids[req_index]
|
||||
prompt_token_ids_cpu[i, : token_ids.numel()] = token_ids
|
||||
prompt_token_ids = prompt_token_ids_cpu.to(device, non_blocking=True)
|
||||
|
||||
return PoolingMetadata(
|
||||
prompt_lens=prompt_lens,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=prompt_token_ids_cpu,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=pooling_states,
|
||||
@@ -150,7 +148,7 @@ class PoolingRunner:
|
||||
query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1],
|
||||
)
|
||||
pooler_output = self.model.pooler(hidden_states, pooling_metadata)
|
||||
finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist()
|
||||
finished_mask = pooling_metadata.get_pooling_cursor().get_finished_mask()
|
||||
pooler_output = self.late_interaction_runner.postprocess_pooler_output(
|
||||
raw_pooler_output=pooler_output,
|
||||
pooling_params=pooling_metadata.pooling_params,
|
||||
@@ -174,19 +172,16 @@ class PoolingRunner:
|
||||
pooling_params = PoolingParams(task=task)
|
||||
pooling_params.verify(self.model_config)
|
||||
self.model.pooler.get_pooling_updates(task).apply(pooling_params)
|
||||
prompt_token_ids = None
|
||||
prompt_token_ids_cpu = None
|
||||
if pooling_params.requires_token_ids:
|
||||
prompt_token_ids = torch.zeros(
|
||||
prompt_token_ids_cpu = torch.zeros(
|
||||
(num_reqs, int(prompt_lens.max())),
|
||||
dtype=torch.int64,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
pooling_metadata = PoolingMetadata(
|
||||
prompt_lens=prompt_lens,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_token_ids_cpu=None
|
||||
if prompt_token_ids is None
|
||||
else prompt_token_ids.cpu(),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=prompt_token_ids_cpu,
|
||||
pooling_params=[pooling_params] * num_reqs,
|
||||
pooling_states=[PoolingStates() for _ in range(num_reqs)],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user