[Pooling] Improve BGE-M3 sync pooling throughput by up to 3.13% (#53464)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -476,9 +476,7 @@ class InputBatch:
|
||||
|
||||
self.pooling_params[req_id] = pooling_params
|
||||
self.pooling_states[req_id] = pooling_states
|
||||
self.logits_processing_needs_token_ids[req_index] = (
|
||||
pooling_params.requires_token_ids
|
||||
)
|
||||
self.logits_processing_needs_token_ids[req_index] = False
|
||||
else:
|
||||
raise NotImplementedError("Unrecognized request type")
|
||||
|
||||
@@ -890,8 +888,8 @@ class InputBatch:
|
||||
not self.no_penalties
|
||||
or self.logits_processing_needs_token_ids[:num_reqs].any()
|
||||
)
|
||||
# The prompt tokens are used only for applying penalties or
|
||||
# step pooling during the sampling/pooling process.
|
||||
# The device prompt tokens are used only for applying penalties or
|
||||
# pooling methods that explicitly request GPU token IDs.
|
||||
# Hence copy these tensors only when there are requests which
|
||||
# need penalties/step_pooler to be applied.
|
||||
prompt_token_ids_cpu = (
|
||||
@@ -979,7 +977,7 @@ class InputBatch:
|
||||
|
||||
return PoolingMetadata(
|
||||
prompt_lens=self.num_prompt_tokens_cpu_tensor[: self.num_reqs].clone(),
|
||||
prompt_token_ids=self.sampling_metadata.prompt_token_ids,
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=prompt_token_ids_cpu,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=pooling_states,
|
||||
|
||||
Reference in New Issue
Block a user